pcgrenier.github.io/atom.xml at master · pcgrenier/pcgrenier.github.io · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000

<!DOCTYPE html>
<!--[if IEMobile 7 ]><html class="no-js iem7"><![endif]-->
<!--[if lt IE 9]><html class="no-js lte-ie8"><![endif]-->
<!--[if (gt IE 8)|(gt IEMobile 7)|!(IEMobile)|!(IE)]><!--><html class="no-js" lang="en"><!--<![endif]-->
<head>
  <meta charset="utf-8">
  <title>NiFi.rocks</title>
  <meta name="author" content="">

<meta name="description" content="How tos, videos, tips and tricks for Apache Nifi.">
<meta name="keywords" content="Apache Nifi, Nifi, nifi, nifi rocks">
  <!-- http://t.co/dKP3o1e -->
  <meta name="HandheldFriendly" content="True">
  <meta name="MobileOptimized" content="320">
  <meta name="viewport" content="width=device-width, initial-scale=1">


  <link rel="canonical" href="http://www.nifi.rocks/atom.xml">
  <link href="/favicon.png" rel="icon">
  <link href="/stylesheets/screen.css" media="screen, projection" rel="stylesheet" type="text/css">
  <link href="/atom.xml" rel="alternate" title="NiFi.rocks" type="application/atom+xml">
  <script src="/javascripts/modernizr-2.0.js"></script>
  <script src="//ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js"></script>
  <script>!window.jQuery && document.write(unescape('%3Cscript src="./javascripts/lib/jquery.min.js"%3E%3C/script%3E'))</script>
  <script src="/javascripts/octopress.js" type="text/javascript"></script>
  <!--Fonts from Google"s Web font directory at http://google.com/webfonts -->
<link href="//fonts.googleapis.com/css?family=PT+Serif:regular,italic,bold,bolditalic" rel="stylesheet" type="text/css">
<link href="//fonts.googleapis.com/css?family=PT+Sans:regular,italic,bold,bolditalic" rel="stylesheet" type="text/css">
<link href='//fonts.googleapis.com/css?family=Open+Sans' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Fjalla+One' rel='stylesheet' type='text/css'>


  <script type="text/javascript">
    var _gaq = _gaq || [];
    _gaq.push(['_setAccount', 'UA-57621921-1']);
    _gaq.push(['_trackPageview']);

    (function() {
      var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
      ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
      var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
    })();
  </script>


</head>

<body   class="collapse-sidebar sidebar-footer" >
  <header role="banner">
	<div class="header-title"><a href="/">NiFi.rocks</a></div>


	<br><div class="header-subtitle">Your home for everything Apache NiFi</div>

</header>
  <nav role="navigation">

<form action="https://www.google.com/search" method="get">
  <fieldset role="search">
    <input type="hidden" name="q" value="site:www.nifi.rocks" />
    <input class="search" type="text" name="q" results="0" placeholder="Search"/>
  </fieldset>
</form>

<ul class="main-navigation">
  <li><a href="/">Home</a></li>
  <li><a href="/archives">Archives</a></li>
  <li><a href="/about">About</a></li>
  <li><a href="/apache-nifi-processors/">Processors</a></li>
</ul>

</nav>
  <div id="main">
    <div id="content">
      <?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">

  <title><![CDATA[NiFi.rocks]]></title>
  <link href="http://www.nifi.rocks/atom.xml" rel="self"/>
  <link href="http://www.nifi.rocks/"/>
  <updated>2016-07-13T19:28:02+00:00</updated>
  <id>http://www.nifi.rocks/</id>
  <author>
    <name><![CDATA[]]></name>

  </author>
  <generator uri="http://octopress.org/">Octopress</generator>


  <entry>

    <title type="html"><![CDATA[Apache Nifi Release 0.7.0 Highlights]]></title>
    <link href="http://www.nifi.rocks/apache-nifi-release-0-dot-7-0-highlights/"/>

    <updated>2016-07-13T02:07:13+00:00</updated>
    <id>http://www.nifi.rocks/apache-nifi-release-0-dot-7-0-highlights</id>

    <content type="html"><![CDATA[<p>Apache Nifi&rsquo;s newest release is out, 0.7.0.  You can grab the binaries from <a href="https://nifi.apache.org/download.html">their site</a> as always.  So lets dive in and see what to look for in this release!</p>

<ul>
<li><a href="#bugfixes">Bug Fixes</a>:           72</li>
<li><a href="#improvements">Improvements</a>:    49</li>
<li><a href="#new-features">New Features</a>:    10</li>
</ul>


<p>As always, a bunch of bug fixes, but this time there are quite a few improvements.</p>

<!--more-->


<h3><a name="bugfixes"></a>Bug Fixes</h3>

<p>Where to begin!  This release has the second most bug fixes we&rsquo;ve seen.  Again, there are a ton to cover so take a peak at the full <a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12316020&amp;version=12335078">release notes</a> if you want to see more, since this is just a quick highlight.  Lots of processor fixes - NullPointerException to non-responsive processors, a few nifi startup fixes, and other fixes such as fixing OS specific issues, looking at you Windows bugs.  UI interaction was fixed when many HDFS and HBase processors were on the graph.  Bug fixes are hard to highlight, since they are so user specific for what you might have run into, but it is nice to know that reported bugs are getting fixed.</p>

<h3><a name="improvements"></a>Improvements</h3>

<p>I didn&rsquo;t know where to start for Bug fixes, but really don&rsquo;t know where to begin for Improvements.  I&rsquo;ll start by saying there are a few new processors that they included in this section, but I&rsquo;m going to list in the New Features below to remain consistent.  Other actual improvements were updates to HTTP Processors to allow proxy authentication which I know I saw someone on here asked about, so it&rsquo;s nice to see that it is now doable!  You are now able to use SSL with AMQP processors.  ExecuteScript processors can now be executed concurrently too.  Performance improvements in StreamScanner, SSL for the mongo processors, and the Mock framework can now register FlowFile Assertions.</p>

<p>It&rsquo;s good to mention that new documentation has been created for an in depth dive for developers, with Nifi and it&rsquo;s design decisions.  This is a great read and I will try to do a quick break down in the future for what points I think are really important for developers.</p>

<h3><a name="new-features"></a>New Features</h3>

<p>A few new processors here, but outside of that Apache Nifi now supports custom properties in the Expression Language.</p>

<p>New Processors:</p>

<ul>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#ConsumeKafka">ConsumeKafka</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#ConsumeMQTT">ConsumeMQTT</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#DebugFlow">DebugFlow</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#DeleteDynamoDB">DeleteDynamoDB</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#ExtractMediaMetadata">ExtractMediaMetadata</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#GetDynamoDB">GetDynamoDB</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#GetHDFSEvents">GetHDFSEvents</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#GetSNMP">GetSNMP</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#JoltTransformJSON">JoltTransformJSON</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#ListenLumberjack">ListenLumberjack</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#ListS3">ListS3</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#PublishKafka">PublishKafka</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#PublishMQTT">PublishMQTT</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#PutDynamoDB">PutDynamoDB</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#PutHiveQL">PutHiveQL</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#PutSlack">PutSlack</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#PutTCP">PutTCP</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#PutUDP">PutUDP</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#SelectHiveQL">SelectHiveQL</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#SetSNMP">SetSNMP</a></li>
</ul>


<p>Most of these could be seen in the pipleine since they had some similarties to current ones, for example AMQP or JMS consume and publish, and now adding MQTT to be supported.  Apache Nifi now has more support for AWS services, Amazon&rsquo;s Dynamo and S3, with the ListS3 and the Delete/Put DynamoDB Processors.  The biggest upgrade, I think, for processors is the DebugFlow processor, which allows you to produce a behavior you&rsquo;d like to test for a flow.  This could be transfering a flow file to a success/failure relationship, wanting to rollback a FlowFile to test behaviour without penalty, or just throwing an exception.
I encourage you to take a look at the descriptions of these procesors on our <a href="http://www.nifi.rocks/apache-nifi-processors/">Processors Page</a> which lists all current processors, and is updated after every release.</p>
]]></content>

  </entry>

  <entry>

    <title type="html"><![CDATA[Apache Nifi Release 0.6.0 Highlights]]></title>
    <link href="http://www.nifi.rocks/apache-nifi-release-0-dot-6-0-highlights/"/>

    <updated>2016-07-12T02:25:08+00:00</updated>
    <id>http://www.nifi.rocks/apache-nifi-release-0-dot-6-0-highlights</id>

    <content type="html"><![CDATA[<p>Apache Nifi release 0.6.0 and 0.6.1 recently and I wanted to go throught and highlight some of the key changes that may affect you Nifi users.  So far Nifi has kept pretty well to thier 6 week release schedule that they have discussed.  That&rsquo;s a really agressive release schedule so we&rsquo;ll see if they can keep that up once they hit the 0.7.0/1.0.0 release, which they say is going to actually be pushed back a little.  The 0.6.0 release was again substantial with 52 bug fixes, 15 improvements and 7 new features.  0.6.1 didn&rsquo;t take to long to follow with 11 bug fixes and 1 improvement.  The release binaries are on the <a href="https://nifi.apache.org/download.html">Apache Nifi download page</a>The full release notes are avaialble on <a href="https://cwiki.apache.org/confluence/display/NIFI/Release+Notes#ReleaseNotes-Version0.6.1">Apaches Nifi&rsquo;s wiki page</a>.  The combined 0.6.* release information is below.</p>

<ul>
<li><a href="#bugfixes">Bug Fixes</a>:           63</li>
<li><a href="#improvements">Improvements</a>:    16</li>
<li><a href="#new-features">New Features</a>:    7</li>
</ul>


<!--more-->


<h3><a name="bugfixes"></a>Bug Fixes</h3>

<p>So many bug fixes, yet again! Even though Apache Nifi has not released a major version, Apache Nifi is going strong and fixing those annoying issues.  There are quite a few bug fixes that resolve issues with Apache Nifi Processors, which is great since these are the main part of nifi; PutKafka, InferAvroSchema, GetHttp, ControlRate, PutElsaticSearch, GetKafka, ListFile, SplitText, all had fixes!  And this just touches on some of the bug fixes.  If you really want to see if a specific bug fix was completed, you can check <a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12316020&amp;version=12334372">here</a>, but really no need, just reset assured the bugs are getting fixed!</p>

<h3><a name="improvements"></a>Improvements</h3>

<p>The big improvements in this release are proxy support for the AWS Processors and RouteText to use Nifi Expression Language to route lines.  The other improvements are pretty good too, but mostly under the covers, for example ListHDFS to set the correct state in Zookeeper.</p>

<h3><a name="new-features"></a>New Features</h3>

<p>Let&rsquo;s get straight to them, new features an new processors:</p>

<ul>
<li>Processors for Apache <a href="http://www.nifi.rocks/apache-nifi-processors/#PutCassandraQL">Cassandra</a></li>
<li>Kerberos authentication</li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#PublishJMS">PublishJMS Processor</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#ConsumeJMS">ConsumeJMS Processor</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#PutLambda">AWS Lambda Processor</a></li>
</ul>


<p>That&rsquo;s pretty much it.  To see all processors, checkout our <a href="http://www.nifi.rocks/apache-nifi-processors/">processor page</a> which lists all current processors and their descriptions.  Check back after next release for a quick break down!</p>
]]></content>

  </entry>

  <entry>

    <title type="html"><![CDATA[Apache Nifi Release 0.5.0 Highlights]]></title>
    <link href="http://www.nifi.rocks/apache-nifi-release-0-dot-5-0-highlights/"/>

    <updated>2016-02-16T20:09:49+00:00</updated>
    <id>http://www.nifi.rocks/apache-nifi-release-0-dot-5-0-highlights</id>

    <content type="html"><![CDATA[<p>Apache Nifi just voted on another release and have annouced and released 0.5.0, probably their most interesting release to date.  This release contains quite a few big features that expand what nifi has to offer and also how you can use it; notably script execution in a flow.  This new processor allows you to execute scripts in Javascript, Groovy, JRuby, Jython, Lua, and Python.  To grab the latest binaries, go to
<a href="https://nifi.apache.org/download.html">Apache Nifi&rsquo;s download page</a>  The full release notes are also available on <a href="https://cwiki.apache.org/confluence/display/NIFI/Release+Notes#ReleaseNotes-Version0.5.0">Apache Nifi&rsquo;s wiki page</a>.</p>

<ul>
<li><a href="#bugfixes">Bug Fixes</a>:           64</li>
<li><a href="#improvements">Improvements</a>:    28</li>
<li><a href="#new-features">New Features</a>:    10</li>
</ul>


<p>This release is substantial and really expands Apache Nifi&rsquo;s usablity, specificall for those users who don&rsquo;t feel comfortable with Java.  Lets take a deeper look at what else is included!</p>

<!--more-->


<h3><a name="bugfixes"></a>Bug Fixes</h3>

<p>Since Nifi is quite a young project, less than a year old in the open source world, it still has quite a few bugs coming up as the user base expands.  It&rsquo;s great to see that as bugs come up, they are getting the attention of the community and developers.  Each release is getting better and better.  For this release, quite a few of the bugs are just under the covers, rewording error messages or documentation fixes.  There are also some functionality bugs that were corrected in this release; Controller Service sorting by type and state now works, PutS3 now allows larger than 5GB files, Unit tests are performing better - no time outs on certain environments, a few fixes to PutSQL processor for improved functionality also.  The full list of bugs addressed in this release can be found in <a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12316020&amp;version=12334158">Apache Nifi&rsquo;s Jira</a></p>

<h3><a name="improvements"></a>Improvements</h3>

<p>Apache Nifi Expression language suppport has been added to a few fields for some processors with this release - the MergeContent Processor&rsquo;s Correlation Attribute Field and the xxxS3Object SECRET_KEY and ACCESS_KEY properties.  The UI now deletes an object when it is selected and the backspace key is pressed, which is a step in the right direction to improve usability.  Their are talks of a new UI that is in the works to allow for better user role management and also use HTML5 and newer web technology for the UI.  Some other improvements were new method for the Expression Langauge, getDelimitedField, PutJMS now works as expected for the uri shcema specified.</p>

<h3><a name="new-features"></a>New Features</h3>

<p>Some big changes for this release!  As I talked about before, there are a few new processors - 14 total - and also one that was deprecated/removed - ConvertCSVToSql.  The big new processors are a new processor to execute scripts and allow them access to the flowfile attributes - the ExecuteScript processor.  There is also a new processor for writing events to Riemann - the PutRiemann processor.  ElasticSearch is also now suppported through it&rsquo;s own processors.  The last big processor for this release is Get/PutAMQP processors for the AMQP connections - similar to JMS for newer message servers such as rabbitMQ and <a href="http://qpid.apache.org/">Apache&rsquo;s QPID</a>.  For a full list of processors, check out our <a href="http://www.nifi.rocks/apache-nifi-processors/">full updated list of nifi processors</a>.</p>

<p>All 14 new processors, with links to their description:</p>

<ul>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#ConsumeAMQP">ConsumeAMQP</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#ConvertJSONToSQL">ConvertJSONToSQL</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#ExecuteScript">ExecuteScript</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#FetchDistributedMapCache">FetchDistributedMapCache</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#FetchElasticSearch">FetchElasticSearch</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#GetHTMLElement">GetHTMLElement</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#InferAvroShema">InferAvroShema</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#InvokeScriptedProcessor">InvokeScriptedProcessor</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#ListenRELP">ListenRELP</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#ModifyHTMLElement">ModifyHTMLElement</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#PutHTMLElement">PutHTMLElement</a></li>
<li><a href="http://www.nifi.rocks/apache-nifi-processors/#PutRiemann">PutRiemann</a></li>
</ul>


<p>Another big new feature in this release is the ability to inspect/interact with FlowFiles inside of the Apache Nifi UI - flow files inside of the flow!  This is so awesome!  You can now select a processor/connection and peak at what is happening in a sense and upload/remove files from a queue!  Exciting stuff!!</p>

<p>This release contained quite a few new and interesting additions that we plan on follwing up on with future posts, so be sure to check back to see how these new features can be used.</p>
]]></content>

  </entry>

  <entry>

    <title type="html"><![CDATA[Developing a Custom Apache Nifi Controller Service]]></title>
    <link href="http://www.nifi.rocks/developing-a-custom-apache-nifi-controller-service/"/>

    <updated>2016-02-07T00:37:57+00:00</updated>
    <id>http://www.nifi.rocks/developing-a-custom-apache-nifi-controller-service</id>

    <content type="html"><![CDATA[<p>Controller services are shared between processors, other controller services and reporting tasks. Normally they provide access to a shared resource, such as a database or ssl context, or externally managed content. This post will cover the basics of a controller service through a simple example. This example will take a file path that contains one or more properties files, and provide a processor access to those properties. The full source is hosted on <a href="https://github.com/pcgrenier/nifi-examples/tree/sample-processor">Github</a>.</p>

<!-- more -->


<h2>Setup</h2>

<p>This project will use a more advanced maven structure than the simple one used in the <a href="http://www.nifi.rocks//developing-a-custom-apache-nifi-processor-json">developing a custom processor</a> post.  If you have looked at the processor post you&rsquo;ll see that most of the setup for services is very similar.</p>

<figure class='code'><figcaption><span>Apache Nifi Controller Servcie Folder Structure </span></figcaption>
<div class="highlight"><table><tr><td class="gutter"><pre class="line-numbers"><span class='line-number'>1</span>
<span class='line-number'>2</span>
<span class='line-number'>3</span>
<span class='line-number'>4</span>
<span class='line-number'>5</span>
<span class='line-number'>6</span>
<span class='line-number'>7</span>
<span class='line-number'>8</span>
<span class='line-number'>9</span>
<span class='line-number'>10</span>
<span class='line-number'>11</span>
<span class='line-number'>12</span>
<span class='line-number'>13</span>
<span class='line-number'>14</span>
<span class='line-number'>15</span>
<span class='line-number'>16</span>
<span class='line-number'>17</span>
</pre></td><td class='code'><pre><code class=''><span class='line'>./sample-bundle
</span><span class='line'>├── pom.xml
</span><span class='line'>├── sample-bundle-nar
</span><span class='line'>│   ├── pom.xml
</span><span class='line'>│   └── src
</span><span class='line'>├── sample-controller-service
</span><span class='line'>│   ├── pom.xml
</span><span class='line'>│   └── src
</span><span class='line'>├── sample-controller-service-api
</span><span class='line'>│   ├── pom.xml
</span><span class='line'>│   └── src
</span><span class='line'>├── sample-controller-service-api-nar
</span><span class='line'>│   ├── pom.xml
</span><span class='line'>│   └── src
</span><span class='line'>└── sample-processor
</span><span class='line'>    ├── pom.xml
</span><span class='line'>    └── src</span></code></pre></td></tr></table></div></figure>


<p>I won&rsquo;t go into details on the pom files but the general idea is that you want a seperate nar for the api interface and the service itself, this allows something much smaller to be used as a dependency in your other bundles. So the sample-bundle-nar will pull in the sample-processor and sample-controller-service packages. The sample-controller-service-api-nar will just pull in the sample-controller-service-api.</p>

<h2>The Controller Service API Interface</h2>

<figure class='code'><figcaption><span>Controller Service API</span><a href='https://github.com/pcgrenier/nifi-examples/blob/sample-processor/sample-controller-service-api/src/main/java/rocks/nifi/examples/PropertiesFileService.java'>PropertiesFileService.java </a></figcaption><div class="highlight"><table><tr><td class="gutter"><pre class="line-numbers"><span class='line-number'>1</span>
<span class='line-number'>2</span>
<span class='line-number'>3</span>
</pre></td><td class='code'><pre><code class='java'><span class='line'><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">PropertiesFileService</span> <span class="kd">extends</span> <span class="n">ControllerService</span><span class="o">{</span>
</span><span class='line'>    <span class="n">String</span> <span class="nf">getProperty</span><span class="o">(</span><span class="n">String</span> <span class="n">key</span><span class="o">);</span>
</span><span class='line'><span class="o">}</span>
</span></code></pre></td></tr></table></div></figure>


<p>This is just a simple interface that extends nifi&rsquo;s ControllerService. We also provide the only entry point to processors, the getProperty function.  This is similar to the current services in Apache Nifi such as <a href="https://raw.githubusercontent.com/apache/nifi/master/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-api/src/main/java/org/apache/nifi/dbcp/DBCPService.java">DBCPService.java</a> providing the getConnection() function.</p>

<h2>The Controller Service</h2>

<p>If you have read the <a href="http://www.nifi.rocks//developing-a-custom-apache-nifi-processor-json">developing a custom processor</a> post a lot of this will be review. Controller services provide the same interfaces for configuration and validation. The initialization method only differs in taking a ControllerServiceInitializationContext.</p>

<p>Just like with the processors, tags are useful for finding your controller services. The capability description annotation provides a simple explanation of what the controller service will provide.</p>

<figure class='code'><figcaption><span>Controller Service</span><a href='https://github.com/pcgrenier/nifi-examples/blob/sample-processor/sample-controller-service/src/main/java/rocks/nifi/examples/StandardPropertiesFileService.java'>StandardPropertiesFileService.java </a></figcaption><div class="highlight"><table><tr><td class="gutter"><pre class="line-numbers"><span class='line-number'>1</span>
<span class='line-number'>2</span>
<span class='line-number'>3</span>
</pre></td><td class='code'><pre><code class='java'><span class='line'><span class="nd">@Tags</span><span class="o">({</span><span class="s">&quot;nifirocks&quot;</span><span class="o">,</span> <span class="s">&quot;properties&quot;</span><span class="o">})</span>
</span><span class='line'><span class="nd">@CapabilityDescription</span><span class="o">(</span><span class="s">&quot;Provides a controller service to manage property files.&quot;</span><span class="o">)</span>
</span><span class='line'><span class="kd">public</span> <span class="kd">class</span> <span class="nc">StandardPropertiesFileService</span> <span class="kd">extends</span> <span class="n">AbstractControllerService</span> <span class="kd">implements</span> <span class="n">PropertiesFileService</span><span class="o">{</span>
</span></code></pre></td></tr></table></div></figure>


<p>Next we create the property descriptors. One will take the file or directory holding the property files, the other will be how often to check. Unlike processors controller services do not contain relationships.</p>

<figure class='code'><figcaption><span>Controller Service</span><a href='https://github.com/pcgrenier/nifi-examples/blob/sample-processor/sample-controller-service/src/main/java/rocks/nifi/examples/StandardPropertiesFileService.java'>StandardPropertiesFileService.java </a></figcaption><div class="highlight"><table><tr><td class="gutter"><pre class="line-numbers"><span class='line-number'>1</span>
<span class='line-number'>2</span>
<span class='line-number'>3</span>
<span class='line-number'>4</span>
<span class='line-number'>5</span>
<span class='line-number'>6</span>
<span class='line-number'>7</span>
<span class='line-number'>8</span>
<span class='line-number'>9</span>
<span class='line-number'>10</span>
<span class='line-number'>11</span>
<span class='line-number'>12</span>
<span class='line-number'>13</span>
<span class='line-number'>14</span>
<span class='line-number'>15</span>
<span class='line-number'>16</span>
<span class='line-number'>17</span>
<span class='line-number'>18</span>
<span class='line-number'>19</span>
<span class='line-number'>20</span>
<span class='line-number'>21</span>
<span class='line-number'>22</span>
<span class='line-number'>23</span>
</pre></td><td class='code'><pre><code class='java'><span class='line'><span class="kd">public</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">PropertyDescriptor</span> <span class="n">CONFIG_URI</span> <span class="o">=</span> <span class="k">new</span>   <span class="n">PropertyDescriptor</span><span class="o">.</span><span class="na">Builder</span><span class="o">()</span>
</span><span class='line'>        <span class="o">.</span><span class="na">name</span><span class="o">(</span><span class="s">&quot;Configuration Directory&quot;</span><span class="o">)</span>
</span><span class='line'>        <span class="o">.</span><span class="na">description</span><span class="o">(</span><span class="s">&quot;Configuration directory for properties files.&quot;</span><span class="o">)</span>
</span><span class='line'>        <span class="o">.</span><span class="na">defaultValue</span><span class="o">(</span><span class="kc">null</span><span class="o">)</span>
</span><span class='line'>        <span class="o">.</span><span class="na">addValidator</span><span class="o">(</span><span class="n">StandardValidators</span><span class="o">.</span><span class="na">NON_EMPTY_VALIDATOR</span><span class="o">)</span>
</span><span class='line'>        <span class="o">.</span><span class="na">build</span><span class="o">();</span>
</span><span class='line'>
</span><span class='line'><span class="kd">public</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">PropertyDescriptor</span> <span class="n">RELOAD_INTERVAL</span> <span class="o">=</span> <span class="k">new</span> <span class="n">PropertyDescriptor</span><span class="o">.</span><span class="na">Builder</span><span class="o">()</span>
</span><span class='line'>        <span class="o">.</span><span class="na">name</span><span class="o">(</span><span class="s">&quot;Reload Interval&quot;</span><span class="o">)</span>
</span><span class='line'>        <span class="o">.</span><span class="na">description</span><span class="o">(</span><span class="s">&quot;Time before looking for changes&quot;</span><span class="o">)</span>
</span><span class='line'>        <span class="o">.</span><span class="na">defaultValue</span><span class="o">(</span><span class="s">&quot;60 min&quot;</span><span class="o">)</span>
</span><span class='line'>        <span class="o">.</span><span class="na">required</span><span class="o">(</span><span class="kc">true</span><span class="o">)</span>
</span><span class='line'>        <span class="o">.</span><span class="na">addValidator</span><span class="o">(</span><span class="n">StandardValidators</span><span class="o">.</span><span class="na">TIME_PERIOD_VALIDATOR</span><span class="o">)</span>
</span><span class='line'>        <span class="o">.</span><span class="na">build</span><span class="o">();</span>
</span><span class='line'>
</span><span class='line'><span class="kd">private</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">List</span><span class="o">&lt;</span><span class="n">PropertyDescriptor</span><span class="o">&gt;</span> <span class="n">serviceProperties</span><span class="o">;</span>
</span><span class='line'>
</span><span class='line'><span class="kd">static</span><span class="o">{</span>
</span><span class='line'>    <span class="kd">final</span> <span class="n">List</span><span class="o">&lt;</span><span class="n">PropertyDescriptor</span><span class="o">&gt;</span> <span class="n">props</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ArrayList</span><span class="o">&lt;&gt;();</span>
</span><span class='line'>    <span class="n">props</span><span class="o">.</span><span class="na">add</span><span class="o">(</span><span class="n">CONFIG_URI</span><span class="o">);</span>
</span><span class='line'>    <span class="n">props</span><span class="o">.</span><span class="na">add</span><span class="o">(</span><span class="n">RELOAD_INTERVAL</span><span class="o">);</span>
</span><span class='line'>    <span class="n">serviceProperties</span> <span class="o">=</span> <span class="n">Collections</span><span class="o">.</span><span class="na">unmodifiableList</span><span class="o">(</span><span class="n">props</span><span class="o">);</span>
</span><span class='line'><span class="o">}</span>
</span></code></pre></td></tr></table></div></figure>


<p>The next step is the onConfigured function which will read the properties set and call any other necessary functions needed to start the service.  We just read the two properties we have, configUri and reloadIntervalMilli, and then call loadPropertiesFile().  After the properties file is loaded, we start up a file watcher and executer so that the properties can be dynamic and not just read in at startup.</p>

<figure class='code'><figcaption><span>Conroller Service</span><a href='https://github.com/pcgrenier/nifi-examples/blob/sample-processor/sample-controller-service/src/main/java/rocks/nifi/examples/StandardPropertiesFileService.java'>StandardPropertiesFileService.java </a></figcaption><div class="highlight"><table><tr><td class="gutter"><pre class="line-numbers"><span class='line-number'>1</span>
<span class='line-number'>2</span>
<span class='line-number'>3</span>
<span class='line-number'>4</span>
<span class='line-number'>5</span>
<span class='line-number'>6</span>
<span class='line-number'>7</span>
<span class='line-number'>8</span>
<span class='line-number'>9</span>
<span class='line-number'>10</span>
<span class='line-number'>11</span>
<span class='line-number'>12</span>
<span class='line-number'>13</span>
<span class='line-number'>14</span>
<span class='line-number'>15</span>
</pre></td><td class='code'><pre><code class='java'><span class='line'><span class="nd">@OnEnabled</span>
</span><span class='line'><span class="kd">public</span> <span class="kt">void</span> <span class="nf">onConfigured</span><span class="o">(</span><span class="kd">final</span> <span class="n">ConfigurationContext</span> <span class="n">context</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">InitializationException</span><span class="o">{</span>
</span><span class='line'>    <span class="n">log</span><span class="o">.</span><span class="na">info</span><span class="o">(</span><span class="s">&quot;Starting properties file service&quot;</span><span class="o">);</span>
</span><span class='line'>    <span class="n">configUri</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="na">getProperty</span><span class="o">(</span><span class="n">CONFIG_URI</span><span class="o">).</span><span class="na">getValue</span><span class="o">();</span>
</span><span class='line'>    <span class="n">reloadIntervalMilli</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="na">getProperty</span><span class="o">(</span><span class="n">RELOAD_INTERVAL</span><span class="o">).</span><span class="na">asTimePeriod</span><span class="o">(</span><span class="n">TimeUnit</span><span class="o">.</span><span class="na">MILLISECONDS</span><span class="o">);</span>
</span><span class='line'>
</span><span class='line'>    <span class="c1">// Initialize the properties</span>
</span><span class='line'>    <span class="n">loadPropertiesFiles</span><span class="o">();</span>
</span><span class='line'>
</span><span class='line'>    <span class="n">fileWatcher</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SynchronousFileWatcher</span><span class="o">(</span><span class="n">Paths</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="n">configUri</span><span class="o">),</span> <span class="k">new</span> <span class="nf">LastModifiedMonitor</span><span class="o">());</span>
</span><span class='line'>    <span class="n">executor</span> <span class="o">=</span> <span class="n">Executors</span><span class="o">.</span><span class="na">newSingleThreadScheduledExecutor</span><span class="o">();</span>
</span><span class='line'>    <span class="n">FilesWatcherWorker</span> <span class="n">reloadTask</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">FilesWatcherWorker</span><span class="o">();</span>
</span><span class='line'>    <span class="n">executor</span><span class="o">.</span><span class="na">scheduleWithFixedDelay</span><span class="o">(</span><span class="n">reloadTask</span><span class="o">,</span> <span class="mi">0</span><span class="o">,</span> <span class="n">reloadIntervalMilli</span><span class="o">,</span> <span class="n">TimeUnit</span><span class="o">.</span><span class="na">MILLISECONDS</span><span class="o">);</span>
</span><span class='line'>
</span><span class='line'><span class="o">}</span>
</span></code></pre></td></tr></table></div></figure>


<p>To see the other private functions you can refer to the github code.</p>

<p>The last step in the process is to create a processor to use the service.  This is exactly the same as creating a normal processor, but in this instance we want to add some specifics to use the PropertiesFileService.  To do this, we just grab a reference from context of the Controller Service, and then call the getProperty(propertyName) function.  We are just going to get the property and add it to the nifi flowfile properties so it is available to other processors down the line for now.  To specify which property we want we will add a PropertyDescriptor so that the user can set it in the Nifi UI, and another PropertyDescriptor to specify which PropertiesFileService to get the property value from.</p>

<figure class='code'><figcaption><span>Controll Service Processor</span><a href='https://github.com/pcgrenier/nifi-examples/blob/sample-processor/sample-processor/src/main/java/rocks/nifi/examples/processors/ControllerServiceProcessor.java'>ControllerServiceProcessor.java </a></figcaption><div class="highlight"><table><tr><td class="gutter"><pre class="line-numbers"><span class='line-number'>1</span>
<span class='line-number'>2</span>
<span class='line-number'>3</span>
<span class='line-number'>4</span>
<span class='line-number'>5</span>
<span class='line-number'>6</span>
<span class='line-number'>7</span>
<span class='line-number'>8</span>
<span class='line-number'>9</span>
<span class='line-number'>10</span>
<span class='line-number'>11</span>
<span class='line-number'>12</span>
<span class='line-number'>13</span>
<span class='line-number'>14</span>
<span class='line-number'>15</span>
<span class='line-number'>16</span>
<span class='line-number'>17</span>
<span class='line-number'>18</span>
<span class='line-number'>19</span>
<span class='line-number'>20</span>
<span class='line-number'>21</span>
<span class='line-number'>22</span>
<span class='line-number'>23</span>
<span class='line-number'>24</span>
<span class='line-number'>25</span>
<span class='line-number'>26</span>
<span class='line-number'>27</span>
<span class='line-number'>28</span>
<span class='line-number'>29</span>
<span class='line-number'>30</span>
<span class='line-number'>31</span>
<span class='line-number'>32</span>
<span class='line-number'>33</span>
<span class='line-number'>34</span>
</pre></td><td class='code'><pre><code class='java'><span class='line'><span class="kd">public</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">PropertyDescriptor</span> <span class="n">PROPERTY_NAME</span> <span class="o">=</span> <span class="k">new</span> <span class="n">PropertyDescriptor</span><span class="o">.</span><span class="na">Builder</span><span class="o">()</span>
</span><span class='line'>            <span class="o">.</span><span class="na">name</span><span class="o">(</span><span class="s">&quot;Property Name&quot;</span><span class="o">)</span>
</span><span class='line'>            <span class="o">.</span><span class="na">required</span><span class="o">(</span><span class="kc">true</span><span class="o">)</span>
</span><span class='line'>            <span class="o">.</span><span class="na">addValidator</span><span class="o">(</span><span class="n">StandardValidators</span><span class="o">.</span><span class="na">NON_EMPTY_VALIDATOR</span><span class="o">)</span>
</span><span class='line'>            <span class="o">.</span><span class="na">build</span><span class="o">();</span>
</span><span class='line'>
</span><span class='line'>    <span class="kd">public</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">PropertyDescriptor</span> <span class="n">PROPERTIES_SERVICE</span> <span class="o">=</span> <span class="k">new</span> <span class="n">PropertyDescriptor</span><span class="o">.</span><span class="na">Builder</span><span class="o">()</span>
</span><span class='line'>            <span class="o">.</span><span class="na">name</span><span class="o">(</span><span class="s">&quot;Properties Service&quot;</span><span class="o">)</span>
</span><span class='line'>            <span class="o">.</span><span class="na">description</span><span class="o">(</span><span class="s">&quot;System properties loader&quot;</span><span class="o">)</span>
</span><span class='line'>            <span class="o">.</span><span class="na">required</span><span class="o">(</span><span class="kc">false</span><span class="o">)</span>
</span><span class='line'>            <span class="o">.</span><span class="na">identifiesControllerService</span><span class="o">(</span><span class="n">PropertiesFileService</span><span class="o">.</span><span class="na">class</span><span class="o">)</span>
</span><span class='line'>            <span class="o">.</span><span class="na">build</span><span class="o">();</span>
</span><span class='line'><span class="o">.......</span>
</span><span class='line'>
</span><span class='line'><span class="nd">@Override</span>
</span><span class='line'><span class="kd">public</span> <span class="kt">void</span> <span class="nf">onTrigger</span><span class="o">(</span><span class="kd">final</span> <span class="n">ProcessContext</span> <span class="n">context</span><span class="o">,</span> <span class="kd">final</span> <span class="n">ProcessSession</span> <span class="n">session</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">ProcessException</span> <span class="o">{</span>
</span><span class='line'>    <span class="kd">final</span> <span class="n">ProcessorLog</span> <span class="n">log</span> <span class="o">=</span> <span class="k">this</span><span class="o">.</span><span class="na">getLogger</span><span class="o">();</span>
</span><span class='line'>    <span class="kd">final</span> <span class="n">AtomicReference</span><span class="o">&lt;</span><span class="n">String</span><span class="o">&gt;</span> <span class="n">value</span> <span class="o">=</span> <span class="k">new</span> <span class="n">AtomicReference</span><span class="o">&lt;&gt;();</span>
</span><span class='line'>
</span><span class='line'>    <span class="kd">final</span> <span class="n">String</span> <span class="n">propertyName</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="na">getProperty</span><span class="o">(</span><span class="n">PROPERTY_NAME</span><span class="o">).</span><span class="na">getValue</span><span class="o">();</span>
</span><span class='line'>    <span class="kd">final</span> <span class="n">PropertiesFileService</span> <span class="n">propertiesService</span> <span class="o">=</span> <span class="n">context</span><span class="o">.</span><span class="na">getProperty</span><span class="o">(</span><span class="n">PROPERTIES_SERVICE</span><span class="o">).</span><span class="na">asControllerService</span><span class="o">(</span><span class="n">PropertiesFileService</span><span class="o">.</span><span class="na">class</span><span class="o">);</span>
</span><span class='line'>    <span class="kd">final</span> <span class="n">String</span> <span class="n">property</span> <span class="o">=</span> <span class="n">propertiesService</span><span class="o">.</span><span class="na">getProperty</span><span class="o">(</span><span class="n">propertyName</span><span class="o">);</span>
</span><span class='line'>    <span class="n">log</span><span class="o">.</span><span class="na">info</span><span class="o">(</span><span class="s">&quot;Property = &quot;</span> <span class="o">+</span> <span class="n">property</span><span class="o">);</span>
</span><span class='line'>
</span><span class='line'>
</span><span class='line'>    <span class="n">FlowFile</span> <span class="n">flowfile</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="na">get</span><span class="o">();</span>
</span><span class='line'>    <span class="c1">// Write the results to an attribute</span>
</span><span class='line'>
</span><span class='line'>    <span class="k">if</span><span class="o">(</span><span class="n">property</span> <span class="o">!=</span> <span class="kc">null</span> <span class="o">&amp;&amp;</span> <span class="o">!</span><span class="n">property</span><span class="o">.</span><span class="na">isEmpty</span><span class="o">()){</span>
</span><span class='line'>        <span class="n">flowfile</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="na">putAttribute</span><span class="o">(</span><span class="n">flowfile</span><span class="o">,</span> <span class="s">&quot;property&quot;</span><span class="o">,</span> <span class="n">property</span><span class="o">);</span>
</span><span class='line'>    <span class="o">}</span>
</span><span class='line'>
</span><span class='line'>    <span class="n">session</span><span class="o">.</span><span class="na">transfer</span><span class="o">(</span><span class="n">flowfile</span><span class="o">,</span> <span class="n">SUCCESS</span><span class="o">);</span>
</span><span class='line'><span class="o">}</span>
</span></code></pre></td></tr></table></div></figure>


<p>Once you have the actual service and processor written, if you have followed the directory setup in the source code, you are good to go!  Just build your service project(mvn clean install), from the root pom.xml directory.  Then copy the nar, from the sample-bundle-nar/target/ directory, into your Apache Nifi instance lib directory, .  Once copied, start/restart Apache Nifi and you now have your service available as usual to be used!</p>

<h2>Configuring the Service</h2>

<p>Once you have deployed the service nar bundle, go to the Controller Settings in the upper right of the web gui.</p>

<p><img src="http://www.nifi.rocks/images/nifi-controller-settings.png"></p>

<p>Then search or select the Controller Services tab and click the &lsquo;+&rsquo; button on the upper right of the model.  You can either search for the StandardPropertiesFileService, or just select it since there aren&rsquo;t many services.  From there configure the service as needed.</p>

<p><img src="http://www.nifi.rocks/images/controller-service.png"></p>

<p>Once the service is configured, add the ControllerServiceProcessor to the flow and configure the PropertyName and PropertyService, the name of the property that you want from the Java properties file and the PropertiesService that you just setup.</p>

<p><img src="http://www.nifi.rocks/images/controller-processor.png"></p>

<p>And that is pretty much it for configuring and setting up a service for use in a flow.  Now you just use the Flow file attribute as you would any other attribute.  This could be extended to grab multiple properties, maybe all of the from a file, and set them as Flow file attributes.  This is just a basic example showing how you can create a controller service that fits your needs.</p>

<p>If you have any questions about custom services, let us know below or at info@nifi.rocks!</p>
]]></content>

  </entry>

  <entry>

    <title type="html"><![CDATA[Apache Nifi Release 0.4.0 highlights]]></title>
    <link href="http://www.nifi.rocks/apache-nifi-release-0-dot-4-0-highlights/"/>

    <updated>2015-12-15T03:00:56+00:00</updated>
    <id>http://www.nifi.rocks/apache-nifi-release-0-dot-4-0-highlights</id>

    <content type="html"><![CDATA[<p>Apache Nifi kicked out thier second release since graduating this past Friday.  With the release of 0.4.0, they are one step closer to a 1.0.0 release which will contain some very interesting things.  While we look forward to the first big release of Apache Nifi, lets break down the changes in the current build.  As always, release 0.4.0 is available on <a href="https://nifi.apache.org/download.html">Apache Nifi&rsquo;s download page</a>.</p>

<ul>
<li><a href="#bugfixes">Bug Fixes</a>:           93</li>
<li><a href="#improvements">Improvements</a>:    44</li>
<li><a href="#new-features">New Features</a>:    13</li>
</ul>


<p>The release notes are available on <a href="https://cwiki.apache.org/confluence/display/NIFI/Release+Notes#ReleaseNotes-Version0.4.0">Nifi&rsquo;s confluence page</a> and a quick highlight can be found below.  First lets just say, this is the biggest release so far, with almost double the amount of bug fixes, and over double the amount of improvements and new features!</p>

<ul>
<li>General UI improvements to usability</li>
<li>Multiple Authentication Mechanisms</li>
<li>New Provenance event types and ability to searh provenance events</li>
<li>Idle CPU usage was reduced</li>
<li><a href="#newprocessors">New Processors!</a></li>
<li>Improved OS support</li>
</ul>


<!--more-->


<h3><a name="bugfixes"></a>Bug Fixes</h3>

<p>Bug fixes, bug fixes everywhere!  So many bug fixes that it&rsquo;s almost impossible to list them all here.  Some of the big ones include fixing quite a few issues with MergeContent (efficiencies with 10,000+ files, queue swapping and an issue with delimiters), improved S3 support with PutS3 and FetchS3 fixes (now works with java 8, logs the URL correctly, corrected provenance evnet) and HDFS now acts accordingly with failures (passes to failure relationship and acts as expected for permission failures).  There were also fixes with the database connection pooling and with the PutSQL supporting multiple data types and booleans.  ConvertJSONToSQL also had quite a few fixes including caching results and metadata.  For the full list, either fire up nifi and check it out or look at the release notes!</p>

<h3><a name="improvements"></a>Improvements</h3>

<p>There are a few new provenance events with this release, including download event, fetch event, and REMOTE_INVOCATION event.  ExecuteSQL can now be run periodically without an input FlowFile allowing for expanded use of SQL tasks.  InvokeHTTP has been improved, it now has unique ids across clusters and has additional unit tests. The management side of nifi has been refactored slightly, mainly to clean up the shell scripts and fix some whitespace bugs.  SSL support has been added to the PutS3 processor. It&rsquo;s good to see improvements in so many different areas of nifi!</p>

<h3><a name="new-features"></a>New Features</h3>

<p>The 13 new features include a few <a name="newprocessors"></a>new processors:</p>

<ul>
<li>AttributesToJSON</li>
<li>DeleteS3Object</li>
<li>ExtractAvroMetadata</li>
<li>FetchFile</li>
<li>FetchSFTP</li>
<li>GetAzureEventHub</li>
<li>GetCouchbaseKey</li>
<li>GetHBase</li>
<li>ListenSyslog</li>
<li>ListFile</li>
<li>ListSFTP</li>
<li>ParseSyslog</li>
<li>PutAzureEventHub</li>
<li>PutCouchbaseKey</li>
<li>PutDistributedMapCache</li>
<li>PutHBaseCell</li>
<li>PutHBaseJSON</li>
<li>PutSyslog</li>
<li>RouteText</li>
<li>SplitAvro</li>
<li>TailFile</li>
</ul>


<p>To see more information about all Nifi Processors, checkout our full list of <a href="http://www.nifi.rocks/apache-nifi-processors/">nifi processors and their quick descriptions</a>.  You can also check <a href="https://nifi.apache.org/docs.html">Nifi&rsquo;s Documentation</a> for a little more information.  In addition to the new processors, Nifi now supports LDAP authentication ontop of username/password and two way SSL.  An enhancement to the UI now allows users to drop queued FLowFiles.  Before you had to stop the processor, add a new processor with a failure to discard the file and restart Nifi just to clear a queue!  Now it&rsquo;s much easier!</p>

<p>There is talks already about the first major release of Apache Nifi, 1.0.0 and redoing some of the decisions they made which would break backwords compatability.  We look forward to their next release which seems to be about every 6-8 weeks.  See you guys then!</p>
]]></content>

  </entry>

  <entry>

    <title type="html"><![CDATA[Apache Nifi Release 0.3.0 highlights]]></title>
    <link href="http://www.nifi.rocks/apache-nifi-release-0-dot-3-0-highlights/"/>

    <updated>2015-09-29T01:17:47+00:00</updated>
    <id>http://www.nifi.rocks/apache-nifi-release-0-dot-3-0-highlights</id>

    <content type="html"><![CDATA[<p>Apache Nifi released their third release, <a href="https://nifi.apache.org/download.html">0.3.0</a> since graduating to a TLP.  It was a minor release but still included a descent number of changes.</p>

<ul>
<li><a href="#bugfixes">Bug Fixes</a>:           56</li>
<li><a href="#improvements">Improvements</a>:    20</li>
<li><a href="#newfeatures">New Features</a>:     5</li>
</ul>


<p>Release Highlights according to nifi can be found on their <a href="https://cwiki.apache.org/confluence/display/NIFI/Release+Notes#ReleaseNotes-Version0.3.0">confluence page</a>, but are also listed below.</p>

<ul>
<li>Performance improvements in handling large volumes of small files.</li>
<li>Performance improvements in Provenance repositories.</li>
<li>Added Reporting Task for ApacheTM AmbariTM.</li>
<li>Improved stability of nifi bootstrap.</li>
<li>Added Processors for working with images.</li>
<li>Support for interacting with Kerberos enabled Hadoop clusters</li>
<li>Added additional Avro capabilities - merging datafiles &amp; converting to json</li>
<li>Added processors for performing INSERT, UPDATE, DELETE statements against relational databases</li>
</ul>


<!--more-->


<h3><a name="bugfixes"></a>Bug Fixes</h3>

<p>The bug fixes in this version are plenty.  Quite a few were related to the management of a nifi instance, the starting and stopping and how the nifi.sh script works.  It&rsquo;s good to see that all parts of the code are getting worked on and not just the core nifi framework.  There were also quite a few fixes dealing with some of the current processors, most notably GetSFTP, GetFTP, GetKafka, GetTwitter, EncryptContent, ExecuteSQL, ExecuteFlumeSource, and the Nifi Spark Receiver.  Processor fixes are great since they are the major part of the flow!</p>

<h3><a name="improvements"></a>Improvements</h3>

<p>The 20 improvements is a gain substantial considering the quick turn around between release 0.2.1 and 0.3.0.  Documentation was updated which is always good, both the admin guide and user guide.</p>

<h3><a name="new-features"></a>New Features</h3>

<p>New features are always the exciting stuff that you&rsquo;ve been waiting for, and if you were waiting for Apache Flume support, this might just be the release for you!  The 5 new features are listed below:</p>

<ul>
<li>Add processors that can run Apache Flume sources/sinks</li>
<li>Create reporting task to deliver metrics to Apach Ambari</li>
<li>Add location bounding box filter to twitter processor</li>
<li>Create a NAR for handling images</li>
<li>Kerberos support for Hadoop processors</li>
</ul>


<p>As always, new features are warmly welcomed.  It&rsquo;s great to see the incorporation of more Apache projects into Apache Nifi processors, with both Flume and Ambari processors being included in this release.</p>

<p>The next release, 0.4.0, has 90 issues currently in <a href="https://issues.apache.org/jira/browse/NIFI/fixforversion/12333070/?selectedTab=com.atlassian.jira.jira-projects-plugin:version-issues-panel">Jira</a>.  I&rsquo;m not completely sure if Nifi has settled on firm release dates or not, but I imagine this one should be out in a month or two, if anything by the end of the year.</p>

<p>The full list of release notes can be found on <a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12316020&amp;version=12329653">Apache Nifi&rsquo;s Jira</a>  This release is available through Nifi&rsquo;s site <a href="https://nifi.apache.org/download.html">download&rsquo;s section</a>, along with their previous releases.</p>

<p>We&rsquo;ve missed a few of Nifi&rsquo;s releases, 0.2.0-incubating and 0.2.1 since they graduated but hopefully will continue to give you a break down of Nifi releases as they come!</p>
]]></content>

  </entry>

  <entry>

    <title type="html"><![CDATA[Apache Nifi Graduation]]></title>
    <link href="http://www.nifi.rocks/apache-nifi-graduation/"/>

    <updated>2015-09-29T01:17:11+00:00</updated>
    <id>http://www.nifi.rocks/apache-nifi-graduation</id>

    <content type="html"><![CDATA[<p>Apache Nifi graduated from a podling to a top level Apache Software Foundation project a few months ago, which is great news!  Congratulations to the NiFi team and all the contributors to making Apache Nifi great and meeting the requirements to move from an incubating project to a top level project.</p>

<!-- more -->


<h3>So&hellip;what does this mean?</h3>

<p>If you are not familiar with Apache, new projects are first accepted into the Apache incubator.  A list of these type of projects can be seen on <a href="http://incubator.apache.org/">Apache&rsquo;s Incubator Page</a>.  The move from a podling, a project that is in incubation, to a Top Level Project (TLP), means Nifi can be taken more seriously and now has Apache behind it.  It&rsquo;s not just some pet project where a guy decided to give out his source code; It is an Apache TLP, with the likes of Hadoop, Cassandra, etc.</p>

<p>So congratulations to Apache Nifi, looking forward to what will come next!</p>
]]></content>

  </entry>

  <entry>

    <title type="html"><![CDATA[Developing a Custom Apache Nifi Processor-Unit Tests (Part I)]]></title>
    <link href="http://www.nifi.rocks/developing-a-custom-apache-nifi-processor-unit-tests-partI/"/>

    <updated>2015-04-04T16:54:52+00:00</updated>
    <id>http://www.nifi.rocks/developing-a-custom-apache-nifi-processor-unit-tests-partI</id>

    <content type="html"><![CDATA[<p>The Apache Nifi framework has built in unit testing with Junit using test runners.  They have a few examples in their code base, but learning first hand really helps.  In this post we&rsquo;ll go over adding unit tests to the <a href="http://www.nifi.rocks/developing-a-custom-apache-nifi-processor-json/">JSON Processor</a> that we developed previously.</p>

<p>To start, we&rsquo;ll checkout the JSON Processor code from <a href="https://github.com/pcgrenier/nifi-examples">Github</a> and then open it up in your favorite text editor.  There is already a test package/folder in the project that contains a unit test, <a href="https://github.com/pcgrenier/nifi-examples/blob/master/src/test/java/rocks/nifi/examples/processors/JsonProcessorTest.java">rocks.nifi.examples.processors/JsonProcessorTest.java</a>.</p>

<p>When unit testing in Apache Nifi, there are a few items on top of the normal JUnit annotations that are required.  While you can unit test with just JUnit, using the built in method in Apache Nifi makes it much easier.  In a future post we&rsquo;ll show you how to unit test using Mockito and JUnit to test helper methods where actually invoking a full processor seems excessive, or if you just don&rsquo;t want to use Nifi&rsquo;s test runner.</p>

<!-- more -->


<p>The first thing to do to unit test Apache Nifi is grabbing the necessary dependencies from maven.</p>

<figure class='code'><figcaption><span>pom.xml</span><a href='https://github.com/pcgrenier/nifi-examples/blob/master/pom.xml#L42'>link</a></figcaption><div class="highlight"><table><tr><td class="gutter"><pre class="line-numbers"><span class='line-number'>1</span>
<span class='line-number'>2</span>
<span class='line-number'>3</span>
<span class='line-number'>4</span>
<span class='line-number'>5</span>
<span class='line-number'>6</span>
</pre></td><td class='code'><pre><code class='xml'><span class='line'>      <span class="nt">&lt;dependency&gt;</span>
</span><span class='line'>            <span class="nt">&lt;groupId&gt;</span>org.apache.nifi<span class="nt">&lt;/groupId&gt;</span>
</span><span class='line'>            <span class="nt">&lt;artifactId&gt;</span>nifi-mock<span class="nt">&lt;/artifactId&gt;</span>
</span><span class='line'>            <span class="nt">&lt;version&gt;</span>${nifi.version}<span class="nt">&lt;/version&gt;</span>
</span><span class='line'>            <span class="nt">&lt;scope&gt;</span>test<span class="nt">&lt;/scope&gt;</span>
</span><span class='line'>        <span class="nt">&lt;/dependency&gt;</span>
</span></code></pre></td></tr></table></div></figure>


<p>After including the nifi-mock dependency, there will be a few includes from the org.apache.nifi.utils package that will be necessary to import; mainly TestRunner, TestRunners and MockFlowFile.</p>

<p>Since we are using JUnit, the usual tags apply for defining a test function inside the class and we&rsquo;ll use the @Test for this one, although the other annotations can be used.  After adding the JUnit tag, there are a few requirements that Nifi puts on you in order to utilize the test runners they have provided.  The first is to create a test runner to use and then to set a flow file or flow file content inorder to run the processor.  In this example, we are using a ByteArrayInputStream as the flow file content, although you could use a JSON file in a resource folder just as well.</p>

<p>Once a test runner is created, you can set flow file properties on it using <code>runner.setProperties(PropertyDescriptor)</code> method and can enqueue a file using <code>runner.enqueue(content)</code>.  Then you can run the test runner and make assertions.</p>

<p>Nifi makes it easy to test assertions with some built in assertions for flowfiles.  You can test the flowfile was transfered to the appropriate relationship and get the flowfile to test for expected attributes and content.</p>

<p>Below is how we test the JSON Processor, with a typical test setup for Apache Nifi.</p>

<figure class='code'><figcaption><span>JSON Processor Unit Test</span><a href='https://github.com/pcgrenier/nifi-examples/blob/master/src/test/java/rocks/nifi/examples/processors/JsonProcessorTest.java'>JsonProcessorTest.java </a></figcaption><div class="highlight"><table><tr><td class="gutter"><pre class="line-numbers"><span class='line-number'>1</span>
<span class='line-number'>2</span>
<span class='line-number'>3</span>
<span class='line-number'>4</span>
<span class='line-number'>5</span>
<span class='line-number'>6</span>
<span class='line-number'>7</span>
<span class='line-number'>8</span>
<span class='line-number'>9</span>
<span class='line-number'>10</span>
<span class='line-number'>11</span>
<span class='line-number'>12</span>
<span class='line-number'>13</span>
<span class='line-number'>14</span>
<span class='line-number'>15</span>
<span class='line-number'>16</span>
<span class='line-number'>17</span>
<span class='line-number'>18</span>
<span class='line-number'>19</span>
<span class='line-number'>20</span>
<span class='line-number'>21</span>
<span class='line-number'>22</span>
<span class='line-number'>23</span>
<span class='line-number'>24</span>
<span class='line-number'>25</span>
<span class='line-number'>26</span>
<span class='line-number'>27</span>
<span class='line-number'>28</span>
<span class='line-number'>29</span>
<span class='line-number'>30</span>
<span class='line-number'>31</span>
<span class='line-number'>32</span>
</pre></td><td class='code'><pre><code class='java'><span class='line'><span class="nd">@org.junit.Test</span>
</span><span class='line'>    <span class="kd">public</span> <span class="kt">void</span> <span class="nf">testOnTrigger</span><span class="o">()</span> <span class="kd">throws</span> <span class="n">IOException</span> <span class="o">{</span>
</span><span class='line'>        <span class="c1">// Content to be mock a json file</span>
</span><span class='line'>        <span class="n">InputStream</span> <span class="n">content</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">ByteArrayInputStream</span><span class="o">(</span><span class="s">&quot;{\&quot;hello\&quot;:\&quot;nifi rocks\&quot;}&quot;</span><span class="o">.</span><span class="na">getBytes</span><span class="o">());</span>
</span><span class='line'>
</span><span class='line'>        <span class="c1">// Generate a test runner to mock a processor in a flow</span>
</span><span class='line'>        <span class="n">TestRunner</span> <span class="n">runner</span> <span class="o">=</span> <span class="n">TestRunners</span><span class="o">.</span><span class="na">newTestRunner</span><span class="o">(</span><span class="k">new</span> <span class="nf">JsonProcessor</span><span class="o">());</span>
</span><span class='line'>
</span><span class='line'>        <span class="c1">// Add properites</span>
</span><span class='line'>        <span class="n">runner</span><span class="o">.</span><span class="na">setProperty</span><span class="o">(</span><span class="n">JsonProcessor</span><span class="o">.</span><span class="na">JSON_PATH</span><span class="o">,</span> <span class="s">&quot;$.hello&quot;</span><span class="o">);</span>
</span><span class='line'>
</span><span class='line'>        <span class="c1">// Add the content to the runner</span>
</span><span class='line'>        <span class="n">runner</span><span class="o">.</span><span class="na">enqueue</span><span class="o">(</span><span class="n">content</span><span class="o">);</span>
</span><span class='line'>
</span><span class='line'>        <span class="c1">// Run the enqueued content, it also takes an int = number of contents queued</span>
</span><span class='line'>        <span class="n">runner</span><span class="o">.</span><span class="na">run</span><span class="o">(</span><span class="mi">1</span><span class="o">);</span>
</span><span class='line'>
</span><span class='line'>        <span class="c1">// All results were processed with out failure</span>
</span><span class='line'>        <span class="n">runner</span><span class="o">.</span><span class="na">assertQueueEmpty</span><span class="o">();</span>
</span><span class='line'>
</span><span class='line'>        <span class="c1">// If you need to read or do aditional tests on results you can access the content</span>
</span><span class='line'>        <span class="n">List</span><span class="o">&lt;</span><span class="n">MockFlowFile</span><span class="o">&gt;</span> <span class="n">results</span> <span class="o">=</span> <span class="n">runner</span><span class="o">.</span><span class="na">getFlowFilesForRelationship</span><span class="o">(</span><span class="n">JsonProcessor</span><span class="o">.</span><span class="na">SUCCESS</span><span class="o">);</span>
</span><span class='line'>        <span class="n">assertTrue</span><span class="o">(</span><span class="s">&quot;1 match&quot;</span><span class="o">,</span> <span class="n">results</span><span class="o">.</span><span class="na">size</span><span class="o">()</span> <span class="o">==</span> <span class="mi">1</span><span class="o">);</span>
</span><span class='line'>        <span class="n">MockFlowFile</span> <span class="n">result</span> <span class="o">=</span> <span class="n">results</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="mi">0</span><span class="o">);</span>
</span><span class='line'>        <span class="n">String</span> <span class="n">resultValue</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">String</span><span class="o">(</span><span class="n">runner</span><span class="o">.</span><span class="na">getContentAsByteArray</span><span class="o">(</span><span class="n">result</span><span class="o">));</span>
</span><span class='line'>        <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">&quot;Match: &quot;</span> <span class="o">+</span> <span class="n">IOUtils</span><span class="o">.</span><span class="na">toString</span><span class="o">(</span><span class="n">runner</span><span class="o">.</span><span class="na">getContentAsByteArray</span><span class="o">(</span><span class="n">result</span><span class="o">)));</span>
</span><span class='line'>
</span><span class='line'>        <span class="c1">// Test attributes and content</span>
</span><span class='line'>        <span class="n">result</span><span class="o">.</span><span class="na">assertAttributeEquals</span><span class="o">(</span><span class="n">JsonProcessor</span><span class="o">.</span><span class="na">MATCH_ATTR</span><span class="o">,</span> <span class="s">&quot;nifi rocks&quot;</span><span class="o">);</span>
</span><span class='line'>        <span class="n">result</span><span class="o">.</span><span class="na">assertContentEquals</span><span class="o">(</span><span class="s">&quot;nifi rocks&quot;</span><span class="o">);</span>
</span><span class='line'>
</span><span class='line'>    <span class="o">}</span>
</span></code></pre></td></tr></table></div></figure>


<p>Watch for our next post about unit testing Apache Nifi with Mockito and JUnit without using nifi&rsquo;s testrunner class.</p>
]]></content>

  </entry>

  <entry>

    <title type="html"><![CDATA[Apache Nifi Release 0.0.2 Highlights]]></title>
    <link href="http://www.nifi.rocks/apache-nifi-release-0-dot-0-2-highlights/"/>

    <updated>2015-03-20T01:45:11+00:00</updated>
    <id>http://www.nifi.rocks/apache-nifi-release-0-dot-0-2-highlights</id>

    <content type="html"><![CDATA[<p>Apache Nifi just kicked out their second release, <a href="https://nifi.apache.org/download.html">0.0.2</a>.  It doesn&rsquo;t have too many new features in it, hence a patch release according to <a href="http://semver.org">semantic versioning</a>, but it is definitely an improvement over the previous version, with it&rsquo;s bug fixes, improvements, and new features numbers below.</p>

<ul>
<li><a href="#bugfixes">Bug Fixes</a>:       35</li>
<li><a href="#improvements">Improvements</a>:        17</li>
<li><a href="#new-features">New Features</a>:        5</li>
</ul>


<!--more-->


<h3><a name="bugfixes"></a>Bug Fixes</h3>

<p>The nifi community came in with some big fixes with this release.  The biggest ones I feel are the ability to now build on Mac OSX and increasing documentation.  There were quite a few other fixes that greatly improved the overall NiFi experience, processors now stop correctly and aren&rsquo;t continued to be scheduled and some clustering issues were resolved for example.</p>

<h3><a name="improvements"></a>Improvements</h3>

<p>The improvements in this release included a few maintanence related tasks and again some over all usability tasks, with most of them falling on the usability side.  The main improvements in Nifi release 0.0.2 were document related and overall usability updates, the cleanup of development documents, and too many to name for the usability.  In this release though, you should expect the following usability improvements:</p>

<ul>
<li>proxy support for the GetFTP Processor</li>
<li>Component toolbox improvements</li>
<li>Dragging/drawing relationships</li>
<li>Labeling with the same color for multiple items</li>
</ul>


<h3><a name="new-features"></a>New Features</h3>

<p>Now on to the good stuff, new features!  This release only had a few additional features added, but from what I&rsquo;ve read there are quite a few more coming.  The highlights in this category are:</p>

<p>Multiple new processors
* JSON Processors
* ExecuteProcess processor
* StoreInKiteDataset processor</p>

<p>New processors are great and mean more possibilities and use cases for Apache Nifi.  We showed you how to create a JSON processor in a previous post, <a href="http://www.nifi.rocks/developing-a-custom-apache-nifi-processor-json">Developing a custom Nifi Processor: JSON</a>, and now someone has contributed a JSON processor to Nifi.  For a full list of Nifi Processors, including the newly added ones, see our <a href="http://www.nifi.rocks/apache-nifi-processors">previous post</a>.</p>

<p>It has been rumored that the next release will contain quite a few changes and be a minor release bumping up to version 0.1.0.  With over 200 tickets open on their JIRA, Nifi could see a great amount of change in the coming months.</p>

<p>The full list of release notes can be found on Apache Nifi&rsquo;s <a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12316020&amp;version=12329373">JIRA page</a> and downloads for all of their releases, source code and binaries, are available from their <a href="https://nifi.apache.org/download.html">website</a>.  Nifi is also making thier releases available through maven via apaches <a href="https://repository.apache.org/content/repositories/releases/org/apache/nifi/">maven repository</a>.</p>

<p>As always, check back for how-tos and Apache Nifi related updates.  New videos and posts are coming!!</p>
]]></content>

  </entry>

  <entry>

    <title type="html"><![CDATA[Developing A Custom Apache Nifi Processor (JSON)]]></title>
    <link href="http://www.nifi.rocks/developing-a-custom-apache-nifi-processor-json/"/>

    <updated>2015-02-07T02:14:33+00:00</updated>
    <id>http://www.nifi.rocks/developing-a-custom-apache-nifi-processor-json</id>

    <content type="html"><![CDATA[<p>The list of available Apache Nifi processors is extensive, as documented in <a href="http://www.nifi.rocks/apache-nifi-processors/">this post</a>. There is still a need to develop your own; to pull data from a database, to process an uncommon file format, or many other unique situations. So to get you started, we will work through a basic processor that takes a json file as input and a json path as a parameter to place into the contents and an attribute. The full source is hosted on <a href="https://github.com/pcgrenier/nifi-examples">Github</a>.</p>

<!-- more -->


<h2>Setup</h2>

<p>Start by creating a simple maven project in your favorite IDE. Then edit the pom.xml.</p>

<div><script src='https://gist.github.com/f99d27d08c3903f9d50c.js?file=pom.xml'></script>
<noscript><pre><code>&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;
&lt;project xmlns=&quot;http://maven.apache.org/POM/4.0.0&quot; xmlns:xsi=&quot;http://www.w3.org/2001/XMLSchema-instance&quot; xsi:schemaLocation=&quot;http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd&quot;&gt;
    &lt;modelVersion&gt;4.0.0&lt;/modelVersion&gt;
    &lt;groupId&gt;rocks.nifi&lt;/groupId&gt;
    &lt;artifactId&gt;examples&lt;/artifactId&gt;
    &lt;version&gt;1.0-SNAPSHOT&lt;/version&gt;
    &lt;packaging&gt;nar&lt;/packaging&gt;

    &lt;properties&gt;
        &lt;project.build.sourceEncoding&gt;UTF-8&lt;/project.build.sourceEncoding&gt;
        &lt;maven.compiler.source&gt;1.7&lt;/maven.compiler.source&gt;
        &lt;maven.compiler.target&gt;1.7&lt;/maven.compiler.target&gt;
        &lt;nifi.version&gt;0.0.1-incubating&lt;/nifi.version&gt;
    &lt;/properties&gt;

    &lt;dependencies&gt;
        &lt;dependency&gt;
            &lt;groupId&gt;org.apache.nifi&lt;/groupId&gt;
            &lt;artifactId&gt;nifi-api&lt;/artifactId&gt;
            &lt;version&gt;${nifi.version}&lt;/version&gt;
        &lt;/dependency&gt;
        &lt;dependency&gt;
            &lt;groupId&gt;org.apache.nifi&lt;/groupId&gt;
            &lt;artifactId&gt;nifi-utils&lt;/artifactId&gt;
            &lt;version&gt;${nifi.version}&lt;/version&gt;
        &lt;/dependency&gt;
        &lt;dependency&gt;
            &lt;groupId&gt;org.apache.nifi&lt;/groupId&gt;
            &lt;artifactId&gt;nifi-processor-utils&lt;/artifactId&gt;
            &lt;version&gt;${nifi.version}&lt;/version&gt;
        &lt;/dependency&gt;
        &lt;dependency&gt;
            &lt;groupId&gt;org.apache.commons&lt;/groupId&gt;
            &lt;artifactId&gt;commons-io&lt;/artifactId&gt;
            &lt;version&gt;1.3.2&lt;/version&gt;
        &lt;/dependency&gt;
        &lt;dependency&gt;
            &lt;groupId&gt;com.jayway.jsonpath&lt;/groupId&gt;
            &lt;artifactId&gt;json-path&lt;/artifactId&gt;
            &lt;version&gt;1.2.0&lt;/version&gt;
        &lt;/dependency&gt;
        &lt;dependency&gt;
            &lt;groupId&gt;org.apache.nifi&lt;/groupId&gt;
            &lt;artifactId&gt;nifi-mock&lt;/artifactId&gt;
            &lt;version&gt;${nifi.version}&lt;/version&gt;
            &lt;scope&gt;test&lt;/scope&gt;
        &lt;/dependency&gt;
        &lt;dependency&gt;
            &lt;groupId&gt;junit&lt;/groupId&gt;
            &lt;artifactId&gt;junit&lt;/artifactId&gt;
            &lt;version&gt;4.10&lt;/version&gt;
            &lt;scope&gt;test&lt;/scope&gt;
        &lt;/dependency&gt;
    &lt;/dependencies&gt;

    &lt;build&gt;
        &lt;plugins&gt;
            &lt;plugin&gt;
                &lt;groupId&gt;org.apache.nifi&lt;/groupId&gt;
                &lt;artifactId&gt;nifi-nar-maven-plugin&lt;/artifactId&gt;
                &lt;version&gt;1.0.0-incubating&lt;/version&gt;
                &lt;extensions&gt;true&lt;/extensions&gt;
            &lt;/plugin&gt;
            &lt;plugin&gt;
                &lt;groupId&gt;org.apache.maven.plugins&lt;/groupId&gt;
                &lt;artifactId&gt;maven-surefire-plugin&lt;/artifactId&gt;
                &lt;version&gt;2.15&lt;/version&gt;
            &lt;/plugin&gt;
        &lt;/plugins&gt;
    &lt;/build&gt;
&lt;/project&gt;</code></pre></noscript></div>


<p>This pom.xml includes a single plug-in for building a nifi nar, which is similar to a war for nifi, that bundles everything up in a way nifi can unpack. The nifi-api is the only other &ldquo;required&rdquo; dependency. The other nifi dependencies are really use full as you will see.</p>

<p>The next important piece is telling nifi which classes to load and register. This is done in a single file located at /src/main/resources/META-INF/services/org.apache.nifi.processor.Processor</p>

<div><script src='https://gist.github.com/f98e563e787c1b73c425.js?file=org.apache.nifi.processor.Processor'></script>
<noscript><pre><code>rocks.nifi.examples.processors.JsonProcessor</code></pre></noscript></div>


<h2>The JSON Processor</h2>

<p>Now that everything is defined and findable by Apache Nifi, lets build a processor. Define a simple java class as defined in the setup process (rocks.nifi.examples.processors.JsonProcessor).</p>

<p>Tags are useful for finding your processor in the list of processors in the GUI. So in this case in the search box you could just type &lsquo;json&rsquo; and your processor will be found. The capability description is also displayed in the processor selection box. Nifi.rocks will make future posts on documenting your custom processors. Finally most processors will just extend the AbstractProcessor, for more complicated tasks it may be required to go a level deeper for the AbstractSessionFactoryProcessor.</p>

<figure class='code'><figcaption><span>Apache Nifi Processor Header </span><a href='https://github.com/pcgrenier/nifi-examples/blob/master/src/main/java/rocks/nifi/examples/processors/JsonProcessor.java'>JsonProcessor.java </a></figcaption><div class="highlight"><table><tr><td class="gutter"><pre class="line-numbers"><span class='line-number'>1</span>
<span class='line-number'>2</span>
<span class='line-number'>3</span>
<span class='line-number'>4</span>
</pre></td><td class='code'><pre><code class='java'><span class='line'><span class="nd">@SideEffectFree</span>
</span><span class='line'><span class="nd">@Tags</span><span class="o">({</span><span class="s">&quot;JSON&quot;</span><span class="o">,</span> <span class="s">&quot;NIFI ROCKS&quot;</span><span class="o">})</span>
</span><span class='line'><span class="nd">@CapabilityDescription</span><span class="o">(</span><span class="s">&quot;Fetch value from json path.&quot;</span><span class="o">)</span>
</span><span class='line'><span class="kd">public</span> <span class="kd">class</span> <span class="nc">JsonProcessor</span> <span class="kd">extends</span> <span class="n">AbstractProcessor</span> <span class="o">{</span>
</span></code></pre></td></tr></table></div></figure>


<p>Not really interesting stuff here. Properties will hold all a list of all the available properties tha are exposed to the user. Relationships will hold the relationships the processor will use to direct the flow files. For more details on relationships, properties, and components of an Apache Nifi flow please read the <a href="https://nifi.apache.org/developer-guide.html">offical developer guide</a>. There is plenty of room to expand on custom validators, but there is a large selection of validators in nifi-processor-utils package.</p>

<figure class='code'><figcaption><span>Variable Declaration</span><a href='https://github.com/pcgrenier/nifi-examples/blob/master/src/main/java/rocks/nifi/examples/processors/JsonProcessor.java'>JsonProcessor.java </a></figcaption><div class="highlight"><table><tr><td class="gutter"><pre class="line-numbers"><span class='line-number'>1</span>
<span class='line-number'>2</span>
<span class='line-number'>3</span>
<span class='line-number'>4</span>
<span class='line-number'>5</span>
<span class='line-number'>6</span>
<span class='line-number'>7</span>
<span class='line-number'>8</span>
<span class='line-number'>9</span>
<span class='line-number'>10</span>
<span class='line-number'>11</span>
<span class='line-number'>12</span>
<span class='line-number'>13</span>
<span class='line-number'>14</span>