From 3d8813709b87e93c081d17b484875d4dfc0c3b3d Mon Sep 17 00:00:00 2001 From: Dian Fu Date: Wed, 2 Jan 2019 11:06:31 +0800 Subject: [PATCH 001/147] [FLINK-11246][table] Fix distinct AGG visibility issue This closes #7394 --- .../scala/org/apache/flink/table/expressions/aggregations.scala | 2 +- .../flink/table/functions/DistinctAggregateFunction.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala index c77bd7a41ba17f..f6ea4bd8c28cf8 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/aggregations.scala @@ -56,7 +56,7 @@ abstract sealed class Aggregation extends Expression { case class DistinctAgg(child: Expression) extends Aggregation { - private[flink] def distinct: Expression = DistinctAgg(child) + def distinct: Expression = DistinctAgg(child) override private[flink] def resultType: TypeInformation[_] = child.resultType diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/DistinctAggregateFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/DistinctAggregateFunction.scala index c75e6faf1077ed..bd221c208fb98c 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/DistinctAggregateFunction.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/DistinctAggregateFunction.scala @@ -28,7 +28,7 @@ import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils.{getAccum private[flink] case class DistinctAggregateFunction[T: TypeInformation, ACC: TypeInformation] (aggFunction: AggregateFunction[T, ACC]) { - private[flink] def distinct(params: Expression*): Expression = { + def distinct(params: Expression*): Expression = { val resultTypeInfo: TypeInformation[_] = getResultTypeOfAggregateFunction( aggFunction, implicitly[TypeInformation[T]]) From 7ca3e0413a1ff52b6ece16163d0d6f538fb85152 Mon Sep 17 00:00:00 2001 From: leesf <490081539@qq.com> Date: Wed, 2 Jan 2019 18:23:05 +0800 Subject: [PATCH 002/147] [FLINK-11217][docs] Add missing back-to-top buttons --- docs/_includes/generated/rest_v1_dispatcher.html | 2 ++ docs/dev/java_lambdas.md | 4 +++- docs/dev/libs/ml/index.md | 2 ++ docs/dev/stream/experimental.md | 2 ++ docs/dev/stream/operators/joining.md | 2 ++ docs/dev/stream/operators/process_function.md | 2 ++ docs/dev/stream/state/broadcast_state.md | 2 ++ docs/dev/stream/state/schema_evolution.md | 2 ++ docs/dev/table/streaming/index.md | 2 ++ docs/dev/table/streaming/joins.md | 2 ++ docs/dev/table/streaming/match_recognize.md | 2 ++ docs/dev/table/streaming/temporal_tables.md | 2 ++ docs/examples/index.md | 4 +++- docs/ops/deployment/hadoop.md | 2 ++ 14 files changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/_includes/generated/rest_v1_dispatcher.html b/docs/_includes/generated/rest_v1_dispatcher.html index 4977e5ccfeb3a2..2fd3dcfe40c9a9 100644 --- a/docs/_includes/generated/rest_v1_dispatcher.html +++ b/docs/_includes/generated/rest_v1_dispatcher.html @@ -3871,3 +3871,5 @@ + +{% top %} diff --git a/docs/dev/java_lambdas.md b/docs/dev/java_lambdas.md index 0b4a5cc069d7bb..bc9704b517d0be 100644 --- a/docs/dev/java_lambdas.md +++ b/docs/dev/java_lambdas.md @@ -135,4 +135,6 @@ public static class DoubleTuple extends Tuple2 { this.f1 = f1; } } -{% endhighlight %} \ No newline at end of file +{% endhighlight %} + +{% top %} diff --git a/docs/dev/libs/ml/index.md b/docs/dev/libs/ml/index.md index fbe3dce3dc6111..f8a75a45e50e9c 100644 --- a/docs/dev/libs/ml/index.md +++ b/docs/dev/libs/ml/index.md @@ -146,3 +146,5 @@ If one wants to chain a `Predictor` to a `Transformer` or a set of chained `Tran The Flink community welcomes all contributors who want to get involved in the development of Flink and its libraries. In order to get quickly started with contributing to FlinkML, please read our official [contribution guide]({{site.baseurl}}/dev/libs/ml/contribution_guide.html). + +{% top %} diff --git a/docs/dev/stream/experimental.md b/docs/dev/stream/experimental.md index db029befb36bd2..f0a78a539126fb 100644 --- a/docs/dev/stream/experimental.md +++ b/docs/dev/stream/experimental.md @@ -77,3 +77,5 @@ Code example: .addSink(new DiscardingSink[Int]) env.execute() {% endhighlight %} + +{% top %} diff --git a/docs/dev/stream/operators/joining.md b/docs/dev/stream/operators/joining.md index c415f63f7815c2..bdd8e427280262 100644 --- a/docs/dev/stream/operators/joining.md +++ b/docs/dev/stream/operators/joining.md @@ -280,3 +280,5 @@ orangeStream + +{% top %} diff --git a/docs/dev/stream/operators/process_function.md b/docs/dev/stream/operators/process_function.md index b2a373eaa5fe2f..f0189f9eeec4bd 100644 --- a/docs/dev/stream/operators/process_function.md +++ b/docs/dev/stream/operators/process_function.md @@ -372,3 +372,5 @@ ctx.timerService.deleteEventTimeTimer(timestampOfTimerToStop) Note Stopping a timer has no effect if no such timer with the given timestamp is registered. + +{% top %} diff --git a/docs/dev/stream/state/broadcast_state.md b/docs/dev/stream/state/broadcast_state.md index 628a6308b57929..73f4dcf7fe4e03 100644 --- a/docs/dev/stream/state/broadcast_state.md +++ b/docs/dev/stream/state/broadcast_state.md @@ -275,3 +275,5 @@ manner. - **No RocksDB state backend:** Broadcast state is kept in-memory at runtime and memory provisioning should be done accordingly. This holds for all operator states. + +{% top %} diff --git a/docs/dev/stream/state/schema_evolution.md b/docs/dev/stream/state/schema_evolution.md index f2f52b22ad74f4..fd0a73c3e1dc0d 100644 --- a/docs/dev/stream/state/schema_evolution.md +++ b/docs/dev/stream/state/schema_evolution.md @@ -91,3 +91,5 @@ Flink fully supports evolving schema of Avro type state, as long as the schema c One limitation is that Avro generated classes used as the state type cannot be relocated or have different namespaces when the job is restored. + +{% top %} diff --git a/docs/dev/table/streaming/index.md b/docs/dev/table/streaming/index.md index 95c6bab8e0a50c..a6fa525a3dbbc8 100644 --- a/docs/dev/table/streaming/index.md +++ b/docs/dev/table/streaming/index.md @@ -40,3 +40,5 @@ Where to go next? * [Joins in Continuous Queries]({{ site.baseurl }}/dev/table/streaming/joins.html): Different supported types of Joins in Continuous Queries. * [Temporal Tables]({{ site.baseurl }}/dev/table/streaming/temporal_tables.html): Describes the Temporal Table concept. * [Query configuration]({{ site.baseurl }}/dev/table/streaming/query_configuration.html): Lists Table API & SQL specific configuration options. + +{% top %} diff --git a/docs/dev/table/streaming/joins.md b/docs/dev/table/streaming/joins.md index 508e8c70221c3e..a572c1be293cf4 100644 --- a/docs/dev/table/streaming/joins.md +++ b/docs/dev/table/streaming/joins.md @@ -221,3 +221,5 @@ applied updates according to the primary key until this point in time. By definition of event time, [watermarks]({{ site.baseurl }}/dev/event_time.html) allow the join operation to move forward in time and discard versions of the build table that are no longer necessary because no incoming row with lower or equal timestamp is expected. + +{% top %} diff --git a/docs/dev/table/streaming/match_recognize.md b/docs/dev/table/streaming/match_recognize.md index 33afc27e75af7b..4090440f8645c0 100644 --- a/docs/dev/table/streaming/match_recognize.md +++ b/docs/dev/table/streaming/match_recognize.md @@ -960,3 +960,5 @@ Unsupported features include: * `MATCH_RECOGNIZE` is supported only for SQL. There is no equivalent in the Table API. * Aggregations: * distinct aggregations are not supported. + +{% top %} diff --git a/docs/dev/table/streaming/temporal_tables.md b/docs/dev/table/streaming/temporal_tables.md index 4ebb4a6e8a775f..d511c1f93f4594 100644 --- a/docs/dev/table/streaming/temporal_tables.md +++ b/docs/dev/table/streaming/temporal_tables.md @@ -186,3 +186,5 @@ which allows us to use the function `rates` in the [Table API](../tableApi.html# Line `(2)` registers this function under the name `Rates` in our table environment, which allows us to use the `Rates` function in [SQL](../sql.html#joins). + +{% top %} diff --git a/docs/examples/index.md b/docs/examples/index.md index a8472026b7c4e0..f7242241167e77 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -45,4 +45,6 @@ There are also a few blog posts published online that discuss example applicatio * [Building real-time dashboard applications with Apache Flink, Elasticsearch, and Kibana](https://www.elastic.co/blog/building-real-time-dashboard-applications-with-apache-flink-elasticsearch-and-kibana) is a blog post at elastic.co showing how to build a real-time dashboard solution for streaming data analytics using Apache Flink, Elasticsearch, and Kibana. -* The [Flink training website](http://training.data-artisans.com/) from data Artisans has a number of examples. Check out the hands-on sections and the exercises. \ No newline at end of file +* The [Flink training website](http://training.data-artisans.com/) from data Artisans has a number of examples. Check out the hands-on sections and the exercises. + +{% top %} diff --git a/docs/ops/deployment/hadoop.md b/docs/ops/deployment/hadoop.md index f2f060c076f863..b07f9259e688d0 100644 --- a/docs/ops/deployment/hadoop.md +++ b/docs/ops/deployment/hadoop.md @@ -45,3 +45,5 @@ export HADOOP_CLASSPATH=`hadoop classpath` {% endhighlight %} in the shell. Note that `hadoop` is the hadoop binary and that `classpath` is an argument that will make it print the configured Hadoop classpath. + +{% top %} From a4c09d337a0699f650740cc483e6ed30179ac7df Mon Sep 17 00:00:00 2001 From: hequn8128 Date: Wed, 19 Dec 2018 10:20:42 +0800 Subject: [PATCH 003/147] [FLINK-10779][tests] Update Java / Scala StatefulJobSavepointMigrationITCase for 1.7 --- .../StatefulJobSavepointMigrationITCase.java | 4 +++- .../_metadata | Bin 0 -> 47107 bytes .../_metadata | Bin 0 -> 86417 bytes .../_metadata | Bin 0 -> 229848 bytes .../_metadata | Bin 0 -> 229848 bytes .../StatefulJobSavepointMigrationITCase.scala | 4 +++- 6 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink1.7-rocksdb-savepoint/_metadata create mode 100644 flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink1.7-savepoint/_metadata create mode 100644 flink-tests/src/test/resources/stateful-scala-udf-migration-itcase-flink1.7-jobmanager-savepoint/_metadata create mode 100644 flink-tests/src/test/resources/stateful-scala-udf-migration-itcase-flink1.7-rocksdb-savepoint/_metadata diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java index fd107058b8ae47..4f3d274a3819f3 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java @@ -82,7 +82,9 @@ public static Collection> parameters () { Tuple2.of(MigrationVersion.v1_5, StateBackendLoader.MEMORY_STATE_BACKEND_NAME), Tuple2.of(MigrationVersion.v1_5, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME), Tuple2.of(MigrationVersion.v1_6, StateBackendLoader.MEMORY_STATE_BACKEND_NAME), - Tuple2.of(MigrationVersion.v1_6, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME)); + Tuple2.of(MigrationVersion.v1_6, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME), + Tuple2.of(MigrationVersion.v1_7, StateBackendLoader.MEMORY_STATE_BACKEND_NAME), + Tuple2.of(MigrationVersion.v1_7, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME)); } private final MigrationVersion testMigrateVersion; diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink1.7-rocksdb-savepoint/_metadata b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink1.7-rocksdb-savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..4acb9a97718361720286980b0498860e2da3c65d GIT binary patch literal 47107 zcmeHQOKc=Z8E!kfd2IHujzUmEqBz1N0#tT)SM{T`yk^%UGOTxPd1hB60hy|B`$rhI4 z$%(C9Mwk$ch)~8v1X72t$A#y3aOfF0hW=ObrIbEnHo)$(GcG*`L5ya=V^R;}4uo)+sOTG7+- zYQ6DU*r-oO&9${=V>;Pb*JqRZYI|CWww|sgt$Jg*s#|rjT0f^-Q~d?ihFEW}G?TRD zqq}I?vL^1;Ej=mqq-u%fW}|)AEk(gv6Do?*+ltZ+olN%B+)>mdD;jFriEbyI!=Qt( z{OV`lTY2Hd*B)IscjVQtx{Jr5$c$()YaU{Li0%>D#bQ=aX6Y&8f6G(|$I>>E?Amg!Us(a2?T*gft8Y z^J7hfA5s!1%>v({0SmNcUH5A^;#zamxK$ZA01t|XMAyK!&NhB(Lwmg0 zi0jM6c%G|1{|{&qx-|aM(=S^G+bwJG=#JL7=^iX&a1TD*vOc zidH?@s94vo>c(=i@_=!z+zn3oUgC^5tEzLA>S zYHjPf7x(4gU>Lp#JDBZ1=W~9aUc)__b>r5;z^k@x@(!3IQ@vBBcy7S88{fsuGnajT z%#AVYP%7O?L#%ZkIM$I9<@?IBx$Ldow*cZcm9gpH9xQJ6+aw{$ z0lvaukuI!|ViD#tn^G1K3Oog2l1XCV3E_%*ifL&(z7L1>W07lyVUYswz%d#7p}fHM zI36=5W$#Bv>j5LE01y)*$-qtfRse`1KE#(6n5{sY4L1>2U*IIA^S0%lA9UP z)_Zl3{TQkjLH5H;m9F=hU15<6`MgUpNr5sZ+rH}zu)!fti5{i7q@=w8K{5b3LoEd0`9ATMBKIs<8eed z1vZAU$P2h9LvHAo8~)U#@ENlX8f8lh&YS*4Sp5;G{u$WqR`@&%e1SWqpAYDrW}_840rN!YDs2N`IV4x~U}J{*{*Z~@iR15B0P46-;P^X4EHPor~n_~7N z{`?_?;u0oGefsiH>T2Y%_MO3znXnV*^J!~H9kZBq!S{epsNi~X$!SFB6xh0WKBbzt z)B(xq#U9~-unE|{s8~zJBLy0!Lla21ph@d?)Ae3Fn9jNU9pjyC(u#Bo(k)20Al=gC z@qzepcOC-h4X~4Vy)w3WAhZiS*ny3knIu0ns19~te*PmjJ_2>4m^`QSjOW;zISRCh zE|p+r#3(pfffHA#LO;;lwL!567D>jD59t}CXONyjdIsqkL%L*ckh^ohLwaVYnbDi( za+k`}iQG6P-6*~add6%|7@=yGLLi%UvBMAx;V2S9I6n9&1>m~OsPBVuu;Y^mJlz$S zI*?QVe^WyK2Kk$v0A@P(VD`a01!vzqb_Cp4F3HH>Ab*4W4e~e0-=uLib_Xysyn_7A zPU7{7@i*IWn4xHRfH!V|M9Mh?T-3+f2XS;NNcgja`D@*wvt!yyi_9XE< z3YObZk-5L3Cf6c&agnuz0Zm(M&}f_Wd}BHW|F+y9H(>9-? zH(q?E^w3|PTY3KzKl%E7zrPL+8O;aZv^_9da(dE%CO&D+Kr#qOotF+gR(|f0-+c4k zzx>d*Ua4G!eczIP{Oej0`;@Rc9(kQ1%JNQ!|`6aao`-*
H8+V=tlc`q;s zHKx75z>k>rVo2^Hba;BB4o_F3h7QwSq>&W1;kL`)MY~LUQLscH{I@iZ=J)FgLM?5Z zMT{vTK?eybvTOx8)gac69eCQ&Dg=8T2(GgYiwq=)3>GOE5crv55!VG9c2`lNBt)BJ zVFW?0Jvb9=xt=1w~`&aNm}M1}-H11T;XxDCQ)vZKXv71c<*a>F0^_k8Zs8se3lo zrhahGWQ=2gyW5x512}11$k>_~6Bz6qBP8p3r*W?(dabLztLLh3vKysqj z8`3REw`>uz7{myJ7-0}23}S>qj4+5%pIq#vZgMhXA1AXhG)vygLLGO_W>(Pc_139_ z-A%BfufQ&&m^^1esJ%!?*w|L$0j(JxrI57*?c2GsD16(fp8omJPr0bE@OmqkAcgQ(iGxT z`&^nNVi<%PqZaL$R5qWI7_|ta7KwEcA?4ZkYBxv*W%tZgaLb^%Olx_2ZkSQjNLMCW z4VufKxlH|>Zova+8?M~Uo=S$lF-yVkjBTcymkmOFF59L1JLP*KC>h1AEaiJ3Xbx~X z0VGB7f$Cuj;Healc`T-Wpja47DJYubpo=@YxTA|Zy10+WXpS=qj(Xdf<6Ow4Xd$1? zy5M_;1fjNr*mt~8fxA-x4&Sj0@uveRh;4gJDiuSLcyPZ$a~ym(#dlLQ$3b&$H0MTh zZZzlKQ~a3m1(;K8ryLJghSmd!SkN^c*tnVD$Zm6-PF-$})2_cloD<)8;o^T@ez^UZ zv;42T9pi`JVQw$wOV*q00XZw$RAYacySZ3x4 zFcS|#;+ov@EQE~mB4(V3$mAL0f$PM;@QevK2E9;!?F%s+E^! zE7jU!<=E+3W&YIC$@$r`b)>p5SE+4Pluf^^BWFr;r^`##@?xblSGm8u2&LmztvNbD zTTNQ^#&T7+>SEQ9XjAXx3AdQ!E-az1NmtqB!H e>1{>nhE67XYVIg%k`)a#?L@be&SB6&SpE+w2_hN* literal 0 HcmV?d00001 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink1.7-savepoint/_metadata b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink1.7-savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..c450c0c6130e05ea12e79384b70bfd42ffcfd264 GIT binary patch literal 86417 zcmeHQU5p!76~1QwW5^}NJSMDN&%_m+?lx^ z&v+cn?q==XoRRDq-#c?>eCPh0^WE>9Q~G92B7`L2!?y$8j@)+7m%j1OE41>$m3v;l z?@;~+zk>7xd`A2|KPI308$X6E;brczTPv2qB-^156W8$kkNFpzcDuj7$KMwcWZcNn zypc6Bl5WbXq*6I2X?j(bC^gkWK~rTpr^BUZ;F3{zkkCDNneJz7cCI)-RhgY%C|9N` z#f9loxiVimcDhoUJ+*LhcDhJLPoIKg_vC$Nre;nT7s|!?($q}pTyY*!lcdtJYAL!* zjYXC+8};S|_)$+8t)-<_Gv%xJlwKVN+H^-rFhyS(q??<@1iA&I|H@yd6=aU$}{A&C$-gJ-(G$j+Df`!w>^Or{`9 z>8z2K)LfoQS}tFeWZlS{vPSb}zCg&%H{g=b@XL~<6k9*%=|$R6@77AwsV}iqWo5a+ zf|@_xYMS+$sP4<OR~RpC zq(nxkRd-fPra>~eT59dA$@irGKI|LsJBHfs}Z(j@#DHTKVRwf1ny!9O6EmU+cMoA_kK@*lL1 z&J!+fvhdj5&;Ihx|2)O(*DVj;SE0WLbGzY8tFse!KAlC5{U7CoAFB*qyTP1RbM?8$ zpMNd&NRdC@x5&52$VH3X7h9~?oOqpk?Nbh(B{4&iX4)a2?RX2b`gvwJ{59O$S8+jU zem(KJ_Y?mHhu`sk|JiTzLt2L$)asICF1FUX8gEP+84ouZF>XNpCD+mNc(o5l+enaI z?&K>q=~8>ZA^T1g&lacssj+f)4rW7l3UvQ={A@UERZI??nmSo5&rKcgnz$eAuQkk3 z3+lvSR)g7gxW{{wNHBSjcl?f&>nd7ILvAgn&a~>)Q&6f6VnfFW>~~Fn-q_%gXoGrD zf~@_|d1H?lfm|to|79)>kYq180Y*_VXt{ENPFcp#m?|lTsz_=kTaXG_Lz8G;S81N+ z(m729IpKY{4CMrr6Hrd*man2>2k(Edx#Ny2Pw^ja5+Nuj+*qXx3kE1BETEi#a)LX4 ztX)pvQ$Clhdu?KDa`!uW2@V2efKpt+LCffw0w6aTabxcl!%o}5lMuK)JXSFgeiI~OYWadd5pC$%98U*b6O1T0i&nyzPLGb3r5 znw3;lp%OJrMat!}X@h3vtj_X;9C;cpM*w1r0}${IZ^MO(dMb(-;6g>*V$TLatSCqH zmLrnn@B&}YwH90?ePY>a8K7^&+whV@V|M%!#hET}-gYcTm!J%H>AtnhEb6o@J5}K~ z3KmGmBfopCA@8{AF9T?6y%YQkzQLS0a*ph+FDXUN_QY&Cc~hve3} z?0W}a^@*%S@P*67MlJY~gSes|zoO2oS*&o}LNiTrD3(2KUi-UwM}v?0?SX6$G&ezWlc?!`7@8F$5v(`5#>>d| z=*{-naSNKI4xI%p5^g+S3rM($#pK!^PK1eb#M1^MtI_!&Mgi)a6mE(S3MAZgaq+l? zho{nSu_nAh>$qGOLc&eE5p0?l7DQGNSw&%g{T&K*h6R)8`VNo3w!z{M1QuOXr1tvhFxa!y9`IL#EL!=Zjf-pn}b+Z zG@>|+^OtvyKUl8lBivK;Jvr}@iaI~SVZ%TU0u15(2uuRm%d|a z`E*vHRY{>~pzkPERRTA$rljTb88fHJvYgg|zVi~4iS!+$?;w2#={wQoS_Eg;3eF;Z zXYeu^`pfY{r65j=I4$C|>k1)*hLLC(*^XrvHt6@2G*|t;DFin3m(#%)=s~p%zXZf- zJL>~++K%XiQA@^6@QEUzAjD}0uM-SU+%XR*M4Wa&I4#n5kiHX*T)QPH@#rrnR=M9C zPCKfFs1~AHh~cQWY&dGr6Oq1y^c{$`j&4+kC{#JBg#%Ixg)@9k!{M92ZlZfHxAU8y zk@RUjS2{EJv7i37pB|LP?<4Ld=sx~Q?VU&uu7a$7Vn4bxpmc^Qh71fKL(NnrmE}x{ z>WV3)4W*jZG6n>OG|-FBDBTXRAE8~~Y>u-z&gNuvD59)D5WQV)uqKIz8S)481lf0@ zc(ypbP_9f>irX(}GZB z-ivrI;=P^+&jbXwEX3N88&O+=LAZV!8s;2_ZUlUTG@za`VN4qt50fp+O}cC^wj8L> z{iY7&gm68W7Tef>7G%#fRJ*E6+rE!vf6X`sp&Ng{T&`=hLvj z(%DYXI;3HzdKhpVZ#B(&O^A1w%hx{iL}m;?jseJP-In?B>SZ6IEXNX2ErbfKr`BAD z8{~x>!+-P+|KY&}NCO%e4aoDzBQP65(Fyi0qT|zg{^U>cK`uWM*9!Vhkek_k{p!@k zzr3>eg`fWVhiCt|6HXrSKl~zoOg{BDejM5oJ#%6`1G(46l7+zS&zmDp6<_)8AAbC~ z-+lQfAC<1dxdVBQw_f`CekMIo`rb2pnqT|(>vu<}P4OCw84u7MtvX<*A zS}4yhsVsG-Rj;0cQf)vmd-|+B)>X(b$o`3S{5fyB7vdFg?7Yq0$I3uFr$Zuh9C&coSojyN`uPxZuD){+T|& zpr#AVtV&rmV}Rhms*-Nx@=~>sVa(K&Y>sLOFl-NdDg+oVxH@-qrbnk)^=7TXoK|!7 zxyPS>&8Mm3vIuv&a#Py|3JxeZEVC7tych;i{HeVo2*%n42Ny{YlYU^Iog^n7dvf*Y zbN8<go*4_v6&;s6A^L$u3AyKJ<}9u~XoB*q57*Z|weOEfxxL?@7D+tz+d`Z3)g z?U_S$C~IxLtd&HgU^EIwqhNM}Py;YB@vQ@*Xd?my*GGE-TQQ6$Gz#v;-rq#yqYav{ zhtCQCje^lAn4(cI8U;^a$heUb*&a*zAb6o#h-zU^n)EgT12;%8LnNh%>=)nQB)rys zYxxKo9`hOxEBC7*J zRu3*X^(AI4xNeOm*hSWKycBpbf$iMBGgC9CiwouAd}(T?bgnoLxk+-S%fFHBSd1=# zg|0isw3eAgot9;%D*PS+s{wJ7%HpLS%d8M5~RTMfuX4neh$b`WAk;JON*p@V8V1}>31 zs&~-NMo=w8wGh=pR0~lp1gdN*GFAuUG(pDOWoC8O2X>jkbs1C(EzWe0%-mg+ZgivC zCI9!k`q$ahEi#+|m4}kP6OO>hJ1XCzdoQ)~o1c;NX+2juGx)Ke{l1|Beh5A%HHal0GpQ_Bx z``#<2OXbRZ>DcK?Y4+5@$=T^5*;k&MDODEQ8AbkFJ-k{}VnMlh{$JU#z-L3C6OKm{ zgyIbMOZIRb&|k@f?0<+^Y)ofkRg2bIP5a25B4Mco86x#Ak-Ey9u~5!ok>e~f$gv$} KJMGK7ck*9nFhrmL literal 0 HcmV?d00001 diff --git a/flink-tests/src/test/resources/stateful-scala-udf-migration-itcase-flink1.7-jobmanager-savepoint/_metadata b/flink-tests/src/test/resources/stateful-scala-udf-migration-itcase-flink1.7-jobmanager-savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..b421e64f31c32eda8966b679b5f3cab0658634c5 GIT binary patch literal 229848 zcmeHwdvF}pneUm=!?tAW;fHK&*?^fq9Po@K%P-6YWN8HBj3uok<5(tfS~D%FQBU`5 zcaJUQ24|@R@WNfV_wG97uy?mYm5bEwrZ#^pxtpXyT@oPKkO~P~Nnn#p9+0g73nZyq zsZ;>7_xsK{{hH|+kE9t{LZ@9G&Gh4()8~BWJKy;pzc2CeiJgRyF#N}3J^Mpyj{Ls% zt^fU~dCUL(zyJLY|Lf#;uY&WTy#M4s`N5wF)zTXpO+9C%6C>`0I20EmEq8P$kH$|9$A=Q# z$;4=U2+oE{LN_L&QdUY$%F%IE(Wc;s61_`0B}J{7temqH)r=ZB%~CRQbR?Hm~la^lxcJq~`7~zDep#gQ$Upn<+_^q8VkvkSs;lqO_;R zbLwGzY*;!aXLUuhj@VZdBR!HSM|vjZ)D*oUr-y0(Mk11?Ymtsl`^)Ox*-!kx|F)(F zhH}~<`%6^qz8IF>qe`afzLfmSubV%gdiI}KF}DO3Q(pHRg2JIg@Cb!TZGzNEhGNYm zNWCg+6V~Jj(lV~dYTB*aw8*{_NvJZ$MkGc@1b!&7l%C1xTFhyBYy?Kfeez(EG*~z~ z^ldECn#4B7R7snN(NhPZPr3qMhL!9YZGXw^1@H~XbQXSuf#~1M8N8u|q;rYl4j#Ct z{R>}hd*{4U9?;3b@)|Y<9SYQ9DB6Vk=HB)h>uFd@2{eS=M=;=5% z@K4DZO2`LhQEXB#);uh&;Z(74$e$KHIn&ZJ4u$?EeWCd=b>rDE2_Gh*u_Ose7Fp{V zX4WwW=^!DefcDel4Isb!;pd+cQd19S@Y=8V1YNWpU#ycZL0#;PJ;X{zW-)U4q!r0O zVY51@COluF#bJ1v4?|_htK!sOu|XJ|#US{nPsIPzHvn_*HSG3VbA9CUwaMCq6opn2=+H$#-I<@u*Yr5;)QN|B{) z%9zokABGQ?@kabViwOgD4{{=()lwQzAAjkZFjQlv6FYIdqkiVFkT{@kEqd zVo4Ba*}f)0!CMmpr!T~S|mbv@aCXN&seJ7E$UC5=)lC7b4e zk(LdRBwF2Pc<0uw_{le(YWw)}H6+|e8dP0M_em*BHxgu(HEGBoAgbx<>@BylQ9Qj8 z{4`5Vt__ny%~PD>Kc zkW;F{@v2FZZ6=p|r7^D*oX|{5(o!%+y(LAr__UmYUe+}>QzAB5_)7yCYPRbE2^(co zUi~Kbbgw+FXbP5vBx#Xj!4+dCc_ckmx17~0Q0`=ppKO?s-drX#(+xE-nnGqlht6&#{DrQ9YL0tacFFBl z@oI1g-TDiwI)gN`)*^L1$?oCdX#&H9XIh~hun}~7l1&~s8xA-dbRU^T-&U_rL)GlZ zM$(jWwuSa4o^{-Z=WB-&7(n}8+Eb0p6?bB_zK&~F5aD;*j`5qR1)Ph{ZOh2Ds_*37N;^CRExS;XLE?m_2rsmkjO`1o99|R@MTHM>; zY)n0!mHYhnUOL}&{Gl*e2{sZvOOD?E1zBBj+gn)<^UlG!TmcO<0|r$yzfOX)p1{pc%lsS3@nE3&xR=0rbm@ucnT# zeI)vPP$im{NP{lzn!~!|@8rV|5 zg#$tV!}CmIMm1J2spSdTSo`wlFZ|)2`}ac=!8CXZxfwLkv+3^7WyWOVzAv8J+W1$$ zc+hSUXusH;qN0i7$QU z-8Tnd2m>b@5L6_h01MP8=?B@PK*JbVbWR>qJ{f!S>aBkgKp*Y&5&x{GM2gf*36)bI zB}3?hFeS)#`ij@kllGMmy|OLfN)5eoRlpUJrKZ0RVi`N|<68W<4!S9L#V8Ohs0`@i z-q1!osv`kl>(D9?o;`1er)nvAVCyI$!JiPPG588R9~nMgBSPZhcHs(%?sJ#88uctL zEL3j;Lxabh#I4&)hyt#-HH@Ow0ITtm-XFECT)5g(R@wQ*p50--mwNu!r+_UcNfUBz zuk^IHOzFul%~wovQcJv&D>>QPv{DW&_B{m}x6WN$!)@s|DvWADNOAvA754lT&^ zpl_C+M@kshp}8BsDYM`U5q9C4g|LJJcR~(DkD7Z4Y4~^cy9%yiVrB_fu_%?l^4)hB z`BhN_L!ZaWP0WRH6MF^Wg%*8qtFR}m6QZko~jTY$+E~SW%BD%g&M@6>}my!~v zV?S5o>`h$#z^{70f3a04alqOW5S4@)2sDx`yRf3#itZJgtG|{(d#?}VQd}U*V+1yv z4U_g!vPH^SI?8EYpX<1WJR*IVY{09>zyI^VBS>~GXsV(FiL@`ZeZN$18kZio8UOF zQ~&fKIcLfV(wKE08zt>o2xYSd4SfO}T{tY7#qzP6sc72qzLY+OTZo>QYsImku(4(Xc&e>5kJ$fdKHAB%!(vs5Ca#GctwDFa`C_kX4`HHe3Y8zHh~j`Gv=&i*OzRW(}#F7 zE=UuqJ|0rI*YG3j%oF!b>VxQqBYB7!)zt@I%M78rLL?FE2hEi-NpA31Kq@1XM z6%ae!eJB|R*0w03V73QE!9WLi;~CtjfqcG;)b4$IB$F#GvT86f7*8hpA-&6Zb?J5a&x}$FxBaWyG|>T{NZ*OK+fV7Pa^_f*&97 z(S|SNYn>al0-lC!e*t}3-KejI%Nz9t)nBftub}S!(YN+oci{9_zFUJqa&;*QLhuC*aybT7SisoMN`Xv=Tre;Abg-LKvFMQUL&>s#^dFD)5&Q^IFVnTwki^ zSi2OxIy?(65dc7Cpi~7q3`O!ih-}Rn(3=1N@&x?icG0!e;ByV z;JziV9WjW&=Akqvh( zFK{-oHpv*ly~yy{#M$Q7VmQ2r*8!U@35%LTYeaRr21 zySsT2Ztb@MQ-Z_EAD$mrCEVJDTl*3Pq!A%kKJAPMxhmlMA{@9-7W2ETvDJY`s~1Da zgXrn5C;RVgQJ;JV5>o-Jr<6(ovK7r4395}eygazOZpBZ&@l@N#pRXa|KGLA-Qo2t{ zK?cYKSq1ROvN@@%>FKNup(oZRZCsg%*wkB=rXHe^-zVT`A{^NAL0n3J)Ek|U0k2j` zMLGCyuVhJ)Zo`mfsEtg7Tp`qFI5@g0Fb)gZoEJy5bSGC2qAo;^2SR*OKm*qy&(5k{ z+!PbE*o%;>#S(ILMGEp))4eCc821BaFo0Xn+ZnhTn)BMnIX;UTNSDl0v_i=pJG!`& zYS*6Ues^En-RJ6slWK5YfC^<;twvgb%4B$}8U$f(0m-;y52xqdZ`VI}=pX+W|AFlt zOSRzuQP(v_sQ$A&@VzI$)T~H<@$lP_==v}TNf2(P0YjA27Aca?mKGgzm`*IO=t6_T>(DWuBBT!T zj;}t2y#)VX=6r3D(rcw=I9Y7qVyydX?&IBm`R8M?tmVr&2%=KdM_HUf_MN~S3T5C7 z`gIQn58TuKg|D`~a~|0yHJ(+FNmgxzsSHD*JiPks8)jpbQk;qjo>e`lkB30uht9?W z*67*gjCt;Od}Orp-fg|3{=elV6dZ-h$~Fbpxl zxl5IBy%4S!s-jK7`YUps0vo<8xlU2sU8H*@p%z>$YQbh1w1Kjbl*b_%8RQ~k@&asE zTRt2=9`8LlJkmW9KRI%IFiz_3=uRGu!wq55Lcg9Ijt?celZnyz5Iq~5!XbZCGmL0A zaayY@d3bQVLuCE2izu_uGOM9rtU1GXf|oFfKU_W2Mw&LO>J(BCs{?+dvtd z@Xams2PqfFs8TwkMW?akqFYy4>rVz#lE)8c$Z7$|{)r)_~z&96bjWdRPBqXm@C3t;^2 z?BKHLftSyJ=iZ|?UwlA_roo~qs>Y#jNHwh2i(0ls8*$~V3u_j{x_}!;{11nn;}U-_ zWNu3_vqTO&q(2dm=?%!NlG*QWE(C}b6G;vZVeK;%i7_By&MzX3<|{epzKLXuElNjt zh}@QC_?(E`mfD0WTe>#$clZDOw=28Z4KG(^j?kJ^mXJbZZ-G|94uUYz+!MV+SJM4b zMmE7&#^ZNGn%z_m^R}s#r!^Tdz%XT){xGegT9b1NX#U!3o4@w`Tb=&(HBa2~5vF6f zv7m3!Re-HZj&3Cy_+FFa&z#-+$Y(ZFwM{FkZDh?_UYeT@yIc2sD)Gjb5B}v7<3e*& zTysNH9V|D$(3*;BZkm>h=0?mcp}E0*jL_V?zce>uUW<7x=C!0v#Juww%_Kr|gMF|_ zT_KoqO#}|=TGZVYQuUcX7fi(e({>f0D12syZ5Je?it%F?UBb1 z9O&G&pOBYcgvV%;!sO0C!&)gYH$QG$#Q(FH5wY$;ZbS@*;wisAXgMA=VX^}7Bh&J@ zlv6#{JdTzvP*V$|WJ>#wZ=^$zvlKO&gh*_bQhAq-U+pL0?DmpNB zmz=Udl)Gs*^o+DSBu^-&#iBD0pTF^}cJCP-;4hM-*`QZsLrznwLZ@!PEZt@TtR=dO z`K{*(4UGI+O14NZBFPj*#F$rdv&#sXTF}>Q9@5Y0Y&Ltwm>uL~e%CwLv9i%Ck1Lvj z(XvRgc0Fq;U`CTW=t9BDD+77M{TodEX0dwA7IaP^DUqC8n}zgclX5|z#WpB^EgvShACVSTyS}4F`Y#H54lca|6FVP z*<%l#jeX=GVkaa?Exx`*)+K2(W7s&EZ7U&x>>2<8a$97P8%u5y9ouybwVSbo_t}G$ ziWuKj;;mgYBNpg`FB1v9YTmtR>vb3C?zpyZ;OG#lSqk1ZLWcz2xn`h0j%#~;lGKgd zJ`}fq)QlV(pyx613L8^TXXQTsy_e269e*fHRs!^-CQn<5bb@T5_eDJSMIvLm1$fPP zbyL7qOV1|KNwS`{hUR$z*&~wC(>V}8$o69GcJ9Rd;+(iF=drED9vhae2t2V8+lxN_ z7G!~EJ>xp}#}XD2gpLV?HwLs=%Sc&^Y%6>&le19S5Ce5gkw;I#qu8a56oi)P83FGg z3kng|b5U>4Ij{A(*M4?=-5WF1KT&&dpJFbAz6+FsY8NFRHCt-Z} zy&+J}PG?U8ns@juIE4dJOW=8yXNYR7U{cEyva$B%&tLe%J@@a2CIaH>Ddc9*M9-$X zKbIMkjr+cMZfoOT{o+BpMWFp+bBbnD*#zZ>EP@lu`CVfXI!B!o1a5#CIZaQr<+TmR zaWgbsa2i+9BViZ7jP}`J`dqW%XsE$03t)syUWY)YL+Eh!)gEkKd30ui&488)- zM}`mIYJ|j9;Sj{RM~*czyq$TATjfQK4Wnq44~kPvxDr@9bW{N2ac+)JySU@i{-(!U z`X7wF3kw7SMh7z<#)BO3Rp;x4I-=!gAus+YFj`1ghiyg0fLeSJLuR zK4hI`73?eXM;A{2@RuKb;`d?nO{^3sF_=E&X*mVnpt{ByZIPWNEmnDMjbNr&XH6RV zX}^=CMY@Aaq55K{>tY-?DK-WEnK8eEab?68P~oa2`r0*r%}a^Xv7ak(_9m`=;8(rh zzt}32IKggHsKh}d$+C++s|8A&d%!W})SOpi7dBTv`V888ec;UDTjo4Q;I^=Rlx&f5 zmX52J*XKH}A&*EOCL8eTF#t+4GLkWc<}m}zVWVVSTA@f~x5*lC(iA>#9wzG?1~nuD zPw$mgX=apMF)k_Uh-6L;Xg!m1YKkV`i5rG)jFMI3x{;E*PZ_%X9XSiMN->jAtCpGp z_^{LzScw_S%YozfVX{&6U$fhlAWa!*8rNl2l~u*efME=bQedhZ71nxJlhheg zF^|A^i$F5668`KxdG`b7Z|6}Ne#Tu^WJX)yqtrCF37jaLF=s8izGTasKE#`GL7GtY zF$vZ2jx+_#sK|oEs5$18!AK%UXgxZ~|{hk`KCt z(Sk4ybzo28{m?1fFL}S+eeE#HmSjMrI(Q+-6EilMQ>wDaoh5sdKT9Je;Mv%auPh+) zi2sbgbgVxQqBYB9nIzt=k9PAur@KmRjCAiUs)Qfw!m40)NPoTz~n zy@}!OL&-R>wnY&Iv;CG^S@WG(7cz|<;PtsL*ef`ciq!6XdnA)9EwXAbF&IxK`s2Om z#}N3QK$6l}uB%i`Z83Ui+?E3zl*$y@9$})Z*6&etdx5Vbg{$-}iVKS(@fAp9I;@U*6FI|Dv=S|S>#vyCVqS}R?RB_D zS9H(9D>AK2h)gS)LMtn7@m#SoFg~|JM!RL3X@$$v2*A`}p%U zB-}?DR9#BRr39hQw9Cc=S93kc7E_vZ`x6(Lt5F0t5) zkgLTKa&<)t@>f$C1YwN(0W%oDt>^6woR|%wVP5t^26}8`U%P2DY9HtLENWok`Kk%} z4s%WJ*wMwERJ-;(_q+Sr?mky9oK%DJ0#wL{y&68Ql|k({NR<-MQi=`S9Bo9Qnl2shJ!A#_9v5PQFXrsvVuHIotX$(WN>od6f_o*QH-J0e4-5F8!V+AM ztQ!9RiZ8{BHae|R#421bgzE+9T_ACnSIkA~ul_!X*RmM*NJGnxpW7m!sKdClCK!~QnqA7}5&^OBup$Tool`*el z7p|$k(gOww6^!^F4m-ys{$9x3mSSd!9C%27)q#^-s++zlX>;!8LV#E?k>ub2s;f}J1nFOg^+Hm#_(ku}Q|YnFrNro-;mJ)cUv@#TYm`NX);+!WW`&{PM@ z%`dd3qMDng<)XO}GfQZ0a33QyH*7n?B3qWe=0?nGF|WnEmIOo|od!q=aJW%Ji`5kZ z+6e6wnj4|HiPEf7pj-^_yBd@XDI;65p~|M|cPn5@05`av{G4z8_ZDoxjhtpF898dA zA8_;zT}k&tsb+Q=cM#8K9lLVZjt1=wF0~u%HPG8YGpBIv$2mN~Zp0 zT|gB(vpRo$n!6=?q(CGJl(rVydMgZji^Z^qyD>B-gvhG;P6|t>6DCa&u@jefWEotE!y9ntiP?H>(l?v zM5lG`jdmg04=!GL^X7x?T)6VwQHexGfl%KkNCRvfQd3~EOUV|gAapM-!z;3o7nPUE zx(&ooURj)301t-yK6KMf7q=h4UO_03m0$tc;`U9!1JbHz9=Ue!OZQT5sRrODa(^x} z2Cka_o5WW~5DSpY0c5lkDPOJ2u@uEviw_|fEC=me{2L9* z-7uq|8HV$HtZX0#-|r~)xJ8<>hMt1mC?6cBqGv5qX!><4viG2h&TC0%*_<`V%F+$M zwq=$1PxCrg^E&B8r$)sJaGoiw(UQz5GdiHj^l4Vh2%NLX#=?`l!Ww9r^9MS<{Ki8q z)DmB-!Wpcv9zq~)xO4$-!B-q2fA-yv{b3s|CX}*B8-oYJ`eW{GV2Xel&3GAYTDM_H zGuV#ld;WFn=e{ldO$`YpNUaIEU?>A1r;+TI*|Gb;Sh}lguhhARkXQZ>JVqCpFuAjU9#B@Hz3vK*Hi=9bGkO#g z|G8cg5&zF(z9MxG^3^C9MWFop_}kZni2%ozOtWHPLL1$QmQ=JUIej}zC*1X9|D7%B zlkdQs8YPWV3VHE>k(LdZoUQIN{O-CHKl#Q}Z6ANWhJ^b_gQ`pEJ}CvfK0#Jlz$VN| zT}@ACgEF&H=2CyjE2~fAWNCr_VILD37`qFY3ry>qW<$@j&S6=TB->0L80>eFN@%7f zX(+MumUM-VPs=G_I=aSY%96?2;9>ZzulYp8KhxQ47BcG|oln<09U&XN^0=ZY*hjcm zwd+|6^$6tY*&|P{wMbo0vU_-Vn!qsOy4ng|3b-6}^3t7yGoaysvqAR}pBN#XGW6(# ztiilUMQ>w&dnHSX&~@Kw@C znjD~GDk*b1oD-l)Rlr9UFkGd>*0y){|HqF%d&A?8J}Q*$LfI~*CgoDr#B$Y*$dd#( z9n;Rj7EWM~fTCeiw}9b_a21dC-mkc~mm^zA6(-9Rdqilp7mHTAN!D^1R07HK7MMl@ z1v{qtbk!1ZsP7I*&B?C77#XIhaB#srS4<~S|3j{m*gx0We)iZyXJa2Zi0Lbnq!wS_ zBI}a0nK5h}Dmq~s1NpetDE`or8%u5yomY(E8X8M@pRL%Ae^-gO&X4EBmx+X4HSgZE z^|}k7=Z3fK8#p?IDZ&cgHbRF4-nnL=KTh>{Nm4g*`%v8eQ8RLEz}D}zvN82^R_^oP zd+B`B@rS}>CBz(S^0bvmCqVg#;~MeY7m1ANmZfL#>ZX9JmYz+dlVm+@4b@n{!Wsd+ zV@`!l`u1Y&cJ9Q)iJZ7B=drED9)qN45qM%Hwii|V7TLmYbz`_<3D~}%<{NI@7|>!Z zBW1zhQTSXYXQ8qo#!|ZlJ$ecr#V&26`bw&;gax7&^_;BdqTZf!Uh8wO{p|Y22P5Yw z{nkhH95fJ%0T`P)Ig_sU;9Y(`JhTPEs+L6 z`kKObRxwS1VD6bB~8-abt1RcWF zmwdkM&Odg3@AYpzYcH;JvBl23s^C%H$|zTW*~>C0XQ#8LA!$SSEgT5?AD(9#GrH-9 zNiBn_eeKJizwn29?%xkhgi!la$jzXMo=tZ@wI$v6#dBL5|LPYH+ARX@7n@U5Z?*}_ z4_O2!mh-#DVsq3vLEr|Mk<;`UH+O~EThuwRgmdqW%Xs9sbEvkJ_b zw?mt1DS6Q5At7NaIgP}KC#YS+`n~y)8j4u560ew1p@Wo!HkD7ZAWU1I{DRIAz1$9*Ug_#J^N3rm|0oj z^3tsx0+Ez*3ChMUJ3tJwWO%9N=btJd)eh@p!+=+i)nRjWM44clt;7gYKY*aH*~h(0 zz~_WdC^54{03Wii>WN@WQJ1~+19Qc^7V}!nYY~_$iNM^&o1{S4sD|T1iSA@#G>*EI zFu5&I316T{5`no6Ye$d#P=J%U?W1azj5=Ov88h=%m&5}HEq@bE)g>cPos&qUn69Md zr`!ZPy3xD zEz%ua3i_hO9Q{R%k7W#9wqkzotcsbx)DnH|n!oIAhy<6q?5jFo%tjb!mkHQgY8yeBL}v);SDnNCuwXE34AX zD7j)>QdEpu8PIwr<(< z&Xg0RG3!1yO4_sF@`a&E6W}0k_U6z5%qih8*@l5WJ?L7lLQ}w-z;3}2MTL3~ldbtT zi~#rsSjpJ+QPK)^_NclE@6)5o61eG(k~TPJ!vgo{nJm@}MJGv1N>9t(nxxK{ig^US zTLf}W8WS-I;5#Pe7zGc3A4-gkH-y#r?+4D`o|ea@oND8o;A% zfsazt+$M0MaK@aq?D~=|bNUc(#sz6Y)yE_iIJqNDL2HUENc`=bG8jojZYwMTOEw(e zWPE2zp~c=4P1(LD2KBqx6UYO%B*_Qe!e~LqAj4r`PvZU1Dcdi3zukT9Fv^x>03Zp> zJb7ZqCUZ(v7P+%zZ}MkN{%mZ>R~8U?#DB(LI#};{m=gIH#u3PME%q6Hq81~Z@Oy2D z1v|N)Ap+qAH39-3B zD;Diti0uo_c?^Z~rAXdsVFOF^*N z&A3Luf8c(2*om}Sw-B=F6*vLc7BZoh*pdq?5z?Ws{@RY2s83z`)?YEN#k>~tT3COD z^%wT-hf3|9anpsUH>Q9CM5|l%HRdR70NhSg0N@x3o1PwJ0Yn;=p)7#AsIs7H86u0R z#3178vH;+OD9jC2kWLf2VYH!fYUmYeD}z`=7sJEZbXQ#)vV1Xcg1UPi+K>f?KU^T~ zh`{KXJw{Jkf!bsP-IXm6IGb3TT*aDLn{LH@6d?}K8p&IM#M&fd1b4HmlBYC->Q7i2 z-zS=Su{OP8|*BQ@BfP!1srL^-uMc1n)2|hU=KTM1OtIiLj*bv_%J%SGakekflkA~ zoau@S>PUf3BgGJeCuSQ#1v*VZ)Lg~Q2y}x)CB#|tc6dsl(_ALA&D>+Hn!zu>51hZi z@qFGQmk7Bk@Vs|CewiOPdX=mZ zksX7Z{dPu-ibJP_UwmRAx5h-OQ$TObsH zNA(=EM97s0xe_5)Y(oHxh6uS5Ay*>g%8p|4TD?Tbm6Ia=(h*Q1~{15aP#TkqRM1}Ln+0)7W2BQn!2SpuhD0p$2yZYU{2l- zKmQaaCZ=1)Yro=QAd6N-YLKq-w6}geFuWQbRBg*bN=M2oCQu0s@ZJ7Acs5uOR3VbW zmFGl5yK0#NZqm6X+}d5M3KcqKs-h4fS2TaPaBIh4JU>3CaBBxBn{6jNST^9ekeWhV zKsgTC!(@|Xj2dz=6Hm=8!c+U zPwKF*bN1)J67itU08~h;zI=w&!GGa;fqfucFQPnkOCFh`t->YKVNgb>vZZS?e|P`i zf4j0elQ3Cq0x1fj)t`sFmc_VN5^BN9lK4E30c6ky%0^Nihh$`siwqYFI6|c5!|~(s z-jl;4-6QdnBgY5hr0$OH43if6^<-WK-C(fmd@!crl%z^=TB~RNYf{#n)Gb=| z`ePUAU#zJb3dWioFfKU_oqqKjMB&>a^ zZNq2Q5y!DGn&(*foAia|$JC8ynWN%Z5}XwS^8tY}QD{vTjn*WktE!yBVAOfr)XLME zKv?W42yXF*X%*F)oLfNi*IwKFweR2R^sldZ;+Bsv9m9=fX>0=DYjXUVvwI);%x0># zX+^aS1OvNb&2rG(blBax=TnI{zI^a6pBNXKo8p=q$g#2fG&eOB)!Z~Km>_wXH8)~r z3C#`PLyH|AIOVa(mZh(`5%XHiYca3AiMH}1)&xXe85vguM4s@|=Ft~Ib5n5zc!ffQ z=I#yM!n~n}l!-~<4qb%KsY+`+rykbFhC%C;)fLS;qD&YP>bnvn7;X)iq2Njm_w=cf zbwtW$WrM4MI(OUpppMsG{_AJXzSQ)z&|Cy-E-Ew`mYvJWQnZW(v@l(0bSS4;N=CMkZr=)VJKpA^w`hNRvi`P)u226v6SCI1H`;}0Ke%}1 z&6^Llb1}?wMs$nWRq9W*eYOUdN`d z#*a1l(d1m{1p8`1vrWs8r3~}}bpg>q-j^)hFr$z=0M7GW01$)kcNBZvB28ICPsygq zQJtNNp0!A!k=U)s&S$3Ryq1KP%~^x2EZwlk6|6G2eEev?O!N zj1FireVWxW0_QBUvG63Xum+lI*Au$IBD+g;q5Wp89&h>0?hjsWy`r7crx=<%3|sg~ z81{mqy#;0@O`4F6C~NR4{6mYvLEd7wF#HB!XuInW97RUB;vY-;D~sJ4?QcJQyWspTz7))=Z}6l6A;0f5U@t z#+JdsTauULzsq|E3OTeas0{dhcsB(t0$I>LPepNxlUYLA$9hN&KvK~If7l1^6YxLx zXL;ctLL{cXk~Z-?#I?`0n)1j@_LdUHisUQs@3oLSA|i9>Yxp zv~PiOAk;m`w-Ui%F?5M`f&m=5o&p5zs2}wiTMb7|n5;;WO=)>t%BddTJSrp?Pf-9I z6q0%r$=VAsnj!}Rh*?Q+9hTF#v#jx5Pxjy0qCWW!2rQ$dQA(j8Jz%6|10<_fSF+-F z*RA-;H=b(y`13U++(#NzT}t;!DG<~XWR(Rn9ssbY>FI1x0&Xf9)?f0LegT1N|8~Jv=;3 zAV(1j(ycIP0KY`%1l3&fD*iMbVAl(UZkS8vA?~NB}M4sOpeZG zMpBONx>|d{{|gfeg~;*B6zyMO=n_I2Nt1(TPt_ch(rL%d9=ED07*h)vmDph$RXh9t z82$b<%ux?wf*d| zht9@6au9=OlcW}3-y-XhRQ`$K?P#TftsbZx)_@P2+afSLmE0scvQrEfg;>J-Y{l@O zc9rO3*EYz)h4Ez~p;yhjH*LM{0vIXbZTkj}4uK7<@NFY>NZ_4o2KwXF2$LjrBexI5 z?H@HG#|G$mR3R{7(bHME&wuZw^G(Me3X_!pEuqQNRwA7M%Mp%i#B*OHGNxOQeFd*> z3b<>biVb zJs0)%oby_rd+le}H$E6SN9ngdqUWH2SPW>w>f}t;nz2ZGi8lr-7~Z`aYS~;cj*JYT zUtWAQb$sn3(dUCI(X>PwoD;7pd}k$Nzx5#nW)5c9wd>l!>T)|a>f9V2LP+KD-{3SBK$^AduGtn!Eav&$r$A$IkD){;g+i@tI0t*qK)qJjz=cF%eF0QY_I z+}6gw`o)8Gi$MFu<`mVNZG!Sc7Qu<-{I0Rs9Cc0*xB+J5G(FLl*Ea0i&CtLNf#djn zW#UWUdH2l$7{b8G1_Tv}D8LjnO8RR($ydNuLxa2wPV$9>IHr6u_U6@F|0IAu+G%V2 zvz`(uQZpq~P8F0ipcBHBNZRQuUPDjXS3>m4wty=&^vYEMS4fs7w)!BJu>(J@#gFTt zn}S!2U_so+y`hbGR1da^c@NE+w?mt1DS6Q5At7NaIgPo`gBcHF z+K$v1T}~P|WXaGs%MX7+AZLo1B@p)HC zsDzD5*r+hPD+-=!mK?o9S7^YH3E0dY3yPOdRov8Hp^)>rYfM&J#>~9cCGirL6L%9% z)g>cPos%lHn69Mdr`!q-$~LU-NB`x*ICTzx)}FskD)hM%FxN3>McFkXuR_2OD(XpQ^ zarP#ze&AQV-@n)@lsLgq6etkUC{^%se-FYd8EkwRa7iP+B(F=)@2M6jv2tka&D96d zg!Wz^ICHr1Baacdt4aGP*&^jE9Y@UTa~;=^N2Cvv4S4k!tV~8mGN#ZxW`H?tl&niD z6nB_=mg4i~VY1F)P(w2C^j=w&W=6>s=2PH!SBxVRV%z$AG zj8YJ@J239_0X9vMO>msosek&AoHOMFY0SEhjgs~(SjaH0bOIdY&E6c^%bXGplWiEK z*@Ld-Dl`QVaCpxgQBwEO4{Juu*Nc? z_UM@`)(k}_NlQvk%iWr!&X|gM1io7YGRGMcMKd_`A_!)Oukr{1)k5bdz zCUBx~#+XdvU62?x$DA@4NkncdECNe5y8bMbz3=fUDAr)xK_B}DE-^HFl9=Ih*KIj%k3pxggYy*1|?}tv=e#!go z?rVopwj={^r@$>mo|v)8oKlrV?kw4x{8^Jf8yoVK1w(!2yWA8=qxLDDf816olj00=i zNN=POz3z~K+r@%L!E6tTf`Jb3`rH>NJsnC#YWKb%lF5}8Sv8m#j3*QQ@m};}2z*cU z)lgp;J@?XwAp9APO?mM>aQ$n&_n!OgSN^$+xi_qadO~(%nP0(<(*Y*~Ux-p=@D+oM zl^W|jQjGcu4@t2PnDQ9#a{m=SNq+OhBWrJc=7AU4s|Guq@ipzz6_%>#*ufwF`Gspf z^D0(^eLpL2_bK2fbBciD+@VOQ8}Nh2$XYtMBDPJiXQHRuLdmy$q~u*e3F zni*U`*~SkRx-}p{fJk-(Hq zpcthCq9}3ih0dK-ODQ98=Ud6ZD7&!`mj~5UF-Xd;2szss9Nb$9g3WHmH3I$v_rt@^ zm)N?6kWH_^3Anb9c@xE!T%dF#9SZBO?U+f!!^o14P!y^*Vf_{JTAPS@=e1|yB?5iF z40Uo~3?_h0qfX8hzHH4I&|5eF&ELsEAfT0N_L~v+B9 z8{`dSn$QiSeWiw8p|&!JHFWiw&ZfKS+K}aoffLl-^U#JYFm~+%X-5oFTH8J^waM!D zr`n{gKy9)CX8wYKvx&9ImF>8h5|vr79G0QThP$XDyK1hs#mqGeZI-fEAh9-4FY9Jk zB~NJvDF&9t_lc%ntWC34=`X9bDHJGs(nTvsN-B2uG60ej_%w?JpJufTz8?UfKX3ov zU}u5+AYAk`;Yb^XB2lx8K&OeiY92ISyK)z9b>X0P8T68vHG+Xary&BJ2D~C1+!+sI zj6kPhV9sogk&r;AkzxqK;}}Z>It^W)Z2Dk6aTP|Jq#GotAv}BD4o?Ym8lNS}zj#(N za&qpmR?Xm--v`cL;CMc7kxPVJQIB76!&hR#zUk;yE<&z44)BnxeKlvE8fahpu?>uu z3EUivu%l}pM*%=~EJCg(FO47KH$8ioNbEupawS5pz!h7BTxE344VMxjSHKz|5JH4p zLB>uIawS5pXpErzgl1eI5$Ab&L-6od=y2h3mu z(!ul-_IgWNJ{&(D?>#v@(mfJCIdXh3PU`OHP9BZJ4Pnwkzn&b94<)*jiP1PFAgzw( zy!LU9&!Pr`)Ue>LO63LU;!dhvd!GBo*FSgYAO9Ht0lpgA+^IGkfV|(!;ja}qd!cOy1KU9Rh(L zdIk?zqiYkSPBIj0CPC^|S(~sXPmq>zMOM@96CnAqbWcTd%s7| zLZI$aoZwzbNTA&cv|ETs$`}0n{G*y-)63QMf@A6~#=Vk2Vp=RnOwBTA17#yAk3%xD zc`5^g7r%Tkrs0&NN^x4ND|vWeq(fx=v5P3P!2jwf7;Db(onR(V`;o6kp#3t}#G{>^ zTp`o3YsZV9RG<6l##e-fami^I#SRdF^z-LMo(}9c=?l$|sT7khSx`CjVzU!MvizIc-iUZrl8MJ+)b^2up&z9I~i zK*7_tz^LZm0i&7@4r)z9kb(s;es^|o+4R86=f89B(VH(mAVkw((G(T!XxUqK)&;Z? zSH`@KUAU(D#$lX7{6GA$b6n!Nh0JX!W|j!Z1oC6tTv!Drn14i-v^jTkAwaB{NOEup zYoCqse(=CO?O*t6+dJn`u%YQyE9mni<~5(!!rCVStyM*v0yS2#besOYj<6Ek9=k`Rok?p+6IDwmzU2UjFN6&c4+2w9s4xYc49`N3R?uJ(`BWIRX`OB3mSwiNByu6!W}w3JlNe6&qrUnEzpQk z;AlE18B#{JWJ8rr({G7oHX_%fmi(OChV~W|ZR9i>Wo}}aSo98EN%upkW)=Whmll(m zdm3GEB7}eFVYsR`6z-(_l#m8e&(HA_VdqjU&FhJ`!=*5(v+oO$yC4Z@6qA5PhgiSb z__c3^_y}w+dW-hAC+lx(==$`(Gtp_Cd!t>5_JfO8-n{u>I~T4zcT^&gQD9il3DN+R z!4Y~Z3f+sIqbssG!-fN9-3DT49vpDE??X4;baDFu>=jgQtON_l7PoH-9*|Z&^T@S( zU%HoKBQ$_TUhdCj#vm4wG^OQ~3SbG^1O+F-2Gl0W%5g=Nk>>gD2AETmI-Ldd3|1Q2 zZjloOQwRMbDkDY%E+f2$C6^9IWR+QD#keY2fRsv;n>9d}yp9cBjUQ|9qsh6@3I6T{ z%{DDVmNL)_)UiX)%7$d=h8YFP0?zlbvVjnbJid$OE)ZX1*^<|n%B9S*GVrrH7ZsB%2i>FmSj$u(E&}SPqSJ^;G9J^ z7M|o4)qlN&BwfLrhthsd9O_hWz9MvDohEYjw|?{;qk zCsLTvjF-`-bsL5>gYB5U=U=ye?%UGe)R0hu)S8gvi826k8rd#`Kf8V!zf#Wa;m)i0 zDg5Iy^jhlc?`QJ0z~N?qnqGn iY!3cMhdJ6bz+9{_xny{*kJ~zpsA# ze?P9@_J9BHfB(b(I`O@0;CvwKkNn9z%x41C^oDv}&1ea6#1IS-ui#^FtolxXgvL9P z`}QB$pWMAaF>zpbhtMwW7WN4HcPB*QKyqB1IIw4YKiv8YxW5J-%qogC;F`gq=y3Pw z;P8o&(eBaciQd@A=y2@NkNshdkVHA@)U6w2|M56f8DVcELf;7dkRk~+l~R?6-So&PydCGst~jYP@a@pv z7^FFlZH&l*G8v(#x?oH?d^^KRcD-$H$*cp|12U6_A3-4c_cNN=&|K1)qqwev_qBfU zYc21dv&#cI=_;;a<1nB=Et;fEIz4x{Pa}usQIfZxBT%X=8mcn;_YeNVcfzNl*uXz0 zrzs&HoJX;7HDB|vw7Qe|-b41Z=*j4YnzAYMH_7vjPsp3k1WB-;1jgeeAQ)u5>ov21 z>7p+QIR&(z9&ZBqJqSPloRF$oID^-IWlqqF9r%2MGzWFD9=nK@jLzfDWs_DY`-H{n zpqk)fi56eOE9Pq`4SAKE`b+i_hUW1SywfM-{pooD3m-Kc_NADI&3B5Lc+I$YO;Qsk zRYkwnq|g7#{=abWx{?36bNB?b}I-Erjf{79Af3}Q4AFN`JTav|#{|J&r zLKtuB5cVC|eLxtW*xiv#?Ad)#=uGb3m)M)!*O_SBBku1YC>we;7U-#vAhf%qI-gJZuvAyp~d*`piyO1<9&7*^(3|gp6!B&7rHD2rKASj3=Vx z5{ZLABPQ<@^r^!_8rI15Igjbclml_nj5Q96lag*QvBbL2sL_+~B9pjaly1U<@vO_l|Ut!!W1;-TK5R?SGquzp|TXaTDz$mKGWxh z99hiJ=dIqVXFV0@8=O}jRWd2}#k4`{j|_|qMSEg>v1qSBcDdRkoYd6tq^Q8VON8%W z|9b^P2z6_kFiQt_E%b6;U7*~_dViv(hk7%q)ND7@#Aphc1$}i^E8#y36;yM~)v`Ia zTluTOSLoEATh(c#ne_&#>4|rbjLZ;tO?ajm+5sCuXC&F=fwMuMvwruHd5mrO#xzjQ zajYi|342>;ZQ|L$eK>k7+G}pRM~@9fNe$gov+-Nb+iPnJ-X{wNtdwa{vv$>c%`fc( z|M`>8-Sp(+kK?w8Z?9eZE;{@8_F8di5a>poZ?6}-MFhUWwz8GGv*X+AaE?l%SnB87 zYt;TOT+Ov{p!;xiWT?9*N2z^>uauSZqJ{)bNQs81$*{HYvMVN|J1TARvbl@x%qiMZ zj>9g%5W{z=QgQ%F)j_51($C^lhJ$KRD|I#p$U-~UAc90b3fnx#>VcglP4gF+#&X6N z?Ga2jiaTLukiZ?lU4oo(jc{lL%?fzIB^c*1P}Kg&e4E8t34Ch%`4f+viF~9BwXtzh zjomlMhB$3z1aB9a5&?E|>%f}fw8$Vg`|s5lak_SE$VDPC_p_yuF<_^>r@$CHW*Igy zu`{8-Yx=!gw%>4`?vAVb29FG*nkA>VA^J+7&sBp1QQU9T#!1cSox@S3iKHqZpAKSnG#=NRwaw z?t550>t!uacnGb!m(nq|ro)RfiGbL1$ z0x227AOtBvw$d(MM^9Q;0`$rbpDR`L$~8V$NSd1dK8R)P!jJ3m;|3Te{}m%gw4gGe zhkHXC@hFc3fUQHTKyd!N9iFPDb5w}KAv>IeJ&guP8-E0b1`-&<%yVx_k^!F1l{Q4xY#W-m| z&h3_-I*XK^ywZHhBqzB*mmJB-(xw%1XtD0e(YUqmF*R-n8lG$#csTMNED&7d=I79Y zOb^Cp<$0t8VI5ky@tZOWb0NYZT(=aKaNthJq3BU{KOuGh!G4#)RgBLp?kX0h@>jO| z4kN!ZieTvTSh|V1G_GRH#s!qmYd){}yyh;TIeTrdtnX&>)L)sqfJ)p(1&-rhPg2~Y zE`i|7H71p!F4-6>(*$HZvRs9_$Z?58+BB4#&9_) zaoP@;N}SHvwGaKO_Xn4nxe^DgJw8!MsDVHu$%+drsxIkXv9bJX8MOCCUoK?|WLb>B zVzWWgI!3k$8AC-m&Fym?Rggy{`^hG}dKA1Tw3MJtqj^jNbJ!T!kd$=1#q`?6=k#GJ6d1 zW?YaaWp!M@_)^>B2bfWw1&L8}#4f}1*1sdS2rStN;lMsq3e6XR!U9BGPYkK|uqTiQ zZi|x-I)%}KFm}XuB=LS2l$~?lZw+57yk$es3{Z#xO;65RWKOBdAa@rWP4ihA7X!~m zhCO8gk%zozyrqNn-cGXc4NcY=^F%eqQRDaOfEiJ3{|gWZFSw}?+sP(FvrJM< z)W9l;o$fvqj{<9}h$xtyeo-(m0Pc7OCu$&@?;^FkdyiytwL#Vl#fGBs*g&)w{TO`v ziM|^0i(_XmerN}iQ?lZF;QH5l??3ywul`F1b8lD+^#rWOGQWadr+iNOc8F4?{}qjl z7kbxOq!{)R9+F}oFy(RJ<=!h^lKj^3$JXEe>_acHuKGKinPWS5g@r0Q+Vzvqp1Du8{6pQ~f-CNcG9Z&L6&=$CMHCU!26xezHY~l4x>;1?*ARYu zz(X6Jkgp9+)CzbSvi${&X?df*>Mn287gT?_qQ3lw`^Vqjcf-L`U;SPc2FcYV1P~<* zvdI++VZRty8rOjY0V3I9kS~Nu5nt}c>lD=^sXwwz$ITXBtmKj40N zSXSTWZG>!j6;8mlrL_L?Ejh((foa8oLcq_G1BEa&m*oNgL{+x{5|raF#pgAj*SNlv z&#`tndUd!KUcv!@ia@FIbr=d|dk|TgGoUvP0OShz#qFYFsln+82c4131iw-NfKbS- z4VyN;rP==HQ$I2_YaXot2fzte2mnri!rUPH&W6URqF1P`3}Ov0hc;y8V&M1<_ad|* zzQBFUUOS=@j?v?S0`f84;H)-}nKZ@dX~|KWY=Rq>44jRxO^(z;aa=^IfQIF;2t_vB zwX(q3_}V051b2nuv+=dbcC+)s*IBH#7)$V?RQgvaew_c>6!4Wj>7wOFiTUnc0YGxx z_qS4ff7goOpf#v)bQ)RU=rkOi2BY~S4u}dzr-_p+2po=1!_jFtI!#tm7}F=tuu<5k z&BUwv&znVqqtmP)Jitcq%kKl{FK|4kc6)OtwjhKW9&(kF1Srcl9aIuLKF!PfO3+r<2P{Q@+)$KJbc71chFVJmkug@63N`^F|C?Obp2ihmeO{ z@sKM>z`;YVFfX0GB=L|dU=0xBz(cMeHxGh!y0!2Ca?3-mSUxHqa+RHPl!shFcrywX z+vbr*jE7w1=18^IA~ST?66w8&6nPA}JmiXpT!p70mOE^bO~vN-s{D{Ez9q*r*}!Rd z$Q2K{QYIl#mHJ66lbq}ML$3I|=JOgTnTK4l_#if~dB_!@Tu}bTlfrSp&C3P2NpS_Z zTf4J);co4>15<*-&L5r~SjFAixm)`(1*G92S03#Q54pj8@Nw zkO$Gz9Tx`fZjvv&3yG-!)>B9%0NIM>j0Dw27G56QU9;+^-+a2|<1bW^U>~WIRUz3Y zBp?H1jI058WKo||<>XA-g3#k@lQJPqhAiqWN>dNf$X5tB8V?6HJP?-xAoYePMZl|- z5@8ekw^uNPP`9QDv(!e$L#`0&Gw2^(Ow$9y3^DWHMlkY{Dp z&ToqGTkLtrRi%VnU6p|R)pYNPFvbIb84Tdoi*^PshvvNcNfVz%_N7baDO#fBj%^*L zlWNDl=YRJ=%e`l7xs$4YUVsv1SS?3dfzo7nD;or1W(mo-ZJ$Zcd*7*j{?I@DG5SNx zJCn`86`Fr1HV{hRC1_w1dV3~FJQD6SN2T+4Qa#J>E=*j4i zmEQIwck#UoUv89yzkKu^NOav#0s@4aDZmiLq(SoJv!z8xY^D>*DmvHT@CFRbrwFNm zyyI(6;wZu2D<)s7PH%~3FaLbZm$iH)2SJpI`Y4Oj$o}J)L!k(q zL9gzi>)?H@U;J9jyXTN?QsY?(nPk~kn9?v5%EQagzF{_&Da9$D;Az=~`gjNgei&># zV2!Shks3jhjM*5el|^OJm^x0HCL~c#T1pSH~r@)3UORiHCcPn(S#MOe8q84lvK^rJ)ad85YkwGppCNIEt zwduprW6|CdBct7;(G#P`hN7hAuI~7eDBKVvP4w%Dk?3%&J02U04%4&#DIBslH6w_2 z6Q#8}l7|b&J3uxby@WCgEwda7#;ViiPB7)WMPLoK_xfSW?H1D2jR+^jv>O90h*X8~fi)kIFQ9jg9vF3ohtKMf|zYAVz)S35e7;i|7n|JRP$=ka9MzZxz~w(MDW4>%zJvu`b}o5&z(@a$Mr? zrOa)~XBN+ahxEq-GTi~0WitESYYG8;#YB>WgIoJFNni|!oAdKXquEN%g>NF+VvEud z9z3^Y5k4n8x1};Ei-xMq{{4gh@SW0bc7rQbnZvavr6r^Q>CDk8SV0gvntQ@`sZw%4 zNQpW)%eee*NTZX=VbM0V;xWb0PKzWth-c=k-^W1rbd)i%wjwhH4duCb;G% zzvhOfI#_9bp;aZ-+%&8d%?+PfTyul_7_PawvNSh*Uh{d)=e3|rM%?on%_LlNgJV#k zt`JPQ#sddcCG6}9srt;D3nt|KX}Jne@?N<8a*5n&a6n;d=|5lpfqo$;akGdsOd~i98%V9iT zP7nJ?cS|IPRp=goi-(mW9-;z)9X0eHW7%uycSO`-yEQ^;>G#$26O9kxsZ2lX=qG#x zU{R)rbprq6oi5+S7ymIpLgRb)?@4y-7j_?PPmJ&G7~gYX_rZzwQ(KBL@!S_SxvIBC@A6;TtDl&a9F8!$_E=m2Yp?qXi+ zc}xK#zmgCQ(u+tkc@Z%dmE7zyLZ;@7HJgX@b0(cmpVsCFIhj55Hg>G2^@pa26u9bI#_T!8dZnBazmV1{(o$(Ox-aiArBWKjt<|4bwjGpFjEBO;0}lIM>2( zEsT&TA_u!y>KFkK*&I(!ft?#bvwu;tt?0LC(0GsY4?aE(k6-JT#w9qV`9oPU65qYy0^V zkDQ5oqzkbV;-nh8Z;%af+RO+xPGs9kNFcioK!BVU8RX`In?&1o9YO78B<6m$t5gx= zdkXa0K{H~3KG>O1;5Gf;E!%H6Pj|=FeS=4aQO%Om+Yo&v(C4bbfhex+wQ*82dgpM| z`cXA{bda9M#4GHbx&|xrh3{WH*Kq8SAXyF2lZrTF#F8-4_ats|MgTG z*vN(5BWHK(^RNH>#`=dtXDR*GhSUr+5Q_nPgBmfFHf9adTA;@OV~uyOg<7`K{`l7N z(rbxh>mLih;8%&F#F8LLUzgiwDP{HgkOVWwaMiVo)#VIq*uFW;ZVs`3ZC8Kv)8Fdr zeRLzZ{`W(V!1Z4Tn!9$+=UeXnWBd2t`1W(S%Ry9Mdf;GQlk=$A%J7|nJ_YZ`yf*;K z*_reyK=Tg14X5yhR1}D| zMeTtvo!wsl*S~n!Y7uC^*qox-RJK6*0fXSgvVT_@gw9d>1c4i1M$XU^Em>{D_qY`r zxXJfD{y>`i@^{~RYY<+c?_>jl3WX$KfeI!4^>Nf9M6jE-`) z^P!Jc+8Xbyr$ma>ObOMbKuU%%2ti7at+b2R(UaDd0KKxq=Smg5a*fXwlAgiGKZqZ@ z@Z);?xB-UAf5pgw?4Y(O5BG*P;!z%X9$QC)7s$MMJG7~qk_T-b5)!u3q%rskJRcc8 ze5>LTSBXOq7aloQ&+v8@Ep8PTH8z5xRW>M2KH-XE?a;RZ7>^5ceA;6=J{@RyvT5Mq z$a}CrAYimV<6%tOmKwu$4hu)*6o2^*{pIQB{+SgsFDu+=rjl{Ok?BFFR-WS$$J*gD z3uI@OxgxK<(Pj$Sz+)VNpE6W`5MqBx?h4jrUjbM%>PNr)=u^KBqHkiUK#9ThAi@CH>C)@Xz5E@-jJa%%)L#Rg+aQ%`xF zBn{H-UkcS1+d~&Iag!p`;GY@sIvAHmd;t}%Twtsn^Vg!3IBf?^B~EAT+J}DC`-4l( zT#4iFHib$YG?J{i=(B2|#DxbOQ%-I2Y8=AG@<*RRdvEleIp&r*ixD_2Y#k%pgp8r$ z>gD#ijw;9_lKo^8UOfsxX0Zy90=kGofk;vHPT^THlegK&vD@4z((YS%42q zOoNq}vAhg8e)p5jviF+Rt{7=Z2{X7Z%d#j-dI}6c#~HAyjsG<8yg#l1HJzb5Sx`pFIhy3>QMd>DaSrlLp$4CpDvt-yFHI>Giq3AeiN~lS(TM^`0UD6N3 zcY{DOvI73>Jz4hy=WjKmGQ5ntsK|^K-$$uwZVNb3*zcUR?8bsEv&Rr`#sz6oR>uWY z$J^2rFrz#R5~JpbT?QkG&>gu&V98FT0QQ+uXub#(G-Z397*g+HPv8XJ7AGHc3Zn&K z8fxE>#QR}TcFuXfHGHk`mJLCJNOkZ+5GQ9XGN)8!kh=?xrui(5lz?X=!=AE$$V1*U z-qOK(kJ}tP{+0bBH#Avi%oEiZ;e_9-?Qka+^z^4c1qg%}+*F9|WRsy;CMhOrU{!By zr29}j3aqUnqF{F3b~|gn9qU47Z~)vs_c?n7n^KY5-MvRLx!NFWhGIj}cx)iri+&8g z{RGl{Nq%wc?8Oi5U~&rcR-iFK=-vR=zutTQ+0T9TUpkn3!&;~(U^SNc73@0YbJDj% zlq&tNXk@(5yUrrTu$SJk_L-_FldWS_Do{+B%PJ9Y@8nXQbjA?nJ zzUnS-)E890MYlJR-smWU9Q9}0d3IDtO754QH|z>HHVBh`!~NrL@4MmPsjq&o3WMZo z5(0=42HE6NGutl)7P@sHL4Zhh7~~6KlD_PX$4MP5Ly2jWw&2Mf$6b%YcFVjFWjL)+ z#6~@nrUHtXjLsxPn(GG^?k4saa;HcfBKV-eOB98MBnljLQNd=NFt|s2nH(i>oEg|N zJ#0Y;Q%2y-H$BvwNu_2{nq{xk4pX#vKm3OgfohJGrW!93qSBnFDjIBdTTZaqt#A`A z^uYb_urjPPZzE*Ot8fCYEoG(^z9r|len^MqQEwt=SdLbr~fUPVI62s^>UoWN>%Qs-J%I>6VYbP(^}$ zq)t|aWS@|Lkgph71NKl+pHk)IOxn_!xfOg==TU233D7H4K{`V`SE}e0YAb_S!^@!! zS-BWEe#5;8ZHRB0?PaeW(Fn)racO6YQk%5os7*G3^tNQ+YF;pj9N z%^z_&6T1L)}wM~_8&PmGLqk48_79vh01n!CE=N1}B5&_utU7>N$Yy5q62C}Iu;2@koV z9=`zdAiS8XpIJdyb z{{4(*;)oPsBllX2yAIyh`o*ucynBwiH)DBVOIu__G?Y?C2Q40Q#Y3)m$Q6UDghhjg zT=9@A9&+X9k-+L#}e8SF*#7=1jE3L#`mHnH@;A2qz;nOY@K` z90eY7b(uo0_?Dal^Obc+&qJ=B`7I9c5{FXqdCliFpVzJ&7d+&Oscmr_f;=gly?b`{ zLo^|XSAot$u58##md3yV-($I_>|_SNNuFzm^6X#40wN@pkE$x#Y3)^DOeg`n@Y7d@sKM!!f%eno`+mj zO32k!3CLefWe|ih9stZ>0JmPWGjMD^h=y6&3mNFKg?;U$&8U9T#AlIx3(r7?4R@A=<7&~oqDTJEIkpBJD+KJ4Z2aV-sMzedWGfG)FyWZbsTr02cw)INXc zpZ*yAA<{F2_d!iCbuh+XtMFF|-Gi~rI{c`I(Cdz(I=o<&BGljy+?>rBvTAVgy$fG% zl!U*0^c|+Z6d>G80fs0h4bth4;Y*8-*!s{&R?&cX3j^xLczXkWeu|J9$UDCFqt5-+0sPF`_oz*kXRR&n*$B9517l!<)4o{d9Vv;+6;oo$2LshcgJw!wb=k^ z;6dZIsIhCM#ZYjM3)|y@Pw-044$ddI!^4U-KBGjn94EM25_%K3^MkN}|2ZtdwaBXB z?^othT(QwUK<@&HyWCnn$XvFfzB{G^IfR4hut z*xqg`WZL%Zdg+t$^FQ1CD%UVBI}KyS$|+nxq^X&xJPpYd(WGH3lPDuIRRbkg?s5S! zpJ3+|b(Y6V81h{Ra#T;0%)I?8kH&Io1jI_+oN(!|Vx&Wr=J*>>O90h*r|%r|l@14K ztqv#8YbkCv1-X4im7QJenO*w(i5Gr-67osJNrMP!4xK9VT7pc-CtIG4c=gd;kc-0K z{*}W`ybrn=CFUkw0^@gkn<<+feC6DC?>}Mxq2m<40A@(`NPMqC>6Iu7Bw z@+&>yC82^5|KPB4T;lJg%x%eM7SDl)^j97@$>qA~E0Z?oUQ-C*D<+a096)u|F;XLF zk}(@2wXXdJwf(}X0-Nk_zirEdRme|ML;gq%33GcsUoq_+FAr2>Pll*3(&D|6CP zn91F?LsK}*p=rJWo>V$Q+T+%~B5*r>)m5Qv{gq269$YMrC(1+I{p|9%pDmFxhigqL zMr-1va#*xYtvId8s3x3*;1+KPabB&-SyNzI3iDT|W&Ya#?=|W-);)FGN0^S`=Atw< zzWth-c=k-^W1rbd)i%wjwh5o=b0=BCXW)_tFfz4?`{zj|tdYi{yuZfL55mF5>( zRZ`7O!%ETI@R`LmH@J`Cnj5wqVUaD%UvtCfHJ{gfUJD!|k4^)m1UQ_ip-OdyfHp!q zx#oszZo)L{6et&c{H}%sO-PA`sL7(Pd)*3{62J+rCoh|u|IVBZxRy~2DJ6z=^aBpx zrAo;GC{<6dpgd_!%o}=Gnw;Y9(8@HWwroqm;@qM4TKb^2*I)UYXU|-0c!p~({52P) z@FRk?gImySP}3X_EFXVP6bs;>k^$P;>y}LMf&N8!a0?o@pg~f}u6pLkU?(-D(!CEogOvt$8sxYycgW^G3zLowva5&(nbVFzR!zu)0eG<#n6wUtEJg;mg8tDeA32H9I+2(6y6dfeqVIzP;EYbE8RIxSj72QR>D z&nennU`kMgNl^>42Cu*yT8M18Y3>4?g0DJ6{^ENd z`@;@eOdw&97KZwR^+%lBz`_7Cn(;E)v~EokX0aVJ_x;=U&woew+bR->k!l@Myif)} zPE{l8rKn~%&QQ-YQb{kl-T6;{!XMT_t;o#VUu9qxKXtOgTano4N}t+?;aVMA@G`T0d9sN1zZj~dFd9zexX61vwrsxkGvtA)YR~#sKC5P zgzsSgdj&%X(FJ&ZWI6(lPZdxUGXD@Ci_AS#&@YKz203ru6af*sS5 zI%OepZ0e(#p;H=GBB zIP|t}@W?Qx2+Qehh`ti&bJgHLlq&qYC4vTlZ~`BRAT`PYX}sS85uU| zJM*>Mz7rECn#5(;k8RKQ7$ijt!4s>oy=Vh4$Tss>fG0Tc3aZ=Qq|r96C$sw>G3^ zpn+Hnz}VD?skAX`kk$e{28}D;y%uWOn)4nR8Nj%_^jhNB`p3dA_*J4Pu_Or6*X8zE zN?E-=BtcM?;i_vFtIHYKuzhow-5g^7+OGcSr@z(L`{+hcUiCwdz!0Q^4&mB4pKrPQ zkL}-owU z3zeS39X)AX z3D7G$e6Cc{E7$m3A!%wG_#l?C3qP*Mj~if^{8tQrzaS6yhBo3+zNipp6___~hc;DH z@}SK_Lc&&>GzMRR=Oe?1Z&id#T$Lhm)iYSNMT=X-1qYAd5}b{FmQT2<=8$UW{=Cq9 zV(oiO|JDNyPc{ua9C;5G2-JW3GakmYZK*MA=T~=xVDXpV&|jW@?w?sP^RmKarCU1$ zA}Qq(l#N|>fEZ-S(5aQ@pUNTC`ql9fz$-|ru(>)cO|s2aY!s;2UEI%O7M6?Y+@==9p_%79&8m4F_ztTgS*YA!Dec;M2=|IhnpD zsvwU@_LEI`^(bgj`J$@PahUDx)>w% zY3H#q(wYXBFN|TD1P6J&H-ipfb_xAt2Sx<-pli7dO#yELs|AN88S3pP+p}*N1@H~9 zlCkS!q#5e$kyRb~(<6%lxap3O7C2|Y0{5t?G}a77$4OH{O^V%$AkXTOei*(R1aeMl zlMw;nJEp`41rLEAQiQ#4h`;gP51hX>DNYC(+3+%M>a{Dv^uNj#z2gZrfJa+=AElgt;w?>nZ2{iU?dT`Bew`FS+jkUvCou3 z^Ys%=*`6nc)O*+y$OE^<$p@XnXhBCI!=dj;;{7lvJLkOL8opL|%Z8uSwQ3=?-_6DV7)iPl*qp_jzDf`vd)+%s;vO0>Hv*(a(x5{ zgcsaYi0x#Pp;;y=CTd_+Z)~LdP&^8(ts zE7*0)=cI3kC{_Aj(a3nAcb!FwVK3n!DfR(V9tU3Tz2YUwZykSZ{q4^_^b+f;zr&fi zrp;Ypp^A=n{p7RfulvkvSP_o>ylijNz)!Lw?#!1vM+BV79oCx+)!Bqt+@KkYb}z)% zg~lv~V)CU>)@kHRPN^n$H*4Kd@LsE;Y6F&Xe4>uJSybcK5Pp2XLmQruFTYTA7}N4b zebrsws4u8~i*9csz0pwyIqJ{0^X#Z_C2_y>ykS?su|b&h8}1)}d*2NQPkr@!Rp%3kA#}rNLE}`>E7Vp7v4#$YhrQ`8yEbIyV&M1<_ad|*ON^7aMA{LK(KCOH zo|YW7$tJohTOx2azBW0EHNH08j{7J=Y@jugwF2?ANyG^5Mn@%2X$IAwuryvFntHxA z&0D3v_R3W0`>#y_U)dAbM$rF)pXfx{jN7PP*ltojdr6?2HWD5!T`AGhYejI-0ocvi57oi~C3N2ehiod$duZQL0bVhl&8VPMX5#RYXF zN2d`Y2*MMwjG!EyCMRmHo3#nEXlli6nBu~v=Xm){4@U*LGYXpxJD zT;+J)+aABnj~l(pdB|1UK{MoPf7R)y2V2*FY!mbE<+wQ*VMo_Is;%l`F1N^z!O#AE z$ey(SI69IRkundX5*&SL*TMT*zxcJ5ch6DxW-Jec_pahfq5#os33dvEBJe1mgBB0D z;vrW&8j(gdhw7eJ4O855l}qj$_i`ZAyVcu#cu-UbHiC z%vZBSu`qH?&so_EnX{~=PWr@%r8%#D(p1dJzJ;^ku2SU%=rEmBJN7;Qy9ZkCJzLA2 zRF|DatTd?oWr?yty?JH{$+&HwNzZ%VseS&?Km9TKL!@Wa;8Sflbuh+Xt6-5UrF$@z zS%)9>PboyiX z(xM|a(}`pi4GIPrP`AP14fy#fLTVuI_}Y`^wO35OR-yD-GIK+vX}a0)S3O`3|K*>L zJbAE7uSaE)(&Q3Crnb^zD3+sTwoD+me1d~dle49*ufn#NUf>vT*Nd>3x+RND(N^J*=?EwzWYJKS*}s4A zAHGvsok@^Xnm~$zX!+-%SxY7EmAG24v?RU=WB?Jgfua@{Cmv>67yFKim8&*Dx+S4P(W+h@wch9Cv_R zKmjK({OJlCYAzW8}rH8q3MEMd9E5{`& zB$qO`C7)S52OiQN56HA&UD;-*EZUrVO(B4 z_Q{qFpIJw2$HH)yW8rU-=Nq4pH=kjSisNx`R`ksW1j%|&T!eET&u@$8w-$3C-_ zs%@H4Z3Dr;j##r2G&gP5u9|oxaOwh z3h)wz2rb+jx`}y14@;9%+#Na(ol}+8L`Lpc$45ZxlvX9hI4n(S0_wYBqZn=tn4#cG z4)^rQf^k?#r$xfk!!isM8M14l(O znT3moNX2*r$r=*eL%-t3nqaWa{1GHI@C~m6!=Nx&nCLS+oKXxZC0ZD@ZwJg#-kSFo zHlEj(FW&6s}SXfm#)5bYgel&K)LQHMY1AaD|7iE5YanyRT^@invlO? zI&2_DWxL!@DHsLnyrF`mE*Pq& zhrw?T&YSW(5QFD;7)RV74QWkH0826PnC*(5Gf1v^*Qv;2B$8^@5>wI{V~CZdY6iKA zRpvcy*16WKlU}rIlnkFlx4AVMf2NjV%&1xBia|YR*d(y113Yu%x6S&DBdkYMq z)iYL)yZlDyhgoi|q?}Zz84f(Ww%`SL?Kwre3rq=$Fez$b*5Ebxj~0b5d7IV3;G2Mc z?aT+@1fl?_R`u+Q>wb+~PjAZ@k{s@_{T1}k5hbN2r3ndr3F4A&yw~`Fwy(VTNE5ZT zSIck)E35_Yx0~iJz$y5uL*y^M_pv|hpv43d25E6&bUU|!xdCQ0<7KpI-I^xMVmoH; z`?u|%|BmptRU{B2)jB{UQwBgzRU_-A5N9{e04wHvcRsM__%1o$Whx5JcNvr7d^gPb z?mYP}%Ec)420UVMz$F)hM9k!35TLww!38+uNYE6UvS3-vM+EaYQAr#A5WR0=3i-Tv*KV;UDWE3Rv_UR0gMld*L^!0!S));1BD-^6}5~XL<1- z10=M6&z|H&XXpOidlCl@?Cxk6#&-*y!k*oGC-&`aZ{M>=>_~PJa`7d23_AxkEm01H znupB|i9c8jUFhv#0DE>!1A=zgi~5YMhNCJ-R>jGdq&OjDWS4Iq6}9uHC;&bTNj>sp z?S&Xko&y1dwK%vAi^)4#*7%MK19vyc7v2T2XN=Sf2^7Z%wWO$leAev9XXf2CtA6^; zr&~V$LKO-2kvdrwl6^t~#QhjqV}PUx04#EHCVkuO^c{K#Z_^#LpPz%yuPq8NDCt<3CJna>ksg1 zhV@2g#01Els=^c_%O(MWOXg=|&0Zn>s(qDwqE zGM^bqIlkj+?E?SLO(+x~$4XPQcZH!#2&pFxHl96ImQYHk12=!%s^uP)*k&13+Xw#h zC!f3N$;Tf@^%w^k<{-mi8q&GAxGdyJ+@{KHsuCtC6g7`YZ;`lY+@_jitGm){syr&O zQlb(YL?x3#JAk-ofoaTFuw&LQM@aox99U>= zKY!wpGm(#UVeo96RAcuIvLQ~J8NtS(tqZn#pmJCTK5R~lz*cr zTs72mED4qh+8U~{02^coOivkrfhIfiwcEZEI?!kmmtjA)J>O%H`!WPitj6}Db=M%< z%v+sM-AD{PB+&Q?H*WT6v62$fkdij{xm3nLWkZB%ulyc836J8C)>D%eHCVxZr5a6L zY~;f3k+Zw?`PYAbWBtRSvy^^oLuv*Zh{b>=tVT?wjah@V7U(h9((vxJP|Mbw_sGZq z#^t5g635m*7Jk965=Dt6!8!4|+&)VwtJjAlm^qkX*Q#q5tIHYKuzhow-5g^7+OGcS zr@z(L`{+hcUiCwdAdpuFn!9$+=UeXnWBd2t`1W(S%Rwgqdf;GQlk=$A%CM;bGw%(6 zVs<8d3Uc!V--c88LaGTo&opLq(+!ha1Z(U1SH5um5BEKI0GbGhDkqVfK@(k@?g8pf z@W7YOZm<9AUp#EJ2(({pPEozt7AQYp5S&={?<#}MQTqge8(>Dx&=W0LZNstM3Ju)k z`yPKFO@8^i@4Yn$uh4h00YQaA5-9%yYNmv0QUxUq7=$1tl2+Qq>*z`AN`PM3;d7;mUb)8S3Q1F|?+3ArUHEZ5 ze%t`Vrz-E>_RbGZ!sx6^8*af=h5V#AQC=s+uEbnjs4F^Ubot81|T74m3R3H1KfbJy;;P z#*H+)Ru-zK?V=nS#%AT=FL2~cKC?K&o;arlnn|F}<{Nkg9_16kmZL6v`QtCN^A!`d z=b&4tGcu5-k1mgf--=nPP%7|9dLJC>8N_E1q3zzlkgkLGwSMtyE$^PQx3sXYTBN$4 z&ufmqz>G+g#hXkXH8?y^Ufy($-f78^GdXf*`PgWQmY#)cOjaA(?4s2r(E^qe+jJAA z>XK2Y&Q6t@Pgm0PGt&^&UR1%JGJkyj)Q^7o(WibNB;cD^suUw&`VhT-iU!$T&|;P4 z)(BF_25|UPPkEgr4btsj3VNMI?4gU8-tiIi28(#Tvr1LSQQUC*Tt^k;5y^hC39lZ7&~GgzXwzsO)4&`yMm8iRiaTt2mg4hzKiOb2s9_O! zdao!8vt#6{2|pOB5XqBYLp;jd^3t+&B zX^huZl?iDwqhYHyL-4Enn$@ltX-Ek(82AoI%n-Dg0>c;>r6BTmaKat~Y?>tM;5e^R z|MX!oql+<8pLQM_Bduw$kYRM{Bsj?Hy&1HZ*(LOo9T@A`gRbQ=GzAfG%#k@P$xv@U z*`9sFCArYEtZ01bJ4M z^uzGoAdopuo6MWRp(pEp;QXy-REC#v7ZsV&;`=By&20fE3j3Y2mfcveW%d~2&A1>< z%Idg)s6sYe0WhOH3lgK|h+PIFiO?OnMPSLAI0*ry=kY5}-d+|dMM2*I#toR$(d|uSuQ(wo5()yiJQ(DtKikf;qr#PBn66gLHPMbg6E$zaz_CG? z^c(IUe|z5z2Ty(VdsXNLSd$PylrYF9mthuMK-tC*7P@sHL4Zhh7~~7UmcW<0v3wXW z=`?W3d4(GlJsdt9wf@M!$WXK=))$NRdZ9H62Sicg?u5aeS4*ei#f6>uRxmKiYAnR% zK{aIzlCmm7XSdV%a&OBCHoFzq2>1i;hliCfv3VOITV90|aBV5`Ch{#gN9jg7Ew$>;0>BASm>c8` zWQNcUqjjZ6kLEHMJ^5@|;?Qdrx*D7DGjD^qRK zlA|`+L}iO50%zlElOx;lwdr@PEL1_k-#w$cq&)24TtMpgY z+7$4WJ%Mcm{V({5PL$2KjoO9nCgro2lod#h}mnIRS0j z=s%)UpaT4Jqc#&SGxrT5qZ=fsAvk~D4o`7(8cVJy zcDyfptW_iU<@bT}7dW0TTIAv(SJdMd-0-D{zi&EvmGh9Rwu5HK)&8o}PY<@P|JWwR z%lrYmvPE`GElia;j$A7K@*Dcg)6e}gD`sB4yQ3I}@04^13U?T6HK4sgayVlaKn`V5 znKY)3lcottl#|XAWkvTucFaSrrsl>E@g^K~-XKc}#zU@n$Q8I^^N_2QsyN|NJmdEyBqN#e5!eg`>bj zt}au^72lHc&C!<4xC)+6Vo86o#Mhg^wTg(B7PkSjA|9T0RLkT8h4^#=)q3)vd$!f=O0V~6o@VA6!E zR94V054qwYSFi*@a!|fD388LH6K1LDjE7vIM;Q;f@m}F43Z}L^~6YYIMyAHjYTm5 zX?Zl~)lZuEEOJq*3EGDR=LP66om4yaJ^#B0TJAkt%bisH^8)146q&%hP7s1O{s1Wp z4yu`^_;vs$UFuX-r9ti2NExCmFacd=3CXx^pGnVq->H56&_De#`a`5=w7FAlI30Za zwF=$|X00q$mM<)`4nOM4uTq2>{GGq&P+Ge9-i0qWO2S`0`VP}y3USPC52*1;gLL|1 z_|l>ywmvkHr4I!K0}?q7D*WN`2K@XKAvKV9eCBh)8n5jTB4$rQ#T5$ae>pvvc)vk_dTyCOmM%3tx4Y#`VIir4si_9Eqt?keC`p z&<2WHT%3SpWQ$Y=`s+`!H#H*(K^CI4R?GE4)LauQKsFw|WbT;DBq*LXcY-NjzQ7u6 z@AborQZTl+n+ln>J-c4|r2PEPHowX>jLS~L$ajF?LYX};vUFhc^$E09Wn4hy0wNa> zRd-s8r4bOxsPFpcFBfNL-Y8=#4lW&5jC80{m&yUP1W=uK`VyP3bT~+Bb)-Y)DQn4y zpBQ=Ow@f~)ZZ-wEeMOa>UF?}%`um9&etj~Cxi1<-@G5nx%xeiUA)jnrPQ+{P?1E&i z{x0=7)5`lG^au}TCAp}3zorpUD2{7 z+K3mf=b#jA(lid?x+SqLpz;y_;IMLB;_s!*ZOLaA56DFN;{loOfXp&!bM7^T0KQ@( z$-x0sciAxwOCbe8S61AD%dLH3YKF<)Zc@T77eg%-u~zDan#n`l{VdlC={4^yXKNpK zwXYag`^JPS%VGk9Q5S7fD^6ZG`IWA}dTN4eZt`nxAjiha z6C|rjs<~-cDViHTv$*C4_ZnPtb7g66_`K%xn$K%@qOI(RHSUJP-Eg=Y&gF2!;r=d_ z;_uSLyrGAs$tms*txN+$%ys)*oICVhOCQws`YV6)?3s%V&v4C!zviM;Mv0Z?vcfHB z+=3R>zlis_JR%u{gw9^s@A#(dV{VK!VA zEbZF?b6~gTy+!%E6Sa5Lb$t3im_)R}xzQ>_`QfFjZ{6C}YRXKmJ4%tC$QQx;IH`jd zZUXO?6!j1tLRY0TnuVmuh7F|4ER^41--m9w<KSPh1aZO+)_JRq!j_Oa_b zFWyg`pz35v5eG74sg$_s8LTw0 z(;&xlW<=?vV}tC1ei&k57&8`uWsp@9vS1t*(iC{H4u<3nZ0K72Sce}C_JwwEW6x=} zZfK&Af>EI2y_yy^!B90F;vDgOA1fP(!Sg$eBW{p}w5BFRT{j_^?TVf=NUlZKsmS6T zl4{lxQ_>k@h?S*k09nf_^PV>ATx-@zFWNOqhR=$gTcaW9(|UMN5$V&cmQgrokj=R# z%?hiaxmG=an+&qIzz|wJWA(VpZ*+c`<AQ`ML`c8QBrDBnvl?aAP(u@ z-fR3o+gILvq=_2Xt7SNY71lyz!%cG+;1qn-A@UdB``90L&|(4!gS5C{xt-epI}2ts z<7KpI6jT)3F>~L)ZU6jtguksKff%XQA$1dF0OS-VOJ=BMH_lMMd}PQ07e`(L*YO Date: Wed, 2 Jan 2019 19:10:59 +0800 Subject: [PATCH 004/147] Merge pull request #7242 from hequn8128/flink-10781 [FLINK-10781][fs] Update BucketingSinkMigrationTest for Flink 1.7 --- .../fs/bucketing/BucketingSinkMigrationTest.java | 3 ++- ...cketing-sink-migration-test-flink1.7-snapshot | Bin 0 -> 1952 bytes 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.7-snapshot diff --git a/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java b/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java index 981a529ddd2de1..fb0796aec6c912 100644 --- a/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java +++ b/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java @@ -90,7 +90,8 @@ public static Collection> parameters () { Tuple2.of(MigrationVersion.v1_3, "/var/folders/tv/b_1d8fvx23dgk1_xs8db_95h0000gn/T/junit4273542175898623023/junit3801102997056424640/1970-01-01--01/part-0-"), Tuple2.of(MigrationVersion.v1_4, "/var/folders/tv/b_1d8fvx23dgk1_xs8db_95h0000gn/T/junit3198043255809479705/junit8947526563966405708/1970-01-01--01/part-0-"), Tuple2.of(MigrationVersion.v1_5, "/tmp/junit4927100426019463155/junit2465610012100182280/1970-01-01--00/part-0-"), - Tuple2.of(MigrationVersion.v1_6, "/tmp/junit3459711376354834545/junit5114611885650086135/1970-01-01--00/part-0-")); + Tuple2.of(MigrationVersion.v1_6, "/tmp/junit3459711376354834545/junit5114611885650086135/1970-01-01--00/part-0-"), + Tuple2.of(MigrationVersion.v1_7, "/var/folders/r2/tdhx810x7yxb7q9_brnp49x40000gp/T/junit4288325607215628863/junit8132783417241536320/1970-01-01--08/part-0-")); } private final MigrationVersion testMigrateVersion; diff --git a/flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.7-snapshot b/flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.7-snapshot new file mode 100644 index 0000000000000000000000000000000000000000..9aeb1c28ee96ae5904f3820bdcff65a07f0af720 GIT binary patch literal 1952 zcmc&!y^j+|6n}dkkZ=bviG-8_2`RMuy|&k)AiiJ|TMKvcE+0zcjMsDaa_iZ>nepA; z6_L_}f`TYY3JSV3iTEd?M50K9M2ZwNP^68-pTL_n$VZL>6e%~dci+7C=J($Fy|)7} zU)mDN8k`(V<1H$6A&I0ynYEdZ_PYk>>#jYtT;J1caqQ?GiR*e}=nr+PS{*bRq*@#J z1mO5ZJhJ>IFomVTx%G`!uNkfP`eD?JTK!9%FzR*Iu0@^otNknMms(H>H-b(yH?+!g zmFvymTB{$ndYxv_`J&ZB?E+lR_{bnRiO1B~OcS<+o5YB-$s}Wj+{@`yCaEws2qzON zDNm^o5voJV6OtzTlz*17&1AG6{`R~K1Mt^_e_&Z~&<{cLDU->B8YMkOm+X)^Ogp^! z=tu9wcMAXohqXgz-(LNMiNKVNFb49Khqp{4mFgArT8z7}$D@O?>tZ3_es3xk+rp!kRV0GqN8ia%o zf?LR^soG^;A^C+O#}=GcIz#D<&xshLe);&5Uw%FF<>N(IXv2v#Bf~aP!8`E29CIqh zSvoB8i=URER(IdScLo2Ngl%(&aB~ywlnawPrW}q7-?obSUNNX|H~It4a<5T%7Oqil zM&_+4OQh%czUx$LR^73yHQdzPnZdVRr|!F+U3WaY>egJxGVMm)(k)y4bj!yAg8xk1 zjK+V_cdQGmF*ZOZ8H;ctyKp9+a!#3SCn>#-gFOr=8_97OPNzi38$6McvM5u|0jwNw zmhj<>`INAc8A^^FyMW!vDI21w@UT0T1KkM{A<^UGgB~0B9gS(cm1hZ)+wc`=<-nC) zc=&_TEEse-X3}ED5|9X8G@`dl3qu zN?JTTRfu+~L{sD$L59@b`TN^H?tb!g0kkeG?~rs#@j9r?ovuzN1IoX7@Wa`a@1Nay k^<|dalu=etTq?sZ9VJoLQ3kh Date: Sat, 22 Dec 2018 21:26:13 +0800 Subject: [PATCH 005/147] [FLINK-10780][tests] Update Java / Scala StatefulJobWBroadcastStateMigrationITCase for 1.7 --- ...tefulJobWBroadcastStateMigrationITCase.java | 4 +++- .../_metadata | Bin 0 -> 17802 bytes .../_metadata | Bin 0 -> 37882 bytes .../_metadata | Bin 0 -> 287008 bytes .../_metadata | Bin 0 -> 287008 bytes ...efulJobWBroadcastStateMigrationITCase.scala | 6 ++++-- 6 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.7-rocksdb-savepoint/_metadata create mode 100644 flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.7-savepoint/_metadata create mode 100644 flink-tests/src/test/resources/stateful-scala-with-broadcast-udf-migration-itcase-flink1.7-jobmanager-savepoint/_metadata create mode 100644 flink-tests/src/test/resources/stateful-scala-with-broadcast-udf-migration-itcase-flink1.7-rocksdb-savepoint/_metadata diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobWBroadcastStateMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobWBroadcastStateMigrationITCase.java index 3f49d84d59594b..da63b9b3096a3c 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobWBroadcastStateMigrationITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobWBroadcastStateMigrationITCase.java @@ -65,7 +65,9 @@ public static Collection> parameters () { Tuple2.of(MigrationVersion.v1_5, StateBackendLoader.MEMORY_STATE_BACKEND_NAME), Tuple2.of(MigrationVersion.v1_5, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME), Tuple2.of(MigrationVersion.v1_6, StateBackendLoader.MEMORY_STATE_BACKEND_NAME), - Tuple2.of(MigrationVersion.v1_6, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME)); + Tuple2.of(MigrationVersion.v1_6, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME), + Tuple2.of(MigrationVersion.v1_7, StateBackendLoader.MEMORY_STATE_BACKEND_NAME), + Tuple2.of(MigrationVersion.v1_7, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME)); } private final MigrationVersion testMigrateVersion; diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.7-rocksdb-savepoint/_metadata b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.7-rocksdb-savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..8ef80ec4cfe913b7b58db52fdc38349f6bffe65d GIT binary patch literal 17802 zcmeHPUyK|#8K2!t5LFRMC=?M1B}hkA6?x?ud+f15h)c7Sqt{$?ySXZrs>2@JmkoEf z$IM0~5fb77s=OhJR6vMUtrWEo=mQVXhgLiQ6^eN2OI7{|sgJ0spjH3y0O9lO&h_SY z^Y?Pe=3+~;p0USXk3WxpKY!onnf}@$3WA^xhYO$RN5O~pe(uqC&dB!JGkee7^C|v= zmms|c$Bsel&Vvhs;Lah``k5ceyT`NCpu8u0rYz&~-*-Q7+Z}^=?;sYnV3KfaBuhgS z$2dZiQHByGDI!$LM8y;f&Y{N&cg@cytsMS{M5|h=0dAEH$62o{YY~TQtLsxn=Llvs!WfXhF$8c zd>t-2jkLSG++At(POh5cz0OkJP%<}-Rxj(UEVfM6kxQLZCY#)t&{~nJ`O$8#sQL4o zs5uCBf5l{GpECP&CKtOa`DgAf6PCM>QKs%GQ%{=SzDmxg%bea(137um1=Whct*=0_okh{%Dj~UF>*d9K$Nst5sRQ>ir;xWHyP@6mQ9vJ zVmWR76Y{?emlpo&FN2$3KYKpdnP3p zieg}lCLhDF(Z>u|q>n^~U?T<5D2fb6G_e*Xi6jW8IuX_|98(IndL#2Qp%x*bBo-ct@HN`aIR8l@(*EH*r%3Y2YL zJ9lY*>9unkjV9Ip@@pc%iYqcU-MD_+^qS~nuf8V!JO98J&VK8{hbBJxlPQ=|oVCCm zZiXo=2cwXx9t>&kqICp$;)}!i;rjezZrX~Royi&$+f;)cG15qA5+Py)O9|s7v0$&5 z_!YFn*_lTDWs&TfdY$Y{W2c`VrPmu%yXVk=a3-d{%x@>@jNK4|w|LF&7p&R+YHN1C z`oQkjxBL}vJP2-MDEUSiJHfh8#--J;AD36<-A;2$h&3Ul9GOUf4N+64xupsjjU!+$ zIE=K{+=9PC6+QaB>}`l6ZFPlfN438^OWB4j1zv*MRTRK2})yFMblKn;)#K4 zgOWJ0G&F*$#IulLKNrtJuE6cA>d`YbwWzP4Qs-&hyXfU%dIZFFp0vk5B)04?6-=iqo=| zej|m(bree(-VWaPF@fR(n{n;q#q<=^?i<7hV5TxwV5V(|3;`Vpl^Av-Vw;Fj8wd_#7h?|wWxwdec6yZ1f(!om|zz4s^R zCMWq)`h^d{2I)igAZ)bcmFWQ07cH}q@eaxyd-Un^Po944d(VC2ji3GI@auO%@=a++ z=WP@|_{1OI`_K6k`H#ZIx1b$c`8Hszxnf|xz&HWz0du&W7zw&V4qsphQ)xpmS^Bpj zyD{HlNdNxRlfSz literal 0 HcmV?d00001 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.7-savepoint/_metadata b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink1.7-savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..645f735a0464ffb040e255af22325a0dc52efb7f GIT binary patch literal 37882 zcmeHQO^h5z74DhMT3E!$4iJQ;;EgaS3aZ(z{%-{#*4ZT)vg}$rvx^cbTB@q6*OTnd zY`S~FYatL9qTCQc3LpegkSqcwIdA|DAP)QpWf}b7D&!_UUU(!x9!)44c1)&Ca zh-KK6c#azq>N|$8CAGP&L(&aMG7AOid4|d}{miYNU0PeHudZ#@>I?Oy&Bbc1zE(Z6 zUazjMY<_HYaY>t9UxDlN$HzAome-dyYfEd@h2`oaOKT8X*6OWjtHO4fI43LNcBAysR zCmxiMoa1sXh}c%E8Q*lUU#Qj7M2D)W-}cbb z!~Ifu_8Meoxz!_+y|+CQ>5rW&)M_dXaMsGuFVmw`rukL5Y~)unWm8?vT02^y>~Nd?8s;{)phf` zQA@};2BomM7t4sGaWRy=^Z$p7;>JKr#7QLC4&;>TpVqF7Skj8(N?kp!OXc+4-`{G6 zjjf`UFVueecX&F~dgepFoRGTpA}<_YVw+^U!{KmPH`biHx>Em-hCS zOuco~nsy7+TWM@H@SdK*Nq$l`6C9V+L>xI+KCk>2_mx|*2S+!=9&;V-Fr+n`@&$Dh9;u7G&==x<4TrEsVw zK)k#E4Zy^7BjO6Z@9amA#);KEj!QRoTvF@6d?lCYG?=j9dB|6|X>o-vq$$M}Y#o>hu1<*~OiC;ZB*F^F@l|AuL;@d-rK^~2;{1KefBg;Dc%Bnew@Eu$*A@!iYgs5$2E>Yz5?1S)gy~#W6?W@aao1|Jl&YlD3}swOd6ECgz``au1gSt&re{z zP!?L2p}U6Q$^{)_N{7P-En-U0VWx7lcvCPXrA4|iB{Kl8E1W_1_CrZ5%3Q*!s}shA z&h5ZAI0a@K@|cjvRQStAObIb1#FP&5;5)6@2=bVa$JEUBH|RLzML*-kq6!vE=w}@L zj2~eBn|g%rCDbCn_Il?8XdiW>?W0EXyD|w($#o3R(5X#A#|yx_ry+@F*)Cxef=yvq z@P(Mt;0`j#Q<@e-hhZ!-f*IT)LGqbjHzq?o0wxV!{KLzDy7SR&a`^1k454j+g|F z?Km_vyuby|uEDTQXGa>g?NNu@=x+)4oyL8qVe{>_enU*DQx`*ZJ=(tE+#Toc+1wp5CB&2vQ&NBu z9Ed5UYH{eGIb?yPY79R3t^?n7G*|%zwfoU-Djwtd0aP4=d?l!I)IRs(i@|sP_Jx%kW`qN>yaV72cxV0AvNm5`_WtDqsDdo;qO7BpCkj1_Ec@bPJ z*=}G7$8w3~FtzXm9)qXD=2Ea%v0<3fbOOtVBn*)zBXP2w>&Bbc1 zzE(Z6USCq>(2j2`EUzye5(Gy(@t}<49G7!J#I{<^`0hK4fSndZ6rm@J(0!TA^~Bs& z#3bh=#Keh=lXkMKO&)vnxy#>p>dDVP|HNOv{{8ir-wL4zdE3eos_*{&$~!UrJC1c? z`hAAyF(;mmE}Z;=_Ro7i{=>~@o_+0?5d5Z;qwy-C4?Xkq*Zy_+eEem)^$L_jiJU4+ zCVFr&!QNnaE)RU^6T@bfqV@>r4TehsVT-_%fhRb)Ai~}NJ7nC5-XQ+?wf*nibI<$e zPrmiDmp-xuRXwb#`bWOFn>pUTfZ=t2w7Ht=*$iSRjWrX~Y$h!285q!l{M db}gw321z#{$@S>M+4Ys`>WV4^D!>5+_&<9qqDcS% literal 0 HcmV?d00001 diff --git a/flink-tests/src/test/resources/stateful-scala-with-broadcast-udf-migration-itcase-flink1.7-jobmanager-savepoint/_metadata b/flink-tests/src/test/resources/stateful-scala-with-broadcast-udf-migration-itcase-flink1.7-jobmanager-savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..88cee1bf675ab867573b92798a813e660d5cbe2b GIT binary patch literal 287008 zcmeIbdvIJ=l{dP(CHW!Qmi&nA#E-n11QXQl*3*hpB(l^_TC^o2$uX7_%u)AgsiW?5 znsZLu5|_zL3PKF2Nv6nzFJ!)?xEB``kgqZ|6eK{X3WN|Qd=z9RVVD$oQBVZF1o&gP z+?kNfU2E_2>^|L+tZun&t>bd5`#kpEXFt|n`?uEm?f8eM+8JX(_>ad$;ty*&^ryz( zf8+nwum8m_{_fBJ^TdxXgY$u+|Lnib!+a*t$Zu%T)q<9k#|+7k@d`c$$LcQxSSZ=i z*V&ayrQ7#)c67CO$z9#;Qg<@lzPB$r*_}>y$f-_vX7v+re-k`dl$0pIWg~}Uqy6I} zqbJ73`^RG^2IFJnqwxbr#^WQyCk~Aa##r0f;Y57gxe$Zo0<7)k{=|{kiLuydygw12 zh>gP8Ad9QoR9MPO$!R&9&SsPu_>l?UCY_YR##~-57@4dd)(VP|$;sjILOv^x$y!Fr zWwS}1*sC9a#NlP}2n1PUoHa>W#+Zwde8^r&YQc-r?CTzNXh%2V~}cAHdJNq%XfX{neeF?7VsnNcHT+*7Lja1Emu5Dt@&iR zb|~rc1>I0{b`SkS>P+hwv)fJwS#XF2CKD_m8EliQnMK9y=Z%C?0LssgH=z671wTK+ zSi>qfgV%m#4!*jr_;QsrPwx_M?CPv+d=V{I+_gf*CoC}r*#wu`X|Wl8X*NT3x>wbE zf7Ke{@FE()JA6XkpPmLd`&z?lUygCO@g`Z5FPoGvOKH-Ss_2)S{qz44fB(M+|8CE9 zk4C@$*Dw6fDkXJE(TTKcJdM`U6h)RhOWoKk&7|@+)pto4QDU2$$$wyg3)3Jy)#}NPqCv zAD+DUf4r3WRIBmT&ukl6uUz;4fbM|sf`AyrUtj$p$Tm)Ds+3Afy3wv92GQPus|8d% z;Aj{uy&iR6{~(|ga-X!X7x0MozGzQRdsnh2+CJGcnQl*|Cwt{&cV|ye7c71o{wJi@ z1pocAa?gO-6C4;HmbLg%=}uX z?g|NGIk*@8Md4ou{Ob&MLc`!V_dCpCYz=1{2*4qXM}zny{#rlGhq@ktu`P9_)BE!c28gN*bybXKRgVO$H!2o0`pEe}ibnQ)}RF8-AZ<{6$yI zdu5&dOweF$Jby$=!y}Wo$w>p=87F56bC` zlEJjFOB(E`f5uozF7(+%`y<8gZkI&Xya@v?35ZKcU5h5ZlFe(DDW45Cp_6}U zR<~oH*<`S$fkgk<*eru)!ZRD89AGeVYEldyI2-gi>vtbpMBUb|P6M^9#~{1%1X{(k z6I{)X$W$Sv^C5vvXq0AxG_%6%P`fbnjRjqS#oYibGst`Be(*wi7cD-HY#YTD;SQvmJsTxtra9D|rLP5Isloiv3dY6`6v zrlxRFiKegxhI>!R{AKxz@QE>evzkc_mkM+~qwqPUM&P^!Ynf}~ru!IM_b6lQKg(Ea zGtOoJU(uq|76Dv4l&$6D0_Fl(4VRw_s(}yJ%T!`5I!mV#>&iKvN@&q(FYryNf`4ey ziPqoqYSBq(qwuV>q%fh4b%8dL;3u@vLQ$;=hR{Y+R5_p~p^YUgp$g9Qi$!e_!g$35 z^6GKPRSk_FUNOjS^sU7bPnE-x28&8V*0O-QdLcvsggOBUE1}*3XT1*J?SZl7ld@tE zf{m&(A=qG5EDtbOG?f#AE#;SJ>$Y&}Qti{aa*kmKLa;Af*ntpiwBLpa!M0*O(Zl2{ z=K^w@3fGC-ROwVg2sT=+0YE2XD1>}-PdRd%S^|(k2sRe_76y;dQSe ze_8HXYJm3|>O~H}NO?$<{AC?qDmOI?61SFOaY?DQ(1hV&{^rj`vW}i8G1RGN_ zgPhDnS-|9G`fn>2?&nz={J7}g$h|$iy{Uc4j`n?fCp&EYQb86iCU+RKYl8=4f~)>&!6ktj1vxxs@Y<>cP($z)$|d%CYP z-QLxal-nm$QnbBO>XRqC`+6s(P8i!199bMwN$yG%90@^&wAiD?-onWwX|X3*H!BC8 zo~MgF3XY`UNL+a})PRM7TuaEcq{cJ~6dYMY8U=DKA=eUO20BxqZqRBYs0!gmmI?ejyyk%&H4!L@{f zBh4MHh~BSyJi>%v6M{_$_Cj|Z#ApmR2Xk4}14TX_HK*nv1e*|SLa+(JCQf-`id`a8 z?26))Cj^@iY(lUJ!3JO9vPwLeE-ygSC2c0+qT%&t7a)4SoT?ClZEeUPkc?^|*k~Ln z3XTlcDmb#YH=UBBd*$}Mdncv#u0GhLx=)U#+N0@IG`hD#N=_yzI1;G^eZ%i4IFf=R zXZV&+C60hTaiJg~YPxeIZkyo_j_j1WJ33{lqdmPhm4vaK?rh)J(b?VJlkV(DrF&A{ zav$spOTm#89O;?6>djA;+51Bl?XCvY zDL4{G5(P&__@-m~XKCk1*N)=zIyMvUHt}wUtvwcJz&j@F?4Ikw8S9BNk9Lma9y7eL zyefgIQq^KEAl_}_-M;p;a_8+T?Az_xIGRtpCh_&T+=Q5Tw`&+@Pp&2TOt_LU)hmcY z-k)L*sWC;$>(i>?>?K8820O|IZ5e!nn7+$minQSZ?HoxvM-~)E`*oy;YpGHsdU3BA zQ!mCpgf=deCf&at$+d)BOJrx(toBI=wxh#))l9QlYB3iOf=viEA=uzAOdE(LJ-)$4Ydl6gpDDS z9o>_??NX|@r@brME4AG1xK1_sj$9Ma3pQwK}px}b7U>diR4;Bt|jDJ zLars`T9WhPiJV@@yV>9vI7Vh=LsjO!eAic=37?7qACQRCHkZE04+KZkeT=Ppl(F@n zWvsOsPQbF5Vq_dYBZ`r^0lNt+&$~l0G87|2F*3D`ks;R-axEd(5^^me*Aj9q0ZlB( zO81E@J%c*i9v3Vt!Kx`dB`dO)NrrC}e+MN)3iWH6G*>$WH&9zQe{wDHE(K}lNbL9& z97(~E(!yQNmn|TM5NtxQ3Be`=n-FY5uqil_f+Hz7l7b@_IvdWM7fi7f9C?$ZPal%< z`xA`(Wnrfzanz^3ESWCJbV;VmTAD7&Uv?gUSrLcw$|G;GCWt2JaRK(}Y(|-}dE#|L zGUV{hY9=)-kwXg)eX?X-c{0Ooe7nTem*jRN{ zK(N6H*9vmtjgkNK>`(tLblL2&-49JYH8t>cIN50aM=J_`XGOu2cuE|BvR8_CqqLrY z?ERnlU;pyIF8TVppM2+qmmdFM)001a4jx!gIWTAo4>NGHC(YRN_^zw)S#qTySBg4xr6})RN4whTV6hgR*2|(~ zu%ii>Q6ggf9O2U?{K!O*Tqt79lUiJ*~31Q>~+H45CKz%6p60Q^mmGq$h| zNlBOak1

ak_*9GFoftoMIL4Z8?oS+v!9}P){WwzYT4GqUkCHW@b#Py_wS}1*sC9a#CfH2{jV|3 znj|e_%*ENNtgK8K)5lp`IwNOOj(fUpu-@bTVo)gjEZZ_2Q~1Wf@T&J*qVzh^Cc^jqJs9_vGYsc-1^o7SigDfbg)p`?_0|yDe=IF zF++o32&c}R8pdX?@@fg`;BY0^w}=8J)N;kc)S5v|`>()?yZk^wH`JWnL;sLE)B45i zw$nit9AbgV1Pe$8+vI9yQ8D{@rJ_=8hJ>&;p!?kgKR*I0JZjB&?N{dDtJ{h%S4ky( z!rQytoq=~QR;Syzi)Q$x*$ma`URCe?RcnNLV@<(oUxsnGWw`%PZ0vCVK&*I1CGdpP z{4a7e2hJK%+aN22-1xL6%S%>0EM*HWO(-;mIt^ZMCa-eZ(W>W7 z$*?dr#wWLzzj)8-$Orf1bdg|<`1%IhoZ!Wb;O)4-0(hW<-mpHS7*2@{cI^Tc5S)!8 zard+PVFLD_+ZNN%Fx~F0P{)o=C?-gJnNZ+${jav~y7~;?TCefo$dOTlU0nLMA>JhL z&J82OF`SK8C0Nt=O`|dEN5lBh5q=)tWTR+PS7jAG|Kpb)*l_HgAX@`UoFdN}@l>4c z;P-`G_k}`}ssVF1Ufu3<)ll>CRDx~cr5VvupnED)p`4m3K#V85qFlM{JK+Ud&CX@m zkL@b=*qCgD;E6R@UPPM>w!^$Nsb;fs61zbpZio#x;Kpq}C024$-e9{+pELL0hzJv@ z-=intQLNGyu9$FG5@zYfJT$YYxv;zD?AQ9-Z(qKq!=8 z%o%KFg*OH@2=878xm;4xj-netz5M+3qG%z4hbYJ2-gn7GS&^f;Y;_IUV}k>gWWJ6 zk5T&sgBxH(&hismi^_(nTml8W$hRGTl9_t+pa1ga2sEMZV1o`73S~e9D!l9aDa@ML zUk%{-po6y*#mhugh~kV4!M)R;+6cQbtS;B?Do0R zz^`29bA{!3`1iXpjXn5r6@FX|)#N{8ln7wHmhwn%C?g)+R8e(R5=)1Oe{k`#9iD3B z-2+Red*lRwE)u z8_6lBTs6FDDp%bV)C?!>;$%V--$?T(D>w3N%Dm&|bQt;4sANs)JWx z6|P@~Ww^jvOgnRf#8~q~T(q%jVbC;cI8!jJ9SDO4VM!RY0QOlu+l^?<(wK!=poYsG zvKxh$-(Xjzbx)A&M)lxhN_L|nW`D99Ew9~ZjiJpg8N!Gy2xj0A#yDg*FMiSXn2BF> zEGS`5nlGF=_55qMe&wy}PPpw>-$%JAY&$qD*ljLKc1^{S z*>#9F<76|HRVStFkUD9HCxI|3GufDLXQv^$MCiuSBrs)7o&st^+$9~~nfK6g?}?^k z&l88$+r$$n19v3YyPd>%LPs;kv~Nw~{ZJ`a%zM98eLJC*4M{V=XAt`I)ST7Ld9O0q zEfs6id{#G~jf{HIf=(Xtp7Ev*)^ZbyWj_=WfnD8ZoiR@|VyF>*Z?prI@C+UScDu&j zTS@E`cSAExwuxIZHaKQ(wA8vUf``NE!T4DJfkX^M+cth9cFnM-Yj!<3d`nyQ$+y7# z1A|#gCSkmA&kR8RgbkfPV|L`G)h~YcsjVM=u7L#)vgWKRr4C9-L)GGJEhs!P47zM; zHh=vMq8z8zz~46fKFfGhUEy~HD!@$zC?X48L4kHp#ojIwX8i53n_OzJwTI(}V~O~1 zY!LkheBTp&BeMTCasNy2*)8;xqBn!<-x|F0{?9)CSeI}oSO@t8tilRA{+?4lCw*V2 z{HsFwCM&gdQCAFmJ05n$gCLYAL6&>3c)R5Hjz75RhNnOMbMdPF4qoP*Hh+edGCI2d zXP-KA%o1I|TAl zn&OSUY9tctrr*?=-1iP~~xc%k3dAkjZIHp`$j@XY&gj)4Cl z{qV3Xv2Fj#*!F*b6L76s7Fk+ulXVCCSnZtI&PVG`u&yn#G-hec(wN17O^W0zVj6>P zK^ED%VEtL47Fn*Gt_quWx8+&3mxB#Os7;yGJkijeXEd~a%j%k=7-0K7mna5ge}SMF zz7hE6J*uIx$|(7g;Er_{Cfu>L1n#(qYc01d8glWXD1|BtxJ1v)gE zn`~IrG%q6L1k>RH0BE?21JJ(y2-WZ?1cB~rW`U`ES3H=ne8FZiB>+ZSQ)sGXgoru))SRB^pT6|a-QXJbSm7MH9G}g zhirscToSB^Du!5Gh{c6iT;`+bHIPj1~n~*OWfNXFJ3-vRi;~}FenOxQWXk=!VP*Ow(B{CL4hI*fe#c01#F)b z21Q{|JnoOepo*e_tmq^PgEB)}xn&(e01ATw!D$CKQ5Y0-2?#7*l3x&oLDe7(ik90M z*b)nH6(Xi2VoGHS0z8yMrq8_n1xtd4gt56Dqs+TaRi<7YTGGE3Z_2w|}?}E4Xk{xtwfr>X~P0o}hZcG^MG95>|)~^22 z4JT54z`siy6Z-MzR}13;uH|a)YgnCa18cUy+6R2WM&X3KfW0&AE*kAN`)cRUM(=(6 zvHPC+1@=`=M*x$N*w+Amt;S!V6YuxUw*>T=cQYmKn1LTHwdeka9Q;|3imOX((r)hz z>XKJaKH8d*{`TJgg3XnOSU`f%Ed>OboHAIMJydzp5xXTKMM(?eo{M0V{SH8}zXlTV z-!ir?43&-7erd|1I_-A>^_R5IYenaj_ zLYJp~bGdtxTZ4H6l$N_-JbwhJ(N~Enl>N$_LhDv)R!xT)UxH#B65b=u__R3FxOiM* z-H{^czdG^8;)T3FJq>X7Gb;A}+E1cpF7>#^AsDV+V7OKp&^K92$Z6O}Yl+eE{*zyE z1oRj(zQlO0zLG7#wj6!ioF{6e1WuUKk3U*iy!_|&k}hNq!iII=f|8dt6C3I1Fuz3i ze(DwNKOR0g^G8DT{CC8x)_h7UH`rp(fB)bGM3Ohxe$ufAE_{K5ls{;!1yR=&>w2X8 zbuCPF_AW$4w=*3H;YYCUP{id?{#czZJV^7a8_iGUd(RE4RiL7*WEr&~qcGR}MtH7{ z=2wmlyx~n#N$d6%1I1cpak8iP=Y4KN@g{`5gNXdo^0V$zTL%Z0cu1!eE z`Gb_un58jGV^&fKDIuhU`8y)|`}>a1zhzW4tiCWdSF9@vAtm*Kl(Y$N-_gv}G`ahl zj+6^`_wBUsm*}2ffBPGIzy8S{LKFPaglb95l?FyMRTYdNcV9WYoYOm5qe^K$xz%Sm z8=>Av1IXPsHg>pwAa-IbHX83w#3y2-To=C4*Jx3qqaT(uDJL7UmX&qg+mD33<)dTY z|07~m@c$Ll&eBtov{q1zOim8#7~&MZS@El!_PIa;6c`$Ri)HHDQT!$oUJvcHCdllv#Htq^*4yz zPhrPz!|$_q9arpjkiBzpbnD>oUeX&})*Ha5$knb5U-Ba|cH+n_<8 zv;Gag$Mpu%Nlgt;$qEocCc`(1zk`w?h59v3nyX!%25MQ4Eo_6$mbhdO7yqDV-jxl& zLQ(4KVQZ2dFXR{E;?hthf8VGDv2?tZKJnEz9{JOHLRbi4Sy23}cWp2RWTIKwP?fna z-}RMeO!yV$4IwPyc}W$Ry}Xx4g1bEx0kIJd7#u%_{wMq}2G zhVi2#{5-+~qD@_uRrvgmUwUA}v3r7S4NxK~@~jb0#n}#iU&wV|C^V@WhML2x+kLJY zYCfJyur0ha9E||oBa~B91yDfP73Iop-w7|!YIZKeer#8{$HrtM1W&BN@*;+5upQ>D zjuU(&4tS{qNC(`w&8NglPRbi>cjtq)`XyE9yQ?Gbr76&tp14vU*Z;!pI_TR;B2 zgM;^O0bptf-U!_2b?9@K&-?t=TmG}-$G`i*v$)E^h$enuWnNbDs9DOe32vG91^{@S z&7WEi-X*_>Q`jJl44xNg7)PUEP|H)Yw&|CjKlA51?%D@Mj05`|N;4>;Ye79+$W6-H zCm*?gSIghOdbd>~K(@q~5*x`w`T>LCz_Ncg7-EdtCm7rSBXX9X*jiLJY{yHWfEW3; z<4-bEkN)#t-W-7@^c`%_p+cby2nB_A{Y@^4V9Qk#&_my(3-V;gB830=x5SHu&;Tg8{3aIfjGwC~kZ*WdTVwGUlG zXqrEoh5`-hhInF}HIV-tw1&tCbZp*z!0Pm|uz}s4T zS{W$sHNFza6wgJ2+>y#VC)f_DV5lgy-Tuol1;>ACh+TwNk3!&umXov@G)HM*ewtvL zQyI>nWqN7i^ZF3mYzv=J8RYe#oR#J#*u`lnlO31znGt1RT29X7)r?}qG)>he*xIzJ zCFOo_{ItHK)PPcD^aSLpBtb-G}%6+{@<$^B9Sxeq|Y=Z5~Yf#9;nmPr}=K5d(eofga z46)q?yv~56B;_m=1%$0tf47VRAwD7qh{`c5;#!7G#(6y>nh z_fc*N+YU|(cAJZmT~o1Sb{*o)IN3~P)k!Hkq)yr%HXw}3Og84**=dL_5xTK72~1hz z{`lxSjql8RXt`((G#z`MIIP|#ocy&Th+G{TG@~^ zi0B5ZzdSW(b#vaU40cP!+BBcl&1WN{p0uEohrDOJse`rL4EbR{G)2PIZPpp{L?cEE z;rB*6bVvjc{r7(d7`W=cx02W??uKTXY#|=_s|Vv_{Ra{;5N&l4QF4V}Lh|fU~3P@562Sm;n*Ph4fwt%`bK2`ZQ}ly-m_ciDMhg|aQ$0@ci#Wm z#~2Bt_yW+i8MvVqc<@iK1cWY?GuOa++m#1%d!lyPnu{7Y> zC)N&7r?risYEH1-y17F0E>_|SNK}}b4K`R&dfKDYoJXvR^QHz9L#vgKtnn)4)dN9; z&S`I;lfjPqXIy5rgfq#gD|)PK2$P7Q@~iF_e$ah&->Juc)PQbxO-Tt<34>kaa<;6F$|JQ^ejBYp@XDF2nQ z?f(EL;9Aw9f@rx-)*b9pVoI+vyBCgnvdCH_3N&V?kF&OPM$V?Jxg3bT4c2?ye>yhT zlc@3V$jJz9@Ff;&hOI(otA$6wpi`i`$A~nPUlL;1(4CWUgfN>Tu4p)VV1V+n9 z++l)Opv)KN5RtQhMLyic7x^`t(8@ts5SesNNMHh->IKDjF+EHJzrt-?5R-Q)+5(4h@W(PNZ&ILNet+y(Z1BX`c>j3p#Q3qpG1hc*f8t1tFCW_Y z*W&FPC=7~w>PBFL0+_RNV91LU`2qqYaiNDn^_l!Kz5f{gZr3}X`S*u~gN{FGRdI-% zZMk!>KKZpL^jBt{{hCN-QU0_=MAZktEGS?z^crde#xqIF7;`}Mmz9+%W4V1i5L2cw zsA;(pJu2rgtIRJ$VNetXr79E#<(+OQ3<{(x1WZvF6zn~Ngq7yR$77^`=#|2t#C}T@ z1_k?_`OU!;2DKx>C=7}_gLwm&C=80ipbo}kgT^Avg4~Ue!k}sp21Uzl3WEaOj>4cU zu6K0hu5V(>D2-VfvovOjn39Mot4^g%VNetX#fnKyD$_G9c#%(GP`-O%ldCzonxCiS zF63%nEmw02gA!4*R=lh~;TFswG&ivTP-IHUB(OyB{SI*>Mjj%ITp89@c1^2a{O(g* zKm1$+3m#<6Syf6Ml#;M}VVtc68ev(VRA|mu@(b>U)bENB#NptA%j^;qp5L&15lK0Pm8Q zg=l}@9Xr4D&8=@efX&2>F2K)H9d`Swqkic$QBP)7maqCeYNvvqxU}k z*nQ9Z0xJP?;bdOCV)55%v_(O+?Dx&rBph}|9`|+?b}YyL8Ir(*8~8BIX8dTWJ@-H4 z;O}f*TwO(+OD=i!2L4-FG0T~LFkqOf=o`8-G+oG9kE*?Qj|2XzG6+lf2eML z{0@HpHIRt^ma%nVID^-IY09HI?TghKz3lKJI+fRjI^_Ln$#kf@;L=7gRDZ*yS^c#g zYvp#qUdaIk)#_O_m=~%g0BbfgR7Y5F`CNO^%m->MbU7Nl=PVkC25*rbtuml*vX+q3u#wghqvQRzC@5aAj3MJojOXfL04~WEU|Wv9ZLV}{%&b&h3(-5Hrry~%*?#R8jy(72Zc;n_ z)lS6E(2};|wuwTUZ1oWT^|c2pP>(SG!=DC8?5l|9s=Njk5%Z&5xYi; zH-N4aq$Jv7A|>75+P`PwpPqW{+k}*qM@l#?{z~(WY^W+yvf)BNN@&z@-WmuirCsbn zkP}iOqJNnG{Jpf`vTC85G-hec5>g@&)hSitDRV&mLg^-eZK}y<$x9@0X zYMR`Al^F;L&x9B5?%Qd(`$qTt`rF^w`}I%u5SrkRCR9set~4-$V!bHVi*HCFYp#fh z;8N)o(6ffDQ5Uo|s!rn!N%gB2)emjbYe5dcClAC31UnHgfCGPu^R834Z=-o@kh8qt z5BD?8)(ycv^wNW4jdg~hA;EipE6>GD5N~d9<<(?87+@Ss%fd1~JX%nUOiqr?CS~*( z{(ZnSqFqw<7Ukg+t8Q%W`uMMfR<_x>(Mm-5>D!mS?~?sHzh834__U^;LUY~=Kj^-? z@6_WzYG92C*0iX7a16F7fq6!-{8mKFan`IGQgQ~YZ^`Pi^r3I->O9aJ?LN-44fARP z1E`r4SdXCy%nsKm7Jap z26YLMi4cJ|q)r|^paE4TaEFQY%c0DaCPDX5;^Vlps!--(^S88!)#GJ^o*Ed$!mY!P z_4u*Dy1+WX#92}>-Oywy2k*%j2r5MUNx;mhLnI)cH{$@IxjnzbSf>Wtkk?dX{xqxK z&gfZ#mD*{Yj4V#FjB4f*SMmkput-bQfZ$tX<~?oZxz5a!U$k?~7(RMKX^w`Z&*#_tCO)cV!>wb-0#c$(0tpjQ{ zD<`pU>7gS^PEBRf8I%x-jBfm;^BSfOCd3VFth)~{*O9F}ADj(^$pxo4z@8dxCC8g=mRro%RUJNh|5To%*v7|ud}ZNJ(6N%@-~nFF!runOn=6{J1S4X z@Dx+i&;!?=R_?rAg$dQej%`n(4GFeeHyKR5E=9|-6Cq^3-O66p2IX`{$)MOnuXlqT zO~AXuhVQtEX2jY39mkf-{l^sYk>Yo^8Jmj}2AKw~5-zDM>e?KL_-8hs&)=@GBTK3J zve{etY=V4W#iLe@RT}VMznH%aFA!;5?`$JM+I-sXE;8>=ko`S_(2*xQLG6R`_N$!yz*r z6m3^}rm(0@{9Bc*v;iLkZDB9BK~@SmL}BD5D?<~$O;;QP5rh10ma+v`4Tr`!9~oRI zZbi!i=d|Xk=grpKcec5`{Kb1tM?Sd!G!TX-SR=l^!8Rv&F(X(wL>gh~1#rUpj5iSD zTK{!eNgxJ_%t+k*?EY#6W9+T))((OzD9!O@LV?%yzuLa*>N9-h+jwx~$S6W6C2t$z zO#<)SFfttDKpH^r<2Q}QtRD^IM@KAxZKG&YS7jAG|Kpb)*l_HgAX^hh0`^%Wo{9td zhwU12-4_Z?s)nKF@alG-tA?77rxI)nFAd+g2@v2A0Dpxntb?y8S8n@Gc!5^4a~bwy zyUIN_CL1AmVhxrTv2ug$FmFxTz+)s1LI$B{xN)0LiItp`H`wmd=W+!DfrN-aru-f~ z36Elxws63Z!-XJg8WCs|H5Yc*oc&s#`|ZouwA>xKpZDKYA+-Pn#AE>FYLat#W6oeZ zE4(qR`tj~{kjo_{?I^kd)XUFbPafO!VE8$|OcW)a0!8}D(s$0}thYXp0qo9j<+VrT zWaP{cUU zh@v!uBDxmT!`xi+$w%(r)$;eR-ffi#kS#H$I4Zdv(hnF62bTT2!C-uh+9w#?03&ji zpV(SdHf+aBpnw zo0sqUfKUI}$xGuM^}Lf}Z{{7y%GC{ZoA!t0UVP)0mzqXA&)1meGF*$!oD?*6 ziEAIahR`&BGz|qBA}mDs1&$lSzuA9|@e(mWjp4y38LYw!gYb(+4H14>y}If_K{!m# zM|bTyC;Xx@OJkPCtm8k*;Tf_UnHz8t;a53kcCQ)2m!h&taDTAw_GLs+ei?XID3+mK zP>eN(Hn(I5BUZth;kYDl2xA3j%@@v`dj7Rrzw@Uc182Ny zv6X{4LY|eA;4!Q!qR<9wuV_Vzd2a`+!)C(@jBqNzVEz87l<^5f?BlE6q)?i_=mji(wig%D}XooXM*h#fWK|s!g!9X;n+g{owd%eMcWk zC{;#JK(0!14x^}MF#bqY(wV6OxNQ!xZ4m0?zGjsx&Nk$vS@hL_I1UKW$blgWj6e{{ zG?KRK0E?E9^$FIZa$oOJxuDB&){=J~n_xTh5c7nwI8z!pd<+&a0?JNdi0#HelmSUe z%2_B12wSTJhca2ncZlsOzF`~!+rTO%u1~Oykmo>F)!}^xva$p&uM=!5oUBMLBHseUzKRwu94x z-R7cX*HkQ-U59uxPBv3nbyC78FWbWggi)Ev#(XI+tuy;F&@q}6G`gTGq8p`gXlNl`=00YGXCHs8OSluPgM0#3VTB!k&ncgizAwakrT-O;O;&2_qOKVBc0BBg2SF%L zf-Lu5@pj4Y9e;4s4Nrgi=i*iU9lXrCoj=1$86DmKvrnD5@)NIPMp*aD6HR7HhcaPf zAUFKO#^6G0kw7u!QmE)O$|XnTHe=Y#Tz6Nz*OEt#22ADnL^F45XvD7}{CJnAZ+OC| zTw&MX*(ZXkp-yWXLDhV2Bd8$za}`15SKUj9VaLYZQoES-Ws2y+w*ZOSa%6byaBLuc zFdiH91`<~eF5;cL2Q>9X#h%S9BFM-R{}AH_+0-!L$jS&M+wE8_*Ovrty$|OI_z%(# zj|NF>z0~%vjBWo1I04tHMp8=4ZL;oQml9Jt5eswcPJLTs2_hq6VVoCe%)+W1fFD>W zi%HR%Fd0SiQ6yiklw1z6A!qqT7FpXNQ3MP)))^5ng!D~(jrxbwnbt36w_%9X5DQEu zSO8d+`~n2bakwgM+TE6C*70uDZ5l!daOy3Pp+On^gn9N3&?e4)3> z#K#)=6>j5#m^=2agjXR9}tU+RI(vsO+t~0#bpKY=Cy)9{Xy`f2Z!+`t6UTY z<&c{D`;?m=HDL{4m$p%RmJMKZMk!>KKZpL^jBt{{hCN- zQHHtkLO$!Vn;rnOpf$Pj9ch|i0~%uvi2ky&GG$C3XKm?>oJ~1-L&!ky@rs`Ewx%5M zYRb^i^KJN!P#A?lP0QgD_jbEd#SVUz`DG{!io&2^g9!?Q0>6=n9l}ImP#|3)zJbD^ zz}A3-mFC395q2PYr7$S5-x7sE@!fbz=U@thg78cVgW`Tz=U^i#3WLh?IBhv)Z#W8z zkv7ksaXRu+Qy3I(hU6S8Eewj5+Y|-`y`MrV+#wZp9tK5Ymc}fNSt6z^G3TM}H%P>k zL`;eM&wF9-Wg?~oO#*h_rZ6anq7gS+6}!ljt2t}Kg;r6p6NN#^xT#jF?(H+Cr~3s$ z=0agmi)MXUQLg3`1|^)cco-;56#j%;aG{uRt6U7QpblIx08Mif%}pW8am9<*)7%8S zrn`^3;{L$VEjfeyuW}4_Wt0^p>D;0h>I-qF5Z!8p=u#Mzt(oef!9p%KhrGx>G_$n1 zV%==6cyZF=FhT0gVc_2dzMv&L=*E4SRyjX~<8x>2w_W5tDF}ou9%*MI>&C3fnX**= z{4UdRv}^6^FWqn=)%O-Tj{5WGS8MYXpj>{ZpqU=73*cQU@^J0DW9OH?x%I6Fu$ee; z3Dm6BVYjbIR0D3X_5t7iUrZvzhne5rnNh2YM!U_v+WE85dmn%7zGr@cm4GXLWP=g= z8sM+h_zUvg@0+hlIP8o(zGJVjV_B-4tRdn82RHCxn$7spQhV-y$id&)y12TEIG0@V z>d8l2Gt%GQ`(J{7NlM_3>bfDP4A$fCRLzr)*mO>jqNGdt(tZaO^VdKk{#(Y@g|U|4 z|1V8>RHuEhTBEn_woVe&-!N%be{IKFxm~bVazH_~dR7hQg=z`Fn#~N=5f)rN*IqQk zNUM>D0}bAD6%9m#w@452YMZ{CmpI(c$9;3Tk_dws4`O^>z{1lwopHY{`)xpfA9@jX;MpQ3sM6EKQZ?cw<)3A{i?4!j+HZC*U z-XA*_8$2;K-aj5YF@EfDj5XcdpEwfZvZsxIJuwy=jrS+w6R}Z#)_;qF;swiCQp!p( zo~wfaxCT~$Z8`cjq$lQ(S^#@}X@26hT{^Q}3Gk+vR z@4Q3wXyumAgp2!p5iYe@nsHE1#6ij*DSt%wuYzUEwoopQ@>j1u)};B>jpnE3asS<6 zwF*>}l`NwU>cKU?5uU3v9LiZ;8{RaPv~FM1l9;fZw2PBHy+?m1`P^?#1{pYxRcn)Y zAZY9IEltT%(u?hwlcCKjS*f}fqIX73y|Zt!{n{@adG6EQq;~qNors^IDzf4>Vn>_o z+2a%L{)$z&epwKIT=3z4I2t6euOj}g)_#f|G-@a)5&9Md5}!YT#587UKSi7mXw16D zEKZUHuwWgGFH9RJ>zXo^l?_#y`|@31sj>%funVYN);iJ7D`sx*@-@1J0WwGwR)>jG^BP70Mo=!h~ zy63@9T*6_Jjj{+9a=@9geyk?>vt1*_q@?>>`}a)z(^Ic~n~;+7NC~IK zUunLP4OK--He3iu35^Qz%8{PBkZ+~O&*FV`qXo5eQ zPz^sBG;*voFoI&e#D-YKflsK}!ciMDV6{3c=~$#=oiiP)PA#FN`qhi-hvxFNAVb9` z3j~6DxlF~6wdhR%XQJjA{3-7A+~m5>w{_&{VS;#nP;kM(Id97DRGZCI86*o21v^cD zzcto*o|M>~w|qKHY?XaKe8KNUuY%{?`8MG_SlR&NR8U#K3BscV#ehBatlejRA25Nq zOUmA&JbYr+jm=#j|Fuy2Haj<3i6}pP`_lJavVZ6IOYRt-*3?sIFMQz#-BD`8EKN%fJJ48Q#U0KNNb;d@T#7d?&Ld^HfJ-6 zJY2|4LJT6?kdl*G*s@fa8bDHNEC5u}nsg>B4@)`O{N0?BjcGMypTSH6AA^ldE0{qm zA*cq6SUUfZI(hVfrb;OwCld>#LzyW}Ld$Y|9Jg*2fl{!$TskWmu$4J4BXsM)AQo;N zeyqoj4b}zL0XEcDXl~E%FxIKTHsm!m2_3>@ zcC<5k)?lTEU?(Gs^em&Axx|%x!8k0^QZ-;{6`6TYn|ZD?^W+!p95aSb5PE5jhNREv z;SojVPm5f};he#?m7X*+Y=Gig`2^l;u+9oqXuX-p$DMww^TSMcRYp0f&Iq=1Xtv;! zUeaT+QL8(_oTR|W)54;_Yw-_H3LANcRl?vKuyd8u<6#8L5^zAZh%c`DHFg!htzcxb z;Q`xgK@S~Ka%w7*&Y*-yWOU;%t?!C{{f&FtxLv+63uiFHRbW(76w0=#K=CB;Icl^t)&pjhO)W8C9)~JI6F(2aW z6ixtw5bT;+l-eX&7eTO!ZAp@qBr8aBlB^_ID@azMS(n#^@j7b&n2aCz!#Yrl#{Xyn zgX4KMz(QS;o#1RtbJ*X|7>gsxO_?EWplfYyfm|!hZ z5*5=CEhTH9scdvKm764e8Y;JG^^4zqYU_ueYhb~HtT_vM_CYBLif^2)H9%+5r`2p~ zHh=vMybakYTo$(B_oDq+%3oP$KNIw5WFwQefqy9IIM*$vKV#n=m8U=r5=(%g2d+J> z+t$Y4iYw~FQ0d^3zA&Rt~hVw}pFoZ_RAzr(TWBwaLxAu_>Q z*an-woP!5ue?KeUB!3#BGLzl{Nz1DhH5;{zo*i$cPki-_NB*>4KwJBK>$ceE93d0l z9!{IduRWo^GV|=$L^6w-?~7jj03>czJI9im06H?}h~Kh=w45l8eC+&0afIljl$@5s zj@(@gkp+xBiok=8RZ?d>+|bJa;>+$@HN*2^Ut#QDE-#_w`OI%go4Hgulr!LCsj$}e zuzWG(@Xcz5+g)^!8?rN}1Hq0`$RXM-FIgFy@a@aRibV|ayIIN>T!jdYabiih;1sMJ z-mvO<6K`0U8sn4O%U`_bbmW8kPlFXR!5Z=P4YoPKiy6V&(V_!ZBedaV6c+*kMw*HX zKpRzwpcIM3-Oui?R#e^I3U%yQ%ESbTFB1y9uK(5cU00vsE8oV0BS%INLMeIM5N{HA z=Z2Bt7&lmeiF5p>(U|q4Vf^TbWyshl+SFB9h0p)^r3W?~yC=xjfD)(3vqn4>2P+G< zYsht9C^V@WhML2x+kLJYYCfKV4bOOKI2r+#wh$Ot3J?U%t|(V-`%ZX)RDY251@?g6yeJ)op5J-p!WXkW+lkg~3 zX$v>ZaI*|d%#CQ!5j7Wf*PQ)YpZo30*R0o2RSUr!#}^kDcozf2S*o&q55%F=hv%CHG;nfC?&be+wgS`W(6@8J|SNF#&i1scW|)G(-Juqkf(<>$}*`Hs8xK@sDu z=_E=sD57gYJfKg}0ND~_ildU-A^m{CaA4WL8w|$BsC|OL4KN~S z`H8JXWy5y71PXYOZ#(`ZGxg{{|K-gQXhPq?1|2FC%76$|c-QxFP=I$5{kF6tPG&w9 zdGqpJAMoiPJ9%lmqn>wC?9IHRnmv$rL#Tux?~ptBD_+S@T2})6%5I-44gAVwK37RW}+}i(qeDvbt3%?9kRl!eclE7gH>jQ?7`)k++K9{%Re2%@Bfpub#U8 zz9+7I=o&)P{38fZph4ZNybu6lZZzHBz$wlgEHr8;wi!f04OOS}G47_m`3Pvt(wL<& zOMCbiU2gH0u0vgp9;DjMeM4Jm^Q%34ERGvyxve%Gf7p#p&XM`Szskc4I^tn3j_>c{QULF-=po3AQ$^YDu{t96zn^C^eu|89f2HD#)%ZOBQphb2t{mR|^k%Yh*Zj6e`EI+C{Q0E?E9 zb#Ovgxv%%A41u+A){=J~n_xTh5L=2dk5k}mt`8P4($r32i0wAubq3H)I}1g@m`}52 z4rQ{C?-1Kne8V^d5Q9}pT%TYYAAyXWMv6lUMJXAIA`r{IiTk9m@_1uU~Ne? zCHE^*c23Xehv2)x!v5k|v^T`-dG80QzS9g<@CxMxMLBHseUzKRwu94x-R7cX*HkQ- zU59uxPBv3nbyCVgG%dCon8d*^x$I%LPud!Qs0`y`=L^6G`gTGq8Ne|)d7=@ch46c0Km-msp8<9| z#F$nRJH_45Op|Q_yZ3w^_^SuwWBmsbF%WIroam5q7g2JBU#tmKfZKn)#5vOLso2}y zZ;##NQiH8M96uaO#D`;p=r`c|p6DBq{kMtxUwRLO=3=!K#md0-Zw=mg|7Rb6tV_5P ztb=?4R$+x5f6pnOlfEy+d!_#sjZIc+>!Pk0_I5n%iU&a`Pl7D>Uh#Ix?;U?|(+y95 z`sdk=E_gJju~OyFUq@o2INUm^qSeY^F))G(xFV)7|0F( zuravMS|m_RxfCiojdID++D++h=DNG$y_P&`G+-*nCz`oiLnD3-;m5l?eZvzz<)XI$ z&pxqsfI6*h1Xc67ji7?;x9j!-aZpZYlng2YI6WBbsK3}Qvs%KLWYiTsX2){Lnm1vP z*q}`MRreCOLa=eS)Q%e@p9d+qsL-7SSBH;8Z8FRMH`E>OoQwT~Pa>D0K#xZ(Ia+Z?edK|Bp0gS7l{o%9uXR+R_<0 zn{xUMEGK)9`%lN_deZK8#aq)PShG>cXGL71!8W=mwYX|k9Y_<}Zp9&N=i``lO_=Nr z5NLNnyL|*)U6KC+ul>rj1=MW_Wz~Yn>XboJG{Y~=W~dH;tFbE2SXFC;6&lIM8+ldO zY~s=Y0lo9he#~5QLrPrbR>XD{5{E`0Ac~((==i z%H6#J$kfOmw_f~l*U8{m1*Ggt_Q~}(@(YFm8=2*V045`amJyKx#NrZm>c?vDoRmJM~N@8n9EQ-RQDs%t(MCiJwKdeFugQ74faF3=i zsGO=qJb@uN*-#jiCv?dh6+&T96b40MQ2zY;6b1$1nG^=aG4XS-krahN<$0X8oU%7| z28Z)D&z^BQ@=`C8!w_oP5Lmh-zaR>OszDeOEw?EQ3YNsRK__w4pN}CG^-W9}r7=rm zmd32hvWJ9>6b2=>`ygUUBBu0#y)iHe{OY@WLpHYEG`^fd4FTFKlu( zCs%WDn1Id0j=KWc!I>aAgB*Ku40gVf6(mC2q8I85)u1q_#c$V5VNm97YtW%7MAz$% zR+ep!!k}!@NOv64ZYwIBBKN!jU=Ozk+vV$O9OmfJE zSG3Hb%@ylrbH$63u=g^bDv)|}7@SzZ1%Am6x^Z9hpv;gTM~W&}58*u77LW8t$1HNN z#;nPivQ+;3F4J+eYwhYU-Ebn+_ZB&h`t#?PLxD^JTSs_T2;Ks~wHD-HbxMw8*|9$0 z+y9HvGd|4x_Rfr2T{PNl_SMdxjo$nCWA{Dt3+$_y3rDem%le%H{#uRI3~_SRqVB>p zoAINi_D~??;Ln0oTwO(Aqf1^r`DklK`rCW|3pQ6CVgV_EJF4r3oHAICKXD#UI%4CP zk)ou*-J!@(@H?;@{56n>|CX_JVe=Xhb*YLgg3|{}4te)HvSd1T)}`%F0lmLr(wq)z zJJ!nWf-O3yBLY`aKtUbP3)K>URj18EVM-njmM%wwcP`F|@pbE451>B7w@Iv`_CkXf zf|!dn7-D=4OL>16bY~XlI*bSdcNV0f;u9V{W{EQn>hYvVo*PASd0ZXLTQH~jE(n?$4-nNI~-$8H}@xw#Nc9(wehbf z#$u!K{zQBtHp;ZZ@SH3L&~4Y9mWMB(fx-@AbYOq zdU1Q-4Qi@|^{^F6_g|-dk2Jr!(frgrZo4Eiy)hPthxs4=G)Q7! zMLbun{S-TB)KE|&b}=C={lSV7LB3FRmCS6a;hMaUnUgoobdIJ2Qxxr|n9V3NU=PWZ zSr`^uJYc-yL1WgU2qu7q0G0(XN_y8H1hCj*rx@HXa=`mT>P+hwv)fJ!hrY=KIQ9AN zK?7E9LNe+G$w;c%teiwNW63tGN(zgEHbB_tNeC?T1~rvKG8zw6$=qaZ2p{eJ&{wZG z{nq6}FAG)n+65sbzGa?HKYhCA!B1SmVUmrq2xN6sma=}VLQjs^HB!6*be$k2(H;{i z>HgOKJrn=*)N9`+q@+Aj!fEljYqw@o_mN|lfj8nZNJ2`O=JrbI{yAti*A*ekTKPnaC8OP4`mC8VTYkdijx z?K_&8nkIK&(-2XN2$pyE?X=u|qkDe+?QiV;`X_q`P4Gt(s^KRC+Dt71`0Ltj<9rO% zM7IqX2mGK|uLU4x6zio-Wo1KE=DvK_SE|fM6C`(EIn4KHT!fLmhH-|Z`qhi-2aa@X zEePoVTZ?-#1_HqjA+z}wjpDrP6u(E@8ssc5_?-KhW-F~=5BJ<-Y>jmWauVq|_cyau z_Eq=-Glna#ChJmwai%8R6FEFuPz>1e&e|O2_W{$0c1hV=l!s5Oy0N+IG^f+6cFK8AkZJ1XZXs<=Aw84Y#x$e5R zujs>fMoeT47(8}3byMueDFxhN zV*PR`Go?vr_>GSv5lWaV^9Ea;&PoPQa_U$n=&6B0{IU)|*5k(p>jLWl6K6@mbVHM+ z9K0tNqH11-JXB4GNI*P)P^1mb?fD(XIyKmayrw3hLztw%c1F({tkh2HWMt70W>hnm zxRNgzhecYdX0VGzX5Q0gp6kp!`9(X&jNwDeRGOn9=`(tGM3MQ^BA0PEXRvLhC(R5S zptx2(f%h7$vqBYGZzl3_r{C)QFw|wfKi8 zg^j$!PUa2RSjy?~FoIadW;ty*&@W;mA zedGVuZv4eB{`yb<^VpBBfb+hB|Lnhw!+6Hm$Zu%Xl)M_3Cv{1e@d`c$$Eq*;SRfvc z_4W?PeI1GZWLHP;Kvz%4K-WOBqc5508R(4l#P{^}z^zZi{Y~&-K~bUsSBxJFPYg|s zPaK<^8k!0p8;ML#O+@w`nu?5%9Xl{S5@zj_2cwZG`$8Cs^Rf0@hoXnV$0oxQk)dd0 zIy?bq{VbxWGeIdQ#b@PUGM&oK!H-n%4(YfQ)E9DcUQea9pqkI>sf-+)%IDJZq^zc- zbn4@>I;>=qshP>Fl+$Jv{o|A0{{ydxH5_^TE!L!|>=w_$wYZd)f|=BeD(NXD8yudM z<8wJBmDT0MB=5UGK*}oFKv%cK))Vc_Ey%g7<6cG zQdd*i8T-wh?NiylC6wfT&q*jXE$d2l;qUJG`{#lu!q~u%u{$}D4lbeCs8XtVSX#^R zQsYoW@_9{HG8TpYK5?q;E9tE#{mehge6c9=Njlr;7-rEid-)(C7l8Kj<4qvHd*J8C z7;9*TGkERajmcNH9bc-G7NIWj#tvd-Q%e}RLedHpo-o-MRO4SR(c&=t(in#7kXO~I zziNYUa0!Fpo<0HhPuBpPeXn7+uf#muaEq+USH$Ei5~?(#WVI^|`uxAd-~aF7zutYr z6J6i`f#mH~c>!JD_?&s2IdwPx-;mHpEm#O2j2i@6b>O(b0{&1r$8s zXc#HJ-sQahVbD_a$gw`TGtt-4C&dRldgY#2M=aLg)3FC+k9cRl+}*dQ8;bg$P+k-K z_o|Yg##$HGR;<}KF+MahJTy7QAM#dH)ag`R|H1H)LXAqDHnoT>x~e@!^){24*XYz; zBVjK4JKRY zgB{6aLh9)4>6bbN`UeL(xN~p=^)CDE@LHMwh zI05(1QcA<7`-~0qlGJh@^_e~3XRD%YTS87s`Lu2~hilAa(8gUV^ed?03`IflDJO1` zwAlkv4&>#HMUQDH?)jo@1J*bp&!jY6s4vY6Z7M$ro5BPxpCv1ply#^eP;mF*ls;S3 zMx)AIW$!iXZ`7gInkffCk++o(Id)gj)m^=($8Kv+pMKlVeABE|ipOP58&?yu3L^Ig z`x&Ein^wK_-DkFZ^h^Wu?`JJ(SgH0)aa~a(Y>hsv%AiP2C+2fE-Xuox#A^84j^F1Q zf6?YeudKD6@$0Oe*N>Xg;CSo~Ij+Nd+^`gSM%s0b_UJ>`pUmEUrvf_PD02)srz#w; zhA7*m85(ja=G2x)valRy<1j`eW!1ppc{vWftYpPZ37BNzE)8g?&8!DB@u=rEs^4ax z9+8u&YzoUllGNE@?~1XL9NK3?+aD@?cdI3`>Mj`clAv)Z>T5BC{Bu5+%iXE6L(A#< zlGU5_Y~;JV_4bg5v-ynkVoqnRhsGukhKD2jBjFL9?RK=J?5~78uEAG7Cdclb|VN7JTT;R>EKCDyZg&qh*VlWlC3r zLul7uT-BXOGaGf*G#nk8oSbJcOn7Dkv;z!Ac2A1Q184mnXT9!YOX%C$^{KCx{pe>` zA498{dW?(Np`5B9>jP86u{=z{4t`C8KC_o$Z`tP2U*O46|SFuc9QMPVQUa(uIvwm-O zBlesQhJ(e}&jDxKl$A2F21ZH8X{1YEbqrl-wuObDjs=aTQi29taI8Aman%wupyUb! z4Y;ecpn*f=uNO&SjUg%cxIDl`U_*v5uQc#0xT8KNk^)Gl=TcHY;TYJIVaj(e=p-pD zQ&MQVFeQaciX?^2Fx~r$)-TgvgfEQ2Ta{E|teBzm6@{-UHB!!-YAs`L-1K?I);_`5 zx-T%+)`F`UsIO?#X|Vt<9m?i*vVye$cEhFDf@t!i1H=V^xiFxH5FD10;v^Mw# zSHVBD=|t=Ad9~>zZKLq4G`TQo8|y;bNQ|GfjiwgWTwqAsXs{|+gVxopLKKQnJ0Rg==r`Y4@56Vw z)Y$TIIjfTf8%1Z*V1re$w1T;;shl*}Qf`^HZc|TPEPa|+&av8oG}sre+JQ9KXul1T z2HTAFL=Tg*Tnor;Dp)6OQ^iXOX|U014GMHJhC;|U_mm^IsbwfKNP|rpY_L3$=~8${ zlfNuS%5b~akiRVVEY-n#4do&mU!=53l>B9FU*z-DUzR)ETnw66(qJ1XLPaWjx0c?p zq*>@QXRpCFZE}Ld_%EM<&Gg@9E!@wuIQVhN!I6n%axggxD|W#BLzoNa3n5~4K+YvAlDLdEvYe#0tH9b5JrJq zOUSi^5P7^{*NfCaWfZ3Mw&^6-5{i+b7#Sl`LTN*$*JjC3Cc&TiTp2_jaxEd(5^^oE zfcMC?gj`E1PhC@*Ba>?hxt3tGm$3?SlTOva;wd;1G8{Sb5UnuG_}RObP;jJ?(Td>x zs)r*?8f?;FlLouetb@2^c49D}dmv!aV3P)$G}xrUCQNxkid`mB>_vqsPa15}V3P)$ zG}w@kq@)l}!I2akDR2m^q$3wWG7$=nR5ET2F~R$xL82`TP@ zl7pR6M|V$R05V5*4?=3{zGO#Ya3DF*+tn-eNiqdTQgGyCcp@?sjZB9pfX4l-?&1z9 zb0iLnJ2*1l8;|!u3}i=lf3h3qHbh7cN{Kxk-LcNjm^2{u_jY$va3lpsx-IE-hlE4H zk>saJeyX+fQ>EZY3XZh0O5r934%o*~=E%ZMq2S06Pdc`Do-#)|GKw$i*i5+Fgu5Ly zb1a@8j?I)gl6%bX!9sr~JKeEZA8QgQPq^EJyM6u1?A>=NAbq(KNAqRZ0KPt#Nr(w| zyM}T0Hmm4%E-6YGY_DCEGWaI3e3t}Vqo{b&Hj=iH zw2kG{F#FgM&zy(l7B8C%FQRQE*Aj9q0i=DKUW^@Q6A**r%@liFMT^dDANeGk4g`%nKoSe8t(q<=ReQ%Tn3hq9f(&38gPHSwG zysPZJX8nzrxK3vqM5UNA)pTR22Fg>t>>&*gG= zs!K}|h=#BF0$91I7(e;Tk_H>1;$i+$=E#-L9Eo90c$h(wtf5xHk=@BXgHm#^uS4$Y zh0Kuy$^MR5Z>*%PHt1;;Ey|MMku(z?Z}fM< zzb^RK4gY%lJ$MfOdmZNFH~*SU!Tj4`JjDOv2mUv1=9wcCJ#v3|?S+PSX}DwBf7 zp1$}DE<(E;$|4oErh{2*;(oogG(wSzp2Aee4q`@W)HfgX)gIy|iku=z5gr*?@3bY~$j#RlP z$fA1)@|V4E_YmYSOa8Jc2$SOItH2&H4&~Ly-(pR`9s3>b!O1j)!dp1;nl9;b@Kz<2 z7?U!xmXqQoAmLQDuc#0Sa0tA^ZZHDh4yPv#c2I(zot1;4^XsACe9IsTGww-*I^{k) z|I@z?Trq!S*CR8}%nW}W=ney_@IT(bgR2UHC*ibl_)1oA2mWwj$$3UZ|&SBi7!O3}*J$0beXQt2cLo48!U z0q`at3?B)P9GjdPnhGDAI&v_~nrG_(QX}AtiVQTuV*iN?{Ow|6+Cn7KC^om-E*qE zTf%5B+{rBq`v6?joY!u8b-V zN-C&_bXiTyn&zf-v98*gkS+$L)7dLe*2~#^W@=WI0lxgsSy?B+9j+$REI74M(j7M#+eC7U=p%3rH3W;;Ho-_m3Z% zKwDnX+XncMz&kgLkA-nHZjQ31saqz(=8uM{!{husyvYVJrjE|af9c0BKeYbH{eHF@ zgt)9cuSXIQww>P>aNHLN#1tLYZoIn9M=lG4ZsttvAw8m*4cLB*0_>R%W>SPh9baZBwbPA#;qPL&Ss>X z&UO_)m&xl=EG>tGiPY=Sb1xvurTz=NEA zn*&N78i>UJ|F^+fn9?K@7sX*Fr6q6^)}H1L&8ZzY#yO@!{YZuS&AnNCM>X ztBc<`l`-FXUkYZ9?x<_GsLSrypmlRl+#C>px-P%{Q{US^^1x<4^NqqArR9tUGyD`P_+hAj13(2ZH~H=Qr4x zCorky8Cl)<%P*b!(_QxrKoeo_avZrCG|_>CGM3N8Wc4$TKe)5?Z(h64Y>{6#<`jqM z-3I0RbcPel`rV*2K1Zz+3~qoKInPgQDQFvx<7LpmOFYN%C#jhy{^`$ejl&RnPBtK@ zKp+JyFw05bOJG(Fk}oF2Sn3m@x31jzArJc4!CT{=^_)nNnmM5w6v)XCI>FBgatD9K ztNBUuijQB}<#DBfU%A5L3d`{{-S5RRcH_r2_;D?CllO{IBsX)IHW&AXHsZl|DpeOH zv302N_b;8d!&8l%Jg{|~kl>HcpfUIgJRcc8USlM2RX0%O>?N*NC5H>kvc;`(>LFX) znnclRT-0c3a!LtT4R0C3RbLq~!!En9*ps`pcjIS%eH_?gl&we3?UbJSE0mtx(tOn< zr*hq8OHL)VY2BTMHf`Xw6E}YT>FXc44!L?MrXm=!-BXsX!tRg@R-1+l4K*wmojMHj zYE?&HfnB(61-9Y1T8RI+Mo5gcJR*KqO9xG}CXFsIuy-IGG%!ojK^yq#^=vnyIZJaE zR)HFBcgSuOXi}oiC{Y7_01-r_8@zjYIUcdc2fAlj5N~xCe0?ZNe zyc`FY8YL@~E;{Qds|N|YZ-<@3CVf^_PM|Hp?f{()d6$B!YKJR~DqPJ%bC5bKglP75 z5%r>2=^8K;EB!A&cl#UqXzNR)SOM#bN7_Er3ZPNqqEoCiR#mZr*ap32_=BwsJZnEA zXzz8NH5kT*f=O<3*Mp4lue-$azhUu-2UY*fiUb152=eP*rABaQGO>qg~i4 zVU+FC;dO?kY+O!5Q-Il;EjW-$L%pMHXW;L;ez{ttM5q2NhzP!-AtML zD3(qYDs0JHJRjwzux;S9V2!zG*>z=GX7wT7jEl`oT8T;NQ6*-1*Z?ysvDg@IXO$sH zB5-qY5m>UFv=rZ&Q)sEMVv!Qn@x(#p4)Fx?!0l1?Ub`?}5N1pD>`A;II_0WG?>D<| z2aK{VsXFY%A*U3)8C?I?$lVWq;i)Hk zg*(AosK;kER@m`(pYS;8`9h^%71|doH`WED7<3aJlHz_~$}!;O?kjGR{NB-rH{SH@ zXMZkU)!V_#Sko4-uv|rl_x|kOQ&)fb4XgiVc@#^ju zf6#aB;EAVx)W90SEh{d8D50}U96ssRz`)YD4kQQ=$qoR&lxAcc+|5T>3oJwNIh3~G z$z4YskHWUwxDW;tyPU=J;hVL5jteMqB0L|LxxF72?soAQ@?F54*3cnqh4L=rTL4FG zJv26XFgzUD9|@1>F6LZ5D-lra~(8IEqUf9MgY=16s_ zaWkRfs_w*Yxe?sJhNDB1lk*Hl1J8T_*9iCz+z$`a65IYSjBWc{I04tHWs#-rHd%L& z#%g0^Kw>^0tvh~J&lXvlvovRE&O*+~kg%A76eAWzbHQX3$yY(Rsi)1EBKf>^!N#uE z-WcIV!WJ!YI>NzMQ4vmgky>QAaM~O+>~1TwZ7-!Z6zb6uyLoU)_TJ6Ig~Hu@sS1Q% zvf@oXlw83kAMWCtd{3KCDO1H#$g`9xYo%3o&4w1N!hAYj+v=KYF)m&}La**#h8Dv! z0{^_%YN)KdmV8<2jtdkpaZ8c9;}TdoR}7$x)+SqOG0cm&OQ*HTWlo~C$)*xeC%0*q zDxK2|QVcAO7l?+P)}|$^;xDST$>(W7XFir32E zvoW3}6qmF_C@zHJf>G}w8*GG7T%xQ1Duz&82*rg^T&e|v&o@Y1LR>OJ7d%BME+*_O zw^6xGEwxP6aQ5*}ZQxJ8ADp(pVVuIC3`cOwSXKUE)-=qz5glWGt6g@g#1wb8ie9#ZZY{vU8q)@5$}$rZ zcJ&&LqrGcZJ$BQv#311BQer|c{QPQQsQ}Nc1^j+>(gxNn!PVf})Hb zs=VlsH4>qMqKom;ehVG*Nr*f74P$GA*h}#Lmj*woQ(COn>}B-^)FryVA!c-cZRZ*# ztaSh>b2*_pd2nD5Yq2g=iviYbWvC9b;QYDvtQiNXHbAd5c_XYjEoW!+*`usInUd29 zQ{fMqQl0HNS~hP+nZPm1n@f3fvopmK3?aS};wwb%1TEGfQmU?8=&--;1j&_gbuenF z$2|_I;p#;V*CvDZO;)3F5)x@a9xaf(@bIzr4~CC~M~+QS4NZlQO&vKHW=*#aMGuAH zqMx<%ug3~9yLu;$FN}bmM8uacuho`3%dlAW;eTr;1Wp?}IB#uX;r73zwxqk; zRGW0(`>OgMj~<`<18MZUGh$XtCjcqb-p#9qlv=^M>Rx5`4&Jrnv2Sj9`ym`A4hCLD z)fMgBNcaP`MSiXo03NufWi4@IE>_`>)v1p)Nq%)B`6)d2+?Y}YD9Un{QG4IVCBJcA zt1bD7*jZEl`}pMd{~%<)hPMnMts@mXFl6UVyCy*iRxQ(Ra3J5LeP}Ztf5^O?YatgT{Ut@K4J(5b-FPf5e;q5z-nwcec-|QSP#8j|yci$dU|I)SlSKt0d=hr{ePuc`;Z9=tl%NLt6 zqN%FNh?W)L?ZRb)WGs|~>eS^@3=zp#7qpC3r?f|+U%iNaxCKT9KVwb&6OmiM$&P*(x}db|2N8SN8RVMoGjwj=p^=r>2Q9~=!OvDj*|vn7l=5kZ^^Bud03HmxR6tqe^;9|(wK8)7 z@(v+v7CUx;2(*n{fq%$+O+DW z?>@8Tqh}hJe?MzUD^g;=6xS6s!q(`sko{{`NhjuWH{Qg@kexs>Z^!TRjDuk}FM4II z^^9Lac+HfOwLqw4bwp04vMC)-V$cx43xH5t z)YoDN`R9Btm%CGChnCazC95~*;`&6!TP8 zlkEV^I`OG}!@TG@IFEpgc%F5W@UCE7Wa&!uI% zouyXPYWe7QnJSjV%Xczab5afL|#z=^!74qc*`2)lvtN zh9yObgU(grAW~)2iz;KioXuwdS6`NwEeuV23T=hK#bK!LRwRmuIfrIl8Ca+yfs`K0YK(~jFQNwWu0ABs@>L|@B(cHap~4$J4-zV>D&YG z#A&6L-p$Kg@z(pCtltf~n4{JS1~lhn);|Mcg##$gCOCmRq{Admv4kmaPm(E$-`2DO6r(6i{g97}y7 z^wyO-Kjc9lJNP5+SuCN@3 zR(~&+u^T_G!H;X9o4i*HZ(Ma3_l7p&Q5$(4TgQV1md@LuO^uv9c=M2uu$2al!B^n< z$nfD?10#v6UL>woffciCajTq@v#lhXM1;UXK$nuLC&GzA+(xf#4&9xG!_vTOCvN=w z)7L+89ck0NwQ0!Epl>d|=qJL7DawIDvxXumfECnGbUGh((DfbtM01wrEX`Spej=P0 z(CK)?uzTwa(5J>$U=t9p<+mYpNQLU@VKZmt?-QrmzLMUG3VBHU7K=jiH=TK@qjIFh zc&=Ww7^`)4Vc9B-a2abxfeK>^spN@c7BEN1^Ku-#FO{rlw9Y!pT9Lx;+rjFv2^=hy6K;4Moeg=Hg04p)tLs9B zi&O}`uR?BbsH&Ny%ITr(InQauOR>^rBJuRU{M_wt=%cMKkz%E^Vg-#77agojBPj3B zJ}?($SvahesBN{6iO*}JY?H-&CS>5(BXU|=m}ZwIrBr%K(&omq!?SXH zE~lijdRSEzb(*b7Dr#IF0>@ADJ8}(ZRZ5FOt=aejKwrh@z$z=)SsvUrN7+`$ee1kt zwkyKcXQcUqk}9Rsayq4Dzz_vSATW)MC#^ofrln*JoX{2S>pdaoH95jsbM|A?Y)4Lo zMjlj^8E`h&M)L4$#wuZy?b6|OhNWy=PD4|G*_tglkV-?nqikp44O0;I09Gk+eVT26 zI)~GW2JbVRmL+g`on~9$oEaQGtYmUnGZYUy{qfPy7vGsvXsKupG#$I1IH=qqo6si0B5ZzdW;Gk~ya;o!wTpH;rdCQ|+7}H2 zYpaWhlB>KTN}vOr{_D2qcp=|KYInXplF8*dTXQgSFdU7Hg-6hD!1F!PHzNJ7(+|G< zzFk62DTtMU>)#r=`@t_f^<=MbCs+&h_{_!%JO1ty9w$9th*PEa6_v%xjdcMj2Hk{* zq_`iLatwI6`-+<+zjyTEjW<2}*`JG7^>*+w`gZXO%T;uE@6YZ%b@iv;z>2W%S0B8@TB2;1X6(@1o0UiwI4Ug7IvXj=JqyB_*e&5o;vDpnAxt8I%B#Cy{6XKfgD0N)Q3JZ& zHN_=A*0yIxRkj(iS|q>!{;V*h(1}!UAIy zirB2>b6h}?6XE%|%=5y)E~4EPEUBSG)(S;*;g|qNZ9OzLc`!U2*&hjy=q~15KDdYz zcRxt#ONu?i*;;9kk;(p{N1&P-1{|3cA!oZ2yXD5Bz^xD98Ug=-`{B_biHsfX|H9a| zzl9TUt!hz0wB1g9Tt;tB%CTL2{Wg*}S!AuWTh+p&q&Z7-7MB4bhF94pb-vD7$AU?C zlrw}!nJLC3Lfw$Li;z{a$kxCj%Z1bCAc4TBDd8~T*yKY*P6eBMxQlP{b+*B=$>YUT z%|_>`%7VzGa{>b6<4`Ziz6;@D8u%4%U&k7ETAQl1Hu*ed zPhcB?FD40IhU*tq-!U}``l`aQwRZgI0-%N^}iHV1F%W#ZubSPk#Mr?bW&Gzb1-Vl0R($Q1w0t)U+paBOsnhYD!-K zpue=7ozZ8Hvi4+3PABZT!Sis>QE!(^V|=Zqhuk*W3jDIrUxC7)DnM~|r#ZAoka169 z3WK6BD98grVNkB+hQgqLyF$Peg+W2?83e2}*d~vW0-#q4gOU@7no40%upjVRgDDJZ zdz4Wa6n6%52QE<<6oo-i7!<6hbrA+d+ieP|09vGw3KQ!cA@l1Sn6iuJEX`S(v)P%D zbIwv26bKZYchzayZXO}SC}n(cc$Qm19dQ3Nk(c<;o;~^(1}4=W@C&&TWZtGQC>x>? zldZ}e#(~11Jab`_t2w!vpQqq1d0}Lg5X#1uIA);8uC)z={>nv^LS& zWWjs6_4VM2LSaznAMhBhP1RbPC=ANl&1iv#etxgvING~r z)nhjuOANkEj-%fA`Bm~6uLb=6+3UAJ^Mprj?N}f3r2oRSE_|AKrg$((w5!jc)t;YC zJn+<$pMUNbNUP}3%eB~mWc_X)f31Qy!fc9rJ?k|9hwYh%`PPF%CvN*7^KLOZ2Fq;0 zkJj3&0-*+PXY0c55=q)E&us_4I{idjO8Tn@{!1jpksx#{3ye%oluSdyiw;@IHbMnO z1L`Z12zIK`E5C)Gp9B!`-!Qf|Xj~%=^|{edi|%iT8OuR!=US;O*cWp^K^?CP)nb5E zPh^J7lz?7MS&k;}Ig19O$y*=?x#OukF)z>8$?KG0-UJEe9uR0g2GHouh6MBP#u8e$ zT(erl%E;9r=)DW5fcmR82nUz2J8}CqoX@TfhC@X??r{i>s9w;B+GNna$!b(iLL#kY zM#p=@ofmFcCK2%^%xkrE0H)TBuMe9KzhkTuHF9&EHkKc6ZJ}tewMPei1@w!Twxqk; zP?B_Y-utThACDfN`vYn8&O41>-O3;&{BgvDMRkT%-!fNC?jYfhgg;PtyG2SvEbH}@ zNekDW0{!q360p}P?UCeHHmet z*kuT=(u4q**+;=?`;emg}xcs@mXFl6UVyCy*i6RtQ(rRG|u4}mwZL-xdRbv;ftGzK6r-$(${xnD;uOgnSRzAgU znl%)Zh$JSMrMI%8h>`Q%hnt&UP@4K0X;`Y1Pm$(qn&vFcSqdcP(*Oto47v>>`!1$laH*R}DMM*=7pG{d2ASgQ`T| zXP*WYUW}oF-^bQ8`}{rpRT*nzO~yI6z+sZou!jy#1rsx4_&9e_*)Bp0_eg@e)TY#;~WBE)B;`7-0gd9&pI?wFP zFakDX1E7;uCsSz|v)&rNTQagftMELy{0!P)e4l_MpJhyh=cfY#t7zW>g8tBO94Qa2z2?L*}()TG9_lIo`$=sDA{TvKBwq;m3OO0_z6*Vo}31 zU6rK_yeF5=m7EN9D5{3AL2!P*C>w^`^_xR{>1=&YRpPRy8QpJH^t{fBjih!(CLmZ! zF=~lqb9w!sC`(ZR!BkY{K5f*w)~J(Tv}#Q09+*hQHR_T!rv=BeGJjgsG6m;!wzc@A zQDFl#*R03)ex3D{=|c0(L_N;(+w31kxy`BUab->*fy1!*PwQ-VQPIurq0@Jiif!_OMs8mD!w@GSJ^fEHq4V799Gh4IgYfY1rB91N+Oj^ zA%}>jH2u$Q@9FyboAnDbVEWnDMV5whGo zx0yljaz4!bgFHblUWZeFB4u2#88n_3H(NZr8J@*MfGO}N@n?hMhSl&}6uQRx#(V{( zqEkFS@M-f9snPLie!QUgSRj$??%LDY-QCeC_jGpj_VmO$2KV$QI^uF?Pmk0yFc3?| z7<=tzc!D=~#h{X#?laaU?-+MZ4VH_71^YLrE+|1aIHd9R4Su#N%C;rsq?AuP+#oo| z1z<0Bse?qKAoGe8i@@suf_7ukV|)rg_z6KJ!;E%ySMTYu+uGBo0RYiA&03{6^6YUn zA*(QnH`tRH^H)OWHm!Q;yU%R-=$QuQ-w%r-EJ6FFIIIZ~wnhhbqRlGl#C-0?o5U!d zK;m!5@AHh0K=Yzk)>_Z_UCe4cb_XyanEp2u@xd~%l1N$;@(geYu{w@Ebp6Tf-FGUm zrn}Iwt#!O1%64f6f~ni3Xhkv+LiO9s>Sc9APNuRc^O)xm@l}l^t46*O#o`tY;%o3E>bg9fx?M&YFg!Lz9#9 z42B8U-wn{EKF#hzD;Le@7%;3h?!JX=K4UzU|fKzz7!e zR958#pN;$-jx2@ZN*9e&!LCyj-^$iou(w>UHs}yK?t|eY;gMsLQ$thXV^c>ChFR0C zL(xNFxB>c)e?2xCo`?)ZBh%pte%2ekqQs(CA=YiRf~~qtU3vH0$zxxA^YK5f6T7|v z&%PBZczdO`pf>IPlV5*Ydv)&luZd!owB8rI`aamInBC5|SxvC3(g_yrJoDQlRe6w* zF+@rd2Y8U*W7mzMnzvpWw!jEFZh^-GTflt};p?6od|lkkt$_b51#Z+!;Kq76o6n#k zNM5!uG+mKk5r+D1mC||Hp7Q~YN`V*bbEP6VnqM&V4V8&89<{ylmHSVIKD_rNsCuHT z5no?to1$F)3E}Oi?1Zfss1w$u+<_R^d+)=F0x@t=2t}OF?yXiZMrWBmww1^NvIbu! z;Cn;+i)}lvJ;isvjr+$BO`r;;=xqahNZ_3t#>c{3mj~Masaqz(=8uM{!{eroZ-WpP z9i5f`(vM$$X#J7<{cLpv0odpDNFo9XNW_^7IPMDsVv4RS8N9m9I5<)^V<@M-scoe&|mFq0I#u8S(M$~nRo(npA&U&pc{^phI zTJH-y$mzE^pyZ)}SPW>nn&eDQU(nf(GH(pKe!P1v)N)zTI0`a=e);(u@go}_4xaI< zB%6&SKoNI!@jIt7=3DPef$mOs)U{jGWp`}Qx;ZFr4v0Tpm*4)W@9iIXU^6JDM&XUX zilf2oxpL9xx7_xh-9P^A51z;E35LD!13UAIqDPHZ1`Xwwajy>)uk*PR>tMV0J2-^{ z(#YU>p$+5KC79GQsM|OG@=K@wbk{ut(8LI=m&nbaiH;5R7`GUG=J5x2w*Jj)_n9pM z&6b!`Tqm{-%J=CECzkcQL1%o9S|=FX05fu)pV(5+HXO&xpn;coj^j^KGf(`}pWhmX zA@rPVKv01|3Rqy4lfH-T4&F)7Z80InQlALDb>+?vdCf$Znx;2=FVph)bRulM;Fi z55CA?7hV{EUo>k7JjkT#ss{z(G&vu2&~*;@MRS(sEX`Tlg_Ua)D8zz7EEbXH+%PPj z5S3MOLHA#*Ar@pesu#P_YF%AewhAL$#+qTfqHq<)6jWy!iA(7(+MhIxL){e>>?-rc zQzu?{{r2zt(a*pcuUcs3V2+UIZYlb`e!GNtM$>*(Rxyij^)CQmy~x=Wc&PA8mb!6f30_D`=Fs=#bPKL3v+2 zB=xr1`!_&)uk++UhOwbwlEawd0ji%J(`>tx*A)~coc_z6HvYmxDOhIfNSf#}EX|@6C98N15 zyw7l2mcZq8nr(q|reD*rlF4DsP;`{F$CZRUl$Fv8T1qMb z;72NCxCRUt+z(uRhY_ma;i9@&l*1O!N4Y6%8#pakV=h{DUD=jdeTX;XVl$IgVp19c z&2ZGfBo2PbB^Mjx?W{5cNd#^#E&@wdxj#Po`Qkfs3N01Qfu>{E69<($#1p9U*&b!@ zwF~0~VLo`zp2Yj1Q?6R{ezW^_z$ojI3ek1o1|rWam}JhWN@ur~?M>rZ9(4fEh9+EP z0g(sXXWXTOwcH5#VLuccfnD2fo-s}|Vzdx`Z}f@4A^X$Ez*YbK<=9Rk85(7>N<8pa zjYKAg_C>?M+P2!!A?Gfl*gQztOTquZ@f$QHIx%AsG2k2CVPkNit$?8zd?`?H8u^kfavL#hMyTBH8kSa0Dip3MH{a0DM#ctc=n0C1N3QaBdA)=Z3Gomf36~^yt;cC zVe~k-n`;*~zd{jR_!huXTMvy*9t;mh_D8}a?pWsX!9|?7`$1A)QtTOL1onI@8)Rg% ze~58|YHAp8WLAWn?N02L8;b(BK7eZk{0HubM}s6%FSY**W83}~PQbOQMFr7zo2)xX zQesJ`6inQ@Q{NU@Qjrm`Fs=(UXGOFaY?Q^KXfBxSP3W9^VEz0UER+J1gRy@%Yyowf zB(iG8z&ZsU<8li{@?|dQNIpKu%|ThXNXw!wMA8a2HoV>uiGq+LISkHUAM$GYc%`=LDdP)+Sj6(`%b%`&yY$ zr{@r(jMgS=INaE9s1?AY5)1CKRPk#sOjW%1+T`<;J)xWZ-WR;clMjp27+v1pQ9C<_ zua>qA^2exQWy_Ab^2Lp{4``6rs2T?fyZpBwOy{ z%)rp!qIbiRBmD`*g%OGi_#RsLEDmH9LU9og#pWa<6c@>rfq_t5#QJ2?N6if1gyI4z zKD-ybs)v48$#!=3xm?=7pMF31(SyVIvPCWmgDUcvw>)))|22ApU*KU-g9g6Lo_`2_ zxA)!8{oA90PkDn@6^i2*6IwI6&`}uFtQ;(IZ@0G}9G18|z6^yyQ5X~?n4mBy@XrWY z%MFD=0e6M?1`30MCL;O|pzBzfYWM`$~E% zW{MhRzF3s`B+sx0GaQjz&28&1xtiZ3IFyywx-b_uxtf!!IXFx}^01?h*a~nah|eL$ zo*ahES8^7C&^BxN`qFAp7!-v;`PnLfTr!{ti}*f;=oZUjZZXUyzm`IqqcA87G?EJx zs7|F2T`)~r5z!Q)3!e7IT%s^2BNW2{eCC0QYW5cVH4w=t49bjaQS(_HTw;Qyk>tEO zl)|9uB@F7)IOJaDO9eu2jsgBIW|!9i%GX?PS^EdWN5Uh=CZ~p`!pEkL91OFjTZf{D z!f=D1wezpXCc_hvp=e|}jL9==qZMyV8<;7}#n10G97lWCta|LGV~N4H$#K*hKfg*o z)xGRd+`dDVP_Y0&| ztc9ar0+#i=dHl5sXC2gAE$A*Rvjsm|Ymfa8HF%>S6?RvFveCh>PCwC>lK$#}{}Osf zDT*1@HC;~Vtlt}coEIIk^q8T7qKom;ehZe*PXdVeZx~w}gwDq9|E0l?>Xa6%HGAtW zb&}}*hM3X)wVi9FvS171v>nJT9k{)-mtTKcdv)&luZiWwo5ne0lU<4_O-Z!O(UdGm(-NL5Mj+}IXk1z9%b#xl$=i3bpu1+bJUymjU)tCr6Kn=8`Nhl zI#$nxf|}u&z^l|^U8oiVtl7#?9cID#bM09(4ibxau4R=Gt_VglbAo()^80^45(dmk zn!Gf5AN%H(w;w|0$gR;;lwN4^&J;^9g!md1*@lW;(5FT`q zRrNm}JwEpb(&(Lc8ogz%qd=+Tn;vrdrgIG;b_*ON{E_hI0juxU{E(k(4g6f|)W@16 zzq*n9lpN;Y9aE|RMOn@=YVWVO*&i_&0nHklYm%95aa65B z?WrIw+N`-YYc(>n(VQjII9b;m7N0_;Vu9n&QUe&dRD%qS6fC4*Aq9&9uESR16~pf3 zy;G~+JS0s<-Doo6N;)maQJb-B8&HF5+-P8a0%eh7a9lUGD`Q={KP*qLI`U^o*LbHal z*yG-UAlIB`?fTY~(43_?OLJBt$TfmotGX9{HLSicH~n=N&_F_l zgie8)mSbR%hfxKIqgqx>8G$}Fc#t)!0*x8PdSz$QvaVzo{_dW?f3CVlgo{?jB6nZL zUN!72{YCZ=6W&0>5U1d+N-8lXU1Xvk?mtA{+chBFb_#wCpKJU+f0yyAi8aADKU;&2 zcs$TP4Lj=Kgn-&I^R)cVXY})BC2z?-daU{8mflbON(e}s>>JHOvOj(2@(*0LcgOdO z?l=zt^Uz0fRW7fZ0I@;dZ6HV%0L1$Dzwd?{-nnWJ-xbVHKR&sgN;lfCW(-%ArD(JE$M)c zsbLFRp#Bm3vKBwq;m3OO0_z5AV^PC2U6rK_yeF5{m7EN9D5?g|(RkhfFaoS^*Y6=9=~R-mkNsGF@oCnW)EEew+QnD7QJ4 zJ+92jX3_rBI@?`Tw6nmBl$B;=H7FXq2LJG)aFDlK#k>hQo@|PTxh_k9meeY~IPO>3 zHT<@`o=OLYE&IL}IF!vOiBvL$w)bdC)BoJ|p02OId4D^%U^k}W3|80-2I%)MUVvTj zrTf@l{^%or+Qp0U#dWsDk%`K_4d4x6&I*3Un>M7X(gL<){;q%C`Nij?M;e$f!WuQm z|G}p?JAoN+1$5SR^T2J1vn~ZD8^@A3D{)rf=EPZvvzFnk9ElJl*HW^rAB7Jg+k*59 zeAwS3l9Qvf%P*i315O)vl`hA)owc<_w{gyvy!~vobw9L? zx5;?g=WYqy24_r(9b3y9z-Naa_`^I<*2n)wM9m9|j|KXqnB3Vn(ACk|-__mGo9L1| zVgr4#j(BWOztofH92`vaG4|4(@EA59+J9Tol8=%iUs!E#I8zGXP`qx`-GKXNDHpk> z`;0AxH&{vK^d{;d6y~&q`)$S!-SKyX4IA*d5>> z3Xg7}U%TnO^j@n1}px#ZtIZ zvXiAJJK3nSrs3$&Q-LiEK<(G1vbxj*qi76mdRVwcUJYnLf7hheU`vzD&UPhV~cRc3yjm z?|d8gj~|-Aw1Gu$8{k6%@7ypx7UpJ{C~KO!Wg=|;XqY-YZklH{2w~CDS@|#h_~nPz zAGsftVE|H|mFM+HA_BG}9M^#3zCa+R=(>`@tJ^%T>Pjw>04oD;&A6I?okw;i0A`~+ zxSg=8O10a%6JDUrATHf{Y-g#*CS^SUPproFqU}{@+l^c8u+&gQ7luo?ajQp*vl%G| z=|zj5%j9)5bcI0GQsU9$@F;d^D>pZBV-rY`jcBG4Jr{KLob_5?{LL%ZwcZzakkfB- zK*>V`u^2$hYLYWKeL-hC%Dge``tk0yP|IaS<0!}g`sL?u#E)!zIC#dZl594T0O!H0 zi{CkwG2eP$3TBS(sB5>V%kJ2qb#qYM91wrHF2DU#-`hX(z-Ca#j=~#3V5qD+5O|+{@{7s<)De4AJ~~!6g_IRGI)!m&BFK?_xeETI-fhS4kY>C!6_V&Mh4Gs zu(!@Isb#Q~Zv5q!PW|bwdj_D1kXYk5ax-Y6V?#a0js2f_{K1{AfAiXXW{W_xCFT^j zscwVveLBO5W&Li@8K0xp2?jU7jGX5uwiL7t$MG^~;3b~p_>A3Jz!+_Rn&DN-{hRD%LJ8A2!c zIYI8=uXr^-XLd*F}&R~UECYmh(~Qa z%dvIByJyL~9op2$$%8i!2?<+i&=`CLo{tP4zBQ1k_>zRkJ^b&m^!n1aH%U3cT?YZ1hmo-9s$i+!XuE1Y+0x%@iVd;6&r@Rea4Na zMX50?Yp_sH6{@E#4}&EjsFK~NUhGDzb#-CcDvWR$Yldz7;!TvSXtd5c%36`a?%Tn1ut}d)l@o5j zN}UaPmx8M5kkxe|!@oC#-d7>FH&oS3Qswke_MGRm;+eSK#K!D@`MKNQ&_`QeBE?E+ z#R?iFE;>J>#;We|+G>xj0PVfblLHyXhJs0s(l#8hN!~Hdwo7?kK~ci#zZ}lu{7;Os zOYrJp2;@{Vk~)XxC>6|4(`-{B#W8ISFHL-28)cg;<})D!zaEj((!w;mG%2Oh7?Cob z9iElrb2%lI)x)Z)sMBmsQc>gb5IBCC-;rxTt5RANYR$$M0E{+1htaD_Hkq2qgWKjP z+X{iA&TD47B5Zv|nm;J15}^D-fL{g-QD6juIN0%|)d$$Ll&np&R)za|Psn*qjG?vOD!s3$ELLu;3rI2OCOjm?{lJuCz{}lN+$8zE zqYrPq>DkZzT)e8cgO|~_i&t2#qQiTCcJHaHKm7()gnhp<(PRd9*k~|RdxK$egAH>9 zfkfs7M+^~XKn7n5c<>p!RD-)4weBi=uUS!}4of*c(Zbyt8u4oYKi=b_4OjS-1GED? z`^4S>`n0wYR4q}qOEY$8?#)Uh3mg@eW}S_c<(`G>h}m)O(qLg|GwYF6-le>IqTo$ZbXNSEjF0os}m1M#ZJr*!qBCGC#MFf>scfa_9zH0|hJoTdn3^;3w zOCU<<>=K8IjWsavKI=e&0Fmqf$QOXafN!(;C~JXEGeAhdzwD^vQP@fqppz^mE5;^qR|Anz_e+wtzTGgV0XuAynA1chc7CoO! zgR~DpVLIF3m|y%;+y=M38AG_SrGZsdwT4)_VnquA&>?1?@}Bw z7kMyMv0@AEZd&!ycc0nv(K8Lqzn`_F0YY-W6bDSl2wMX-N?Ds#(uw(;sq1no=JGD(Ee2G7foL%R*weA4GS=<$UaLVk zHA}~-*;1tLxP;3VD+EwRYm>c?ptb2HTy^I%-=cTJk|X^I#f1@y3wSeF_$(k+pedP9T+%Y3xDbj+LUG)RrL@_d{QA?{t8>qPO%$^vf7&T@jIvAG2XKbA1ltIRXOO;3UjU%LbK>KH ztd_!{Dlqj&l~O(=*o71ZMPX3j9!+6T86_LCT+b;C3RnXKOi>sV@_2P#6@L_^i}5C{|Gz6fjQ9UyH(^fFvNWl)|8l4J7`li!dnKZc`W(&?1FY zI72Gx8-CJjtR65W zg+XbE!DJha$kqHNU`lXM7!=~fSr94}fw_>YIk}p9L^Sh&KznU7z)gea2kHxjK^fWB zz?Y*0;1)!}dIN63g+fhpdFa53akwZ%m$l<6RS<|vVNetX1xwKR2s}n%P!tAbp!ZpZ zIbd)U2DK6ynM!T6i}Vs@;$!$=^%w2eO=%Bc5p_9utCC7!%3cjnZyc_Q?9wAlqXpiR0zlYA)%p0O2@cknHZW6`i=W?XIF9zNS@qaW z#}b18=7G4hvdC`B0fxMK!>wANk~2sL;+TNieh zNYZY3ZaetZ=_lG!(qBFBUyxjRl=-A6W>g0^lZ4Lty>V=L(IG3@MyR0ZV!X89LdSd( zK*WE;*xDdxxCGRd0Y^|r1Si*=yIrUQ?w=-4M`B%G4nm;*hM2J&)ON1Xy5K-1s{eJN zS`4seD?@b_F3z88&zku}t%WQ{llPoO1666qK`70IxiV}$yT?ihGYJpwJ+*R^ms}ml z)nO4bb?X%<2dC9ti|jrs&%udeYMt(J2#u&-(1_Y((7wrPR8B%7tz`ld?JJ{vuotPx zxRjQ{yjI%@#j!*4vCW6y!6gg+*T~Iv+StK)OPGZdu%x!6yW3EbbameQs`?*~9-sRI zY4j*XM@8uF6)b-!uAPr5I!O2<;m?w0itI_`=eja}u2Y_ivG;HkW|pX7stJ<(>PGTY za+r2+OsN7CWjV`ef#f&NYqcf6QdZZ7w+tbz&y$dakY<-%SnSE&+Pm>HzdjE65~`OZ z9*ni#?&(F-%Dud!1_lnUguy`la@}RwhJvN#&Mrgl9E^2b|K&qxKHEoPr?=RNA{6w^ z#h2O*ZL-zV_^(?LjX%nb_#ci2N#s?;-_^>e*iEyBf)atYD3I70lvpDT%MoMorj^c? zbYO|vbJW`{M92>zOPmS0M7ESBqwzqM z%uQxO_^v%4`RY|C-@bD66|pQ{Uy+8yv(2-~XHWJ&{OQZMnq-5VmNT*wiM^yB>tX^+ zooGtB`VCD<-?#Sep8m&YUjH^}N=j==I4nMQ?N$v*wSrv51(NFET{|B8=9ae~!hSw4 z5^6V8Ra3J5LeP}ZtVxrmgft}u@J;otDWN$_a~7s7X*UUS?c=hFlmLe?IKmgwl#r%` zG$r}04oNhETG_aCK5=TqMysUv^s%!Ix^EZWz7wgLS#tNy&cXIJ6)e;UmUg@7G2MN; zcK_ZJ~&Jg;_w6~gs2>zkIU$;`#Yb(JT5DFOZL%Y%{RC7e(G03B-&)(Xcm(F={uKy z;Ih3tzF%}lHImTiY@-7&!tii9%346!kI#YWtI800xV)~)o6-gZMnQJ|V}` zkj^tZ!@)4H0gX|%I+;q#2(M-QZpp~{til1e_!+dp_&x!SN1j}9anV<2pihCYKmZFG zRbq$tsfv^U;4ZOwIFOoAdDgAS6oTUjNm^&Cl4(glAmw=KsTQbz1e>xJKi1*Ldh-J7 z2K!=B!!%u$r3}0$m(G=(40R}~2JX;!e!nOihTHX>{muK^xy`yU4QH^zW-vLwfAIqBf-l|2{_;m3`O_|5j4!UUErNky z{UQ4{fMkL>EBG02+K{SB3)qhNyZ(9S7oU?JX<)txYt+DG#4Voe1m?UIU|HAAbA&86 z&uwHdDFn~;_wxj~od2Sh4%h_0U^8f57C&e4Y-H7VG#fYbAMmu}hSk=kHP$!gbC&x5 E1K2d{F#rGn literal 0 HcmV?d00001 diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobWBroadcastStateMigrationITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobWBroadcastStateMigrationITCase.scala index 4f236e7c8cafcc..4d9af6fca3fbd4 100644 --- a/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobWBroadcastStateMigrationITCase.scala +++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobWBroadcastStateMigrationITCase.scala @@ -56,13 +56,15 @@ object StatefulJobWBroadcastStateMigrationITCase { (MigrationVersion.v1_5, StateBackendLoader.MEMORY_STATE_BACKEND_NAME), (MigrationVersion.v1_5, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME), (MigrationVersion.v1_6, StateBackendLoader.MEMORY_STATE_BACKEND_NAME), - (MigrationVersion.v1_6, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME)) + (MigrationVersion.v1_6, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME), + (MigrationVersion.v1_7, StateBackendLoader.MEMORY_STATE_BACKEND_NAME), + (MigrationVersion.v1_7, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME)) } // TODO to generate savepoints for a specific Flink version / backend type, // TODO change these values accordingly, e.g. to generate for 1.3 with RocksDB, // TODO set as (MigrationVersion.v1_3, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME) - val GENERATE_SAVEPOINT_VER: MigrationVersion = MigrationVersion.v1_6 + val GENERATE_SAVEPOINT_VER: MigrationVersion = MigrationVersion.v1_7 val GENERATE_SAVEPOINT_BACKEND_TYPE: String = StateBackendLoader.MEMORY_STATE_BACKEND_NAME val NUM_ELEMENTS = 4 From 4febcdc289bc0dce1090d831c7f99848a26f6465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E7=A3=8A?= <2217232293@qq.com> Date: Wed, 2 Jan 2019 19:31:59 +0800 Subject: [PATCH 006/147] [FLINK-11235][es] Close transportclient if no connection could be established --- .../elasticsearch2/Elasticsearch2ApiCallBridge.java | 5 +++++ .../elasticsearch5/Elasticsearch5ApiCallBridge.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/java/org/apache/flink/streaming/connectors/elasticsearch2/Elasticsearch2ApiCallBridge.java b/flink-connectors/flink-connector-elasticsearch2/src/main/java/org/apache/flink/streaming/connectors/elasticsearch2/Elasticsearch2ApiCallBridge.java index 73a69ebde3470b..34f2ad3a49e5a8 100644 --- a/flink-connectors/flink-connector-elasticsearch2/src/main/java/org/apache/flink/streaming/connectors/elasticsearch2/Elasticsearch2ApiCallBridge.java +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/java/org/apache/flink/streaming/connectors/elasticsearch2/Elasticsearch2ApiCallBridge.java @@ -21,6 +21,7 @@ import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchApiCallBridge; import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; import org.apache.flink.streaming.connectors.elasticsearch.util.ElasticsearchUtils; +import org.apache.flink.util.IOUtils; import org.apache.flink.util.Preconditions; import org.elasticsearch.action.bulk.BackoffPolicy; @@ -72,6 +73,10 @@ public TransportClient createClient(Map clientConfig) { // verify that we actually are connected to a cluster if (transportClient.connectedNodes().isEmpty()) { + + // close the transportClient here + IOUtils.closeQuietly(transportClient); + throw new RuntimeException("Elasticsearch client is not connected to any Elasticsearch nodes!"); } diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/java/org/apache/flink/streaming/connectors/elasticsearch5/Elasticsearch5ApiCallBridge.java b/flink-connectors/flink-connector-elasticsearch5/src/main/java/org/apache/flink/streaming/connectors/elasticsearch5/Elasticsearch5ApiCallBridge.java index a3453ec44452a3..d59b8e9f912ffe 100644 --- a/flink-connectors/flink-connector-elasticsearch5/src/main/java/org/apache/flink/streaming/connectors/elasticsearch5/Elasticsearch5ApiCallBridge.java +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/java/org/apache/flink/streaming/connectors/elasticsearch5/Elasticsearch5ApiCallBridge.java @@ -21,6 +21,7 @@ import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchApiCallBridge; import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; import org.apache.flink.streaming.connectors.elasticsearch.util.ElasticsearchUtils; +import org.apache.flink.util.IOUtils; import org.apache.flink.util.Preconditions; import org.elasticsearch.action.bulk.BackoffPolicy; @@ -78,6 +79,10 @@ public TransportClient createClient(Map clientConfig) { // verify that we actually are connected to a cluster if (transportClient.connectedNodes().isEmpty()) { + + // close the transportClient here + IOUtils.closeQuietly(transportClient); + throw new RuntimeException("Elasticsearch client is not connected to any Elasticsearch nodes!"); } From f6995f01ef748e4992a37defd60197d1ab4bd00e Mon Sep 17 00:00:00 2001 From: hequn8128 Date: Thu, 3 Jan 2019 10:10:20 +0800 Subject: [PATCH 007/147] [FLINK-11181][tests] Fix SimpleRecoveryFailureRateStrategyITBase test error This colses #7316 --- .../recovery/SimpleRecoveryFailureRateStrategyITBase.java | 1 + .../flink/test/recovery/SimpleRecoveryITCaseBase.java | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFailureRateStrategyITBase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFailureRateStrategyITBase.java index 6783df0a85b0aa..1a5b2a32f9b75d 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFailureRateStrategyITBase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFailureRateStrategyITBase.java @@ -33,6 +33,7 @@ public class SimpleRecoveryFailureRateStrategyITBase extends SimpleRecoveryITCas @ClassRule public static final MiniClusterWithClientResource MINI_CLUSTER_RESOURCE = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() + .setConfiguration(getConfiguration()) .setNumberTaskManagers(2) .setNumberSlotsPerTaskManager(2) .build()); diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCaseBase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCaseBase.java index d81274d497341a..43de1f4645b30c 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCaseBase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCaseBase.java @@ -110,6 +110,8 @@ public Long reduce(Long value1, Long value2) { catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); + } finally { + FailingMapper1.failuresBeforeSuccess = 1; } } @@ -158,6 +160,8 @@ public Long reduce(Long value1, Long value2) { catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); + } finally { + FailingMapper2.failuresBeforeSuccess = 1; } } @@ -194,6 +198,8 @@ public Long reduce(Long value1, Long value2) { catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); + } finally { + FailingMapper3.failuresBeforeSuccess = 3; } } From 903a9090324f8d14af15621b7ec026f4a8485050 Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 20 Dec 2018 12:59:09 +0100 Subject: [PATCH 008/147] [FLINK-11163][tests] Use random port in RestClusterClientTest --- .../program/rest/RestClusterClientTest.java | 314 +++++++++++------- 1 file changed, 185 insertions(+), 129 deletions(-) diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java index 3e0f2f525a0426..bbacd842dfce09 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java @@ -89,6 +89,7 @@ import org.apache.flink.runtime.rpc.RpcUtils; import org.apache.flink.runtime.util.ExecutorThreadFactory; import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; +import org.apache.flink.util.ConfigurationException; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; import org.apache.flink.util.OptionalFailure; @@ -158,8 +159,6 @@ public class RestClusterClientTest extends TestLogger { private RestServerEndpointConfiguration restServerEndpointConfiguration; - private RestClusterClient restClusterClient; - private volatile FailHttpRequestPredicate failHttpRequest = FailHttpRequestPredicate.never(); private ExecutorService executor; @@ -167,28 +166,60 @@ public class RestClusterClientTest extends TestLogger { private JobGraph jobGraph; private JobID jobId; - @Before - public void setUp() throws Exception { - MockitoAnnotations.initMocks(this); + private static final Configuration restConfig; + static { final Configuration config = new Configuration(); config.setString(JobManagerOptions.ADDRESS, "localhost"); config.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 10); config.setLong(RestOptions.RETRY_DELAY, 0); + config.setInteger(RestOptions.PORT, 0); + + restConfig = config; + } - restServerEndpointConfiguration = RestServerEndpointConfiguration.fromConfiguration(config); + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + + restServerEndpointConfiguration = RestServerEndpointConfiguration.fromConfiguration(restConfig); mockGatewayRetriever = () -> CompletableFuture.completedFuture(mockRestfulGateway); executor = Executors.newSingleThreadExecutor(new ExecutorThreadFactory(RestClusterClientTest.class.getSimpleName())); - final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(config), executor) { + + jobGraph = new JobGraph("testjob"); + jobId = jobGraph.getJobID(); + } + + @After + public void tearDown() { + if (executor != null) { + executor.shutdown(); + } + } + + private RestClusterClient createRestClusterClient(final int port) throws Exception { + final Configuration clientConfig = new Configuration(restConfig); + clientConfig.setInteger(RestOptions.PORT, port); + return new RestClusterClient<>( + clientConfig, + createRestClient(), + StandaloneClusterId.getInstance(), + (attempt) -> 0, + null); + } + + @Nonnull + private RestClient createRestClient() throws ConfigurationException { + return new RestClient(RestClientConfiguration.fromConfiguration(restConfig), executor) { @Override public , U extends MessageParameters, R extends RequestBody, P extends ResponseBody> CompletableFuture

sendRequest( - final String targetAddress, - final int targetPort, - final M messageHeaders, - final U messageParameters, - final R request) throws IOException { + final String targetAddress, + final int targetPort, + final M messageHeaders, + final U messageParameters, + final R request) throws IOException { if (failHttpRequest.test(messageHeaders, messageParameters, request)) { return FutureUtils.completedExceptionally(new IOException("expected")); } else { @@ -196,26 +227,6 @@ public void setUp() throws Exception { } } }; - restClusterClient = new RestClusterClient<>( - config, - restClient, - StandaloneClusterId.getInstance(), - (attempt) -> 0, - null); - - jobGraph = new JobGraph("testjob"); - jobId = jobGraph.getJobID(); - } - - @After - public void tearDown() throws Exception { - if (restClusterClient != null) { - restClusterClient.shutdown(); - } - - if (executor != null) { - executor.shutdown(); - } } @Test @@ -230,22 +241,27 @@ public void testJobSubmitCancelStop() throws Exception { .netRuntime(Long.MAX_VALUE) .build())); - try (TestRestServerEndpoint ignored = createRestServerEndpoint( + try (TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint( submitHandler, terminationHandler, testJobExecutionResultHandler)) { + RestClusterClient restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort()); - Assert.assertFalse(submitHandler.jobSubmitted); - restClusterClient.submitJob(jobGraph, ClassLoader.getSystemClassLoader()); - Assert.assertTrue(submitHandler.jobSubmitted); + try { + Assert.assertFalse(submitHandler.jobSubmitted); + restClusterClient.submitJob(jobGraph, ClassLoader.getSystemClassLoader()); + Assert.assertTrue(submitHandler.jobSubmitted); - Assert.assertFalse(terminationHandler.jobCanceled); - restClusterClient.cancel(jobId); - Assert.assertTrue(terminationHandler.jobCanceled); + Assert.assertFalse(terminationHandler.jobCanceled); + restClusterClient.cancel(jobId); + Assert.assertTrue(terminationHandler.jobCanceled); - Assert.assertFalse(terminationHandler.jobStopped); - restClusterClient.stop(jobId); - Assert.assertTrue(terminationHandler.jobStopped); + Assert.assertFalse(terminationHandler.jobStopped); + restClusterClient.stop(jobId); + Assert.assertTrue(terminationHandler.jobStopped); + } finally { + restClusterClient.shutdown(); + } } } @@ -257,16 +273,21 @@ public void testDetachedJobSubmission() throws Exception { final TestJobSubmitHandler testJobSubmitHandler = new TestJobSubmitHandler(); - try (TestRestServerEndpoint ignored = createRestServerEndpoint( + try (TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint( testJobSubmitHandler)) { + RestClusterClient restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort()); - restClusterClient.setDetached(true); - final JobSubmissionResult jobSubmissionResult = restClusterClient.submitJob(jobGraph, ClassLoader.getSystemClassLoader()); - - // if the detached mode didn't work, then we would not reach this point because the execution result - // retrieval would have failed. - assertThat(jobSubmissionResult, is(not(instanceOf(JobExecutionResult.class)))); - assertThat(jobSubmissionResult.getJobID(), is(jobId)); + try { + restClusterClient.setDetached(true); + final JobSubmissionResult jobSubmissionResult = restClusterClient.submitJob(jobGraph, ClassLoader.getSystemClassLoader()); + + // if the detached mode didn't work, then we would not reach this point because the execution result + // retrieval would have failed. + assertThat(jobSubmissionResult, is(not(instanceOf(JobExecutionResult.class)))); + assertThat(jobSubmissionResult.getJobID(), is(jobId)); + } finally { + restClusterClient.shutdown(); + } } } @@ -365,29 +386,34 @@ public void testSubmitJobAndWaitForExecutionResult() throws Exception { failHttpRequest = (messageHeaders, messageParameters, requestBody) -> messageHeaders instanceof JobExecutionResultHeaders && !firstPollFailed.getAndSet(true); - try (TestRestServerEndpoint ignored = createRestServerEndpoint( + try (TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint( testJobExecutionResultHandler, new TestJobSubmitHandler())) { + RestClusterClient restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort()); - JobExecutionResult jobExecutionResult; + try { + JobExecutionResult jobExecutionResult; - jobExecutionResult = (JobExecutionResult) restClusterClient.submitJob( - jobGraph, - ClassLoader.getSystemClassLoader()); - assertThat(jobExecutionResult.getJobID(), equalTo(jobId)); - assertThat(jobExecutionResult.getNetRuntime(), equalTo(Long.MAX_VALUE)); - assertThat( - jobExecutionResult.getAllAccumulatorResults(), - equalTo(Collections.singletonMap("testName", 1.0))); + jobExecutionResult = (JobExecutionResult) restClusterClient.submitJob( + jobGraph, + ClassLoader.getSystemClassLoader()); + assertThat(jobExecutionResult.getJobID(), equalTo(jobId)); + assertThat(jobExecutionResult.getNetRuntime(), equalTo(Long.MAX_VALUE)); + assertThat( + jobExecutionResult.getAllAccumulatorResults(), + equalTo(Collections.singletonMap("testName", 1.0))); - try { - restClusterClient.submitJob(jobGraph, ClassLoader.getSystemClassLoader()); - fail("Expected exception not thrown."); - } catch (final ProgramInvocationException e) { - final Optional cause = ExceptionUtils.findThrowable(e, RuntimeException.class); + try { + restClusterClient.submitJob(jobGraph, ClassLoader.getSystemClassLoader()); + fail("Expected exception not thrown."); + } catch (final ProgramInvocationException e) { + final Optional cause = ExceptionUtils.findThrowable(e, RuntimeException.class); - assertThat(cause.isPresent(), is(true)); - assertThat(cause.get().getMessage(), equalTo("expected")); + assertThat(cause.isPresent(), is(true)); + assertThat(cause.get().getMessage(), equalTo("expected")); + } + } finally { + restClusterClient.shutdown(); } } } @@ -420,43 +446,48 @@ testSavepointHandlers.new TestSavepointHandler( failHttpRequest = (messageHeaders, messageParameters, requestBody) -> messageHeaders instanceof SavepointStatusHeaders && !firstPollFailed.getAndSet(true); - try (TestRestServerEndpoint ignored = createRestServerEndpoint( + try (TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint( triggerHandler, savepointHandler)) { + RestClusterClient restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort()); - JobID id = new JobID(); - { - CompletableFuture savepointPathFuture = restClusterClient.triggerSavepoint(id, null); - String savepointPath = savepointPathFuture.get(); - assertEquals(savepointLocationDefaultDir, savepointPath); - } + try { + JobID id = new JobID(); + { + CompletableFuture savepointPathFuture = restClusterClient.triggerSavepoint(id, null); + String savepointPath = savepointPathFuture.get(); + assertEquals(savepointLocationDefaultDir, savepointPath); + } - { - CompletableFuture savepointPathFuture = restClusterClient.triggerSavepoint(id, targetSavepointDirectory); - String savepointPath = savepointPathFuture.get(); - assertEquals(savepointLocationRequestedDir, savepointPath); - } + { + CompletableFuture savepointPathFuture = restClusterClient.triggerSavepoint(id, targetSavepointDirectory); + String savepointPath = savepointPathFuture.get(); + assertEquals(savepointLocationRequestedDir, savepointPath); + } + + { + try { + restClusterClient.triggerSavepoint(id, null).get(); + fail("Expected exception not thrown."); + } catch (ExecutionException e) { + final Throwable cause = e.getCause(); + assertThat(cause, instanceOf(SerializedThrowable.class)); + assertThat(((SerializedThrowable) cause) + .deserializeError(ClassLoader.getSystemClassLoader()) + .getMessage(), equalTo("expected")); + } + } - { try { - restClusterClient.triggerSavepoint(id, null).get(); + restClusterClient.triggerSavepoint(new JobID(), null).get(); fail("Expected exception not thrown."); - } catch (ExecutionException e) { - final Throwable cause = e.getCause(); - assertThat(cause, instanceOf(SerializedThrowable.class)); - assertThat(((SerializedThrowable) cause) - .deserializeError(ClassLoader.getSystemClassLoader()) - .getMessage(), equalTo("expected")); + } catch (final ExecutionException e) { + assertTrue( + "RestClientException not in causal chain", + ExceptionUtils.findThrowable(e, RestClientException.class).isPresent()); } - } - - try { - restClusterClient.triggerSavepoint(new JobID(), null).get(); - fail("Expected exception not thrown."); - } catch (final ExecutionException e) { - assertTrue( - "RestClientException not in causal chain", - ExceptionUtils.findThrowable(e, RestClientException.class).isPresent()); + } finally { + restClusterClient.shutdown(); } } } @@ -540,32 +571,38 @@ public void testDisposeSavepoint() throws Exception { OptionalFailure.of(AsynchronousOperationInfo.completeExceptional(new SerializedThrowable(testException))), OptionalFailure.ofFailure(testException)); - try (TestRestServerEndpoint ignored = createRestServerEndpoint( + try (TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint( testSavepointDisposalStatusHandler, testSavepointDisposalTriggerHandler)) { - { - final CompletableFuture disposeSavepointFuture = restClusterClient.disposeSavepoint(savepointPath); - assertThat(disposeSavepointFuture.get(), is(Acknowledge.get())); - } + RestClusterClient restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort()); + + try { + { + final CompletableFuture disposeSavepointFuture = restClusterClient.disposeSavepoint(savepointPath); + assertThat(disposeSavepointFuture.get(), is(Acknowledge.get())); + } - { - final CompletableFuture disposeSavepointFuture = restClusterClient.disposeSavepoint(savepointPath); + { + final CompletableFuture disposeSavepointFuture = restClusterClient.disposeSavepoint(savepointPath); - try { - disposeSavepointFuture.get(); - fail("Expected an exception"); - } catch (ExecutionException ee) { - assertThat(ExceptionUtils.findThrowableWithMessage(ee, exceptionMessage).isPresent(), is(true)); + try { + disposeSavepointFuture.get(); + fail("Expected an exception"); + } catch (ExecutionException ee) { + assertThat(ExceptionUtils.findThrowableWithMessage(ee, exceptionMessage).isPresent(), is(true)); + } } - } - { - try { - restClusterClient.disposeSavepoint(savepointPath).get(); - fail("Expected an exception."); - } catch (ExecutionException ee) { - assertThat(ExceptionUtils.findThrowable(ee, RestClientException.class).isPresent(), is(true)); + { + try { + restClusterClient.disposeSavepoint(savepointPath).get(); + fail("Expected an exception."); + } catch (ExecutionException ee) { + assertThat(ExceptionUtils.findThrowable(ee, RestClientException.class).isPresent(), is(true)); + } } + } finally { + restClusterClient.shutdown(); } } } @@ -626,14 +663,18 @@ protected CompletableFuture restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort()); + + try { CompletableFuture> jobDetailsFuture = restClusterClient.listJobs(); Collection jobDetails = jobDetailsFuture.get(); Iterator jobDetailsIterator = jobDetails.iterator(); JobStatusMessage job1 = jobDetailsIterator.next(); JobStatusMessage job2 = jobDetailsIterator.next(); Assert.assertNotEquals("The job status should not be equal.", job1.getJobState(), job2.getJobState()); + } finally { + restClusterClient.shutdown(); } } } @@ -642,17 +683,22 @@ public void testListJobs() throws Exception { public void testGetAccumulators() throws Exception { TestAccumulatorHandler accumulatorHandler = new TestAccumulatorHandler(); - try (TestRestServerEndpoint ignored = createRestServerEndpoint(accumulatorHandler)){ + try (TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint(accumulatorHandler)){ + RestClusterClient restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort()); - JobID id = new JobID(); + try { + JobID id = new JobID(); - { - Map> accumulators = restClusterClient.getAccumulators(id); - assertNotNull(accumulators); - assertEquals(1, accumulators.size()); + { + Map> accumulators = restClusterClient.getAccumulators(id); + assertNotNull(accumulators); + assertEquals(1, accumulators.size()); - assertEquals(true, accumulators.containsKey("testKey")); - assertEquals("testValue", accumulators.get("testKey").get().toString()); + assertEquals(true, accumulators.containsKey("testKey")); + assertEquals("testValue", accumulators.get("testKey").get().toString()); + } + } finally { + restClusterClient.shutdown(); } } } @@ -698,11 +744,17 @@ public void testRetriableSendOperationIfConnectionErrorOrServiceUnavailable() th CompletableFuture.completedFuture(EmptyResponseBody.getInstance())); try (final TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint(pingRestHandler)) { - final AtomicBoolean firstPollFailed = new AtomicBoolean(); - failHttpRequest = (messageHeaders, messageParameters, requestBody) -> - messageHeaders instanceof PingRestHandlerHeaders && !firstPollFailed.getAndSet(true); + RestClusterClient restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort()); - restClusterClient.sendRequest(PingRestHandlerHeaders.INSTANCE).get(); + try { + final AtomicBoolean firstPollFailed = new AtomicBoolean(); + failHttpRequest = (messageHeaders, messageParameters, requestBody) -> + messageHeaders instanceof PingRestHandlerHeaders && !firstPollFailed.getAndSet(true); + + restClusterClient.sendRequest(PingRestHandlerHeaders.INSTANCE).get(); + } finally { + restClusterClient.shutdown(); + } } } @@ -716,11 +768,15 @@ public void testSendIsNotRetriableIfHttpNotFound() throws Exception { FutureUtils.completedExceptionally(new RestHandlerException(exceptionMessage, HttpResponseStatus.NOT_FOUND))); try (final TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint(pingRestHandler)) { + RestClusterClient restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort()); + try { restClusterClient.sendRequest(PingRestHandlerHeaders.INSTANCE).get(); fail("The rest request should have failed."); } catch (Exception e) { assertThat(ExceptionUtils.findThrowableWithMessage(e, exceptionMessage).isPresent(), is(true)); + } finally { + restClusterClient.shutdown(); } } } From 7ce6375cddbb2e2239b909ef5d02fc4c410b3bab Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 2 Jan 2019 12:04:06 +0100 Subject: [PATCH 009/147] [hotfix][tests] Remove mocks from RestClusterClientTest --- .../client/program/rest/RestClusterClientTest.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java index bbacd842dfce09..bc0432ea4ee263 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java @@ -32,7 +32,6 @@ import org.apache.flink.runtime.client.JobStatusMessage; import org.apache.flink.runtime.clusterframework.ApplicationStatus; import org.apache.flink.runtime.concurrent.FutureUtils; -import org.apache.flink.runtime.dispatcher.Dispatcher; import org.apache.flink.runtime.dispatcher.DispatcherGateway; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobStatus; @@ -88,6 +87,7 @@ import org.apache.flink.runtime.rest.util.RestClientException; import org.apache.flink.runtime.rpc.RpcUtils; import org.apache.flink.runtime.util.ExecutorThreadFactory; +import org.apache.flink.runtime.webmonitor.TestingDispatcherGateway; import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; import org.apache.flink.util.ConfigurationException; import org.apache.flink.util.ExceptionUtils; @@ -106,8 +106,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; import javax.annotation.Nonnull; @@ -151,10 +149,8 @@ */ public class RestClusterClientTest extends TestLogger { - @Mock - private Dispatcher mockRestfulGateway; + private final DispatcherGateway mockRestfulGateway = new TestingDispatcherGateway.Builder().build(); - @Mock private GatewayRetriever mockGatewayRetriever; private RestServerEndpointConfiguration restServerEndpointConfiguration; @@ -180,8 +176,6 @@ public class RestClusterClientTest extends TestLogger { @Before public void setUp() throws Exception { - MockitoAnnotations.initMocks(this); - restServerEndpointConfiguration = RestServerEndpointConfiguration.fromConfiguration(restConfig); mockGatewayRetriever = () -> CompletableFuture.completedFuture(mockRestfulGateway); From 9f99d4b366a5658f2d1c4e52d5170b3fd69f9232 Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 29 Nov 2018 12:03:05 +0100 Subject: [PATCH 010/147] [FLINK-11023] Add LICENSE & NOTICE files for flink-connector-cassandra --- .../src/main/resources/META-INF/NOTICE | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 flink-connectors/flink-connector-cassandra/src/main/resources/META-INF/NOTICE diff --git a/flink-connectors/flink-connector-cassandra/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-connector-cassandra/src/main/resources/META-INF/NOTICE new file mode 100644 index 00000000000000..a0449ff300703f --- /dev/null +++ b/flink-connectors/flink-connector-cassandra/src/main/resources/META-INF/NOTICE @@ -0,0 +1,16 @@ +flink-connector-cassandra +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.datastax.cassandra:cassandra-driver-core:3.0.0 +- com.datastax.cassandra:cassandra-driver-mapping:3.0.0 +- com.google.guava:guava:18.0 +- io.netty:netty-handler:4.0.33.Final +- io.netty:netty-buffer:4.0.33.Final +- io.netty:netty-common:4.0.33.Final +- io.netty:netty-transport:4.0.33.Final +- io.netty:netty-codec:4.0.33.Final From 842e424bad25241090f1bf01c1e261bc906d2229 Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 29 Nov 2018 12:03:31 +0100 Subject: [PATCH 011/147] [FLINK-11023] Add LICENSE & NOTICE files for flink-connector-elasticsearch --- .../src/main/resources/META-INF/NOTICE | 133 ++++-------------- .../resources/META-INF/licenses/LICENSE.antlr | 29 ++++ 2 files changed, 55 insertions(+), 107 deletions(-) create mode 100644 flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/licenses/LICENSE.antlr diff --git a/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/NOTICE index f588e0e873c141..a58d21ae6c6b0a 100644 --- a/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/NOTICE +++ b/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/NOTICE @@ -1,109 +1,28 @@ -This project includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -------------------------------------------------------------- - -This project bundles the following dependencies under -the Apache Software License 2.0 - - - org.apache.lucene : lucene-core version 4.10.4 - - org.apache.lucene : lucene-analyzers-common version 4.10.4 - - org.apache.lucene : lucene-grouping version 4.10.4 - - org.apache.lucene : lucene-highlighter version 4.10.4 - - org.apache.lucene : lucene-join version 4.10.4 - - org.apache.lucene : lucene-memory version 4.10.4 - - org.apache.lucene : lucene-misc version 4.10.4 - - org.apache.lucene : lucene-queries version 4.10.4 - - org.apache.lucene : lucene-queryparser version 4.10.4 - - org.apache.lucene : lucene-sandbox version 4.10.4 - - org.apache.lucene : lucene-spatial version 4.10.4 - - org.apache.lucene : lucene-suggest version 4.10.4 - - com.spatial4j : spatial4j version 0.4.1 - - com.fasterxml.jackson.core : jackson-core version 2.5.3 - - com.fasterxml.jackson.dataformat : jackson-dataformat-smile version 2.5.3 - - com.fasterxml.jackson.dataformat : jackson-dataformat-yaml version 2.5.3 - - com.fasterxml.jackson.dataformat : jackson-dataformat-cbor version 2.5.3 - - org.joda : joda-convert (copied classes) - -=================================== - Notice for Yaml -=================================== - -This project bundles yaml (v. 1.12) under the Creative Commons License (CC-BY 2.0). - -Original project website: http://www.yaml.de - -Copyright (c) 2005-2013, Dirk Jesse - -YAML under Creative Commons License (CC-BY 2.0) -=============================================== - -The YAML framework is published under the Creative Commons Attribution 2.0 License (CC-BY 2.0), which permits -both private and commercial use (http://creativecommons.org/licenses/by/2.0/). - -Condition: For the free use of the YAML framework, a backlink to the YAML homepage (http://www.yaml.de) in a -suitable place (e.g.: footer of the website or in the imprint) is required. - -=================================== - Notice for Tartarus -=================================== - -This project bundles tartarus under the MIT License. +flink-connector-elasticsearch +Copyright 2014-2018 The Apache Software Foundation -Original source repository: https://github.com/sergiooramas/tartarus - -Copyright (c) 2017 Sergio Oramas and Oriol Nieto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -=================================== - Notice for Antlr -=================================== - -This project bundles antlr-runtime (v. 3.5) under the BSD 2-Clause License - -Copyright (c) 2010 Terence Parr - ------------------------------------------------------------------------------ -** Beginning of "BSD 2-Clause License" text. ** - - Copyright (c) 2010 Terence Parr - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - THE POSSIBILITY OF SUCH DAMAGE. +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.spatial4j:spatial4j:0.4.1 +- org.apache.lucene:lucene-analyzers-common:4.10.4 +- org.apache.lucene:lucene-core:4.10.4 +- org.apache.lucene:lucene-grouping:4.10.4 +- org.apache.lucene:lucene-highlighter:4.10.4 +- org.apache.lucene:lucene-join:4.10.4 +- org.apache.lucene:lucene-memory:4.10.4 +- org.apache.lucene:lucene-misc:4.10.4 +- org.apache.lucene:lucene-queries:4.10.4 +- org.apache.lucene:lucene-queryparser:4.10.4 +- org.apache.lucene:lucene-sandbox:4.10.4 +- org.apache.lucene:lucene-spatial:4.10.4 +- org.apache.lucene:lucene-suggest:4.10.4 +- org.elasticsearch:elasticsearch:1.7.1 +- org.yaml:snakeyaml:1.12 + +This project bundles the following dependencies under the BSD License. +See bundled licenses for details. + +- org.antlr:antlr-runtime:jar:3.5 diff --git a/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/licenses/LICENSE.antlr b/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/licenses/LICENSE.antlr new file mode 100644 index 00000000000000..6d149604e134b0 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch/src/main/resources/META-INF/licenses/LICENSE.antlr @@ -0,0 +1,29 @@ +Copyright (c) 2010 Terence Parr + +----------------------------------------------------------------------------- +** Beginning of "BSD 2-Clause License" text. ** + + Copyright (c) 2010 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. From da57ce78a7aa66ba8446266f3938111a9ff4c411 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 11 Dec 2018 13:25:24 +0100 Subject: [PATCH 012/147] [hotfix][ES2] Set findbugs to provided --- flink-connectors/flink-connector-elasticsearch2/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flink-connectors/flink-connector-elasticsearch2/pom.xml b/flink-connectors/flink-connector-elasticsearch2/pom.xml index f15a3548c0dee5..756abd4a0bdfae 100644 --- a/flink-connectors/flink-connector-elasticsearch2/pom.xml +++ b/flink-connectors/flink-connector-elasticsearch2/pom.xml @@ -44,6 +44,12 @@ under the License. + + com.google.code.findbugs + jsr305 + provided + + org.apache.flink flink-streaming-java_${scala.binary.version} From 4d1fd4477d006f347fb57b9f08234c94064a1617 Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 29 Nov 2018 12:03:54 +0100 Subject: [PATCH 013/147] [FLINK-11023] Add LICENSE & NOTICE files for flink-connector-elasticsearch2 +es2 --- .../flink-connector-elasticsearch2/pom.xml | 18 ++ .../src/main/resources/META-INF/NOTICE | 212 +++++++----------- .../META-INF/licenses/LICENSE.base64 | 26 +++ .../META-INF/licenses/LICENSE.hdrhistogram | 125 +++++++++++ .../META-INF/licenses/LICENSE.jsr166e | 121 ++++++++++ .../META-INF/licenses/LICENSE.jsr166y | 26 +++ .../resources/META-INF/licenses/LICENSE.jzlib | 26 +++ .../META-INF/licenses/LICENSE.webbit | 38 ++++ 8 files changed, 461 insertions(+), 131 deletions(-) create mode 100644 flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.base64 create mode 100644 flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram create mode 100644 flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166e create mode 100644 flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166y create mode 100644 flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jzlib create mode 100644 flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.webbit diff --git a/flink-connectors/flink-connector-elasticsearch2/pom.xml b/flink-connectors/flink-connector-elasticsearch2/pom.xml index 756abd4a0bdfae..de4339653df0c1 100644 --- a/flink-connectors/flink-connector-elasticsearch2/pom.xml +++ b/flink-connectors/flink-connector-elasticsearch2/pom.xml @@ -200,6 +200,24 @@ under the License. META-INF/maven/org.y*/** + + + *:* + + META-INF/LICENSE.txt + + + + io.netty:netty + + + META-INF/license + + META-INF/NOTICE.txt + + diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/NOTICE index c3d95ca99c0162..4288bc1b481748 100644 --- a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/NOTICE +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/NOTICE @@ -1,133 +1,83 @@ -This project includes software developed at +flink-connector-elasticsearch2 +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at The Apache Software Foundation (http://www.apache.org/). -------------------------------------------------------------- - -This project bundles the following dependencies under -the Apache Software License 2.0 - - - org.apache.lucene : lucene-core version 5.5.0 - - org.apache.lucene : lucene-analyzers-common version 5.5.0 - - org.apache.lucene : lucene-backward-codecs version 5.5.0 - - org.apache.lucene : lucene-grouping version 5.5.0 - - org.apache.lucene : lucene-highlighter version 5.5.0 - - org.apache.lucene : lucene-join version 5.5.0 - - org.apache.lucene : lucene-memory version 5.5.0 - - org.apache.lucene : lucene-misc version 5.5.0 - - org.apache.lucene : lucene-queries version 5.5.0 - - org.apache.lucene : lucene-queryparser version 5.5.0 - - org.apache.lucene : lucene-sandbox version 5.5.0 - - org.apache.lucene : lucene-spatial version 5.5.0 - - org.apache.lucene : lucene-spatial-extras version 5.5.0 - - org.apache.lucene : lucene-spatial3d version 5.5.0 - - org.apache.lucene : lucene-suggest version 5.5.0 - - com.carrotsearch : hppc version 0.7.1 - - com.google.guava : guava version 18.0 - - com.google.code.findbugs : jsr305 version 1.3.9 - - com.ning : compress-lzf version 1.0.2 - - com.spatial4j : spatial4j version 0.5 - - com.twitter : chill-java version 0.7.4 - - com.fasterxml.jackson.core : jackson-core version 2.6.6 - - com.fasterxml.jackson.dataformat : jackson-dataformat-smile version 2.6.6 - - com.fasterxml.jackson.dataformat : jackson-dataformat-yaml version 2.6.6 - - com.fasterxml.jackson.dataformat : jackson-dataformat-cbor version 2.6.6 - - com.tdunning : t-digest version 3.0 - - io.netty : netty version 3.10.5.Final - - joda-time : joda-time version 2.5 - -=================================== - Notice for Yaml -=================================== - -This project bundles yaml (v. 1.15) under the Creative Commons License (CC-BY 2.0). - -Original project website: http://www.yaml.de - -Copyright (c) 2005-2013, Dirk Jesse - -YAML under Creative Commons License (CC-BY 2.0) -=============================================== - -The YAML framework is published under the Creative Commons Attribution 2.0 License (CC-BY 2.0), which permits -both private and commercial use (http://creativecommons.org/licenses/by/2.0/). - -Condition: For the free use of the YAML framework, a backlink to the YAML homepage (http://www.yaml.de) in a -suitable place (e.g.: footer of the website or in the imprint) is required. - -=================================== - Notice for Tartarus -=================================== - -This project bundles tartarus under the MIT License. - -Original source repository: https://github.com/sergiooramas/tartarus - -Copyright (c) 2017 Sergio Oramas and Oriol Nieto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -=================================== - Notice for HdrHistogram -=================================== - -This project bundles HdrHistogram (v. 2.1.9) under the BSD 2-Clause License - -Original source repository: https://github.com/HdrHistogram/HdrHistogram - -The code in this repository code was Written by Gil Tene, Michael Barker, -and Matt Warren, and released to the public domain, as explained at -http://creativecommons.org/publicdomain/zero/1.0/ - -For users of this code who wish to consume it under the "BSD" license -rather than under the public domain or CC0 contribution text mentioned -above, the code found under this directory is *also* provided under the -following license (commonly referred to as the BSD 2-Clause License). This -license does not detract from the above stated release of the code into -the public domain, and simply represents an additional license granted by -the Author. - ------------------------------------------------------------------------------ -** Beginning of "BSD 2-Clause License" text. ** - - Copyright (c) 2012, 2013, 2014, 2015, 2016 Gil Tene - Copyright (c) 2014 Michael Barker - Copyright (c) 2014 Matt Warren - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - THE POSSIBILITY OF SUCH DAMAGE. +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.carrotsearch:hppc:0.7.1 +- com.fasterxml.jackson.core:jackson-core:2.6.6 +- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.6 +- com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.6.6 +- com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.6.6 +- com.google.guava:guava:18.0 +- com.ning:compress-lzf:1.0.2 +- com.spatial4j:spatial4j:0.5 +- com.tdunning:t-digest:3.0 +- commons-cli:commons-cli:1.3.1 +- io.netty:netty:3.10.5.Final +- joda-time:joda-time:2.5 +- org.apache.lucene:lucene-analyzers-common:5.5.0 +- org.apache.lucene:lucene-backward-codecs:5.5.0 +- org.apache.lucene:lucene-core:5.5.0 +- org.apache.lucene:lucene-grouping:5.5.0 +- org.apache.lucene:lucene-highlighter:5.5.0 +- org.apache.lucene:lucene-join:5.5.0 +- org.apache.lucene:lucene-memory:5.5.0 +- org.apache.lucene:lucene-misc:5.5.0 +- org.apache.lucene:lucene-queries:5.5.0 +- org.apache.lucene:lucene-queryparser:5.5.0 +- org.apache.lucene:lucene-sandbox:5.5.0 +- org.apache.lucene:lucene-spatial:5.5.0 +- org.apache.lucene:lucene-spatial3d:5.5.0 +- org.apache.lucene:lucene-suggest:5.5.0 +- org.elasticsearch:elasticsearch:2.3.5 +- org.elasticsearch:securesm:1.0 +- org.joda:joda-convert:1.7 +- org.yaml:snakeyaml:1.15 + +This project bundles the following dependencies under the Creative Commons CC0 1.0 Universal Public Domain Dedication License (http://creativecommons.org/publicdomain/zero/1.0/) +See bundled license files for details. + +- org.hdrhistogram:HdrHistogram:2.1.6 + +This project bundles the following dependency under the CC0 1.0. +See bundled licenses for details. + +- com.twitter:jsr166e:1.1.0 + +This project bundles io.netty:netty:3.10.5.Final from which it inherits the following notices: + +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * licenses/LICENSE.jsr166y (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.base64 (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.jzlib (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * licenses/LICENSE.webbit (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.base64 b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.base64 new file mode 100644 index 00000000000000..31ebc840539c1a --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.base64 @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuate of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram new file mode 100644 index 00000000000000..09c38ea51608e0 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram @@ -0,0 +1,125 @@ +The code was Written by Gil Tene, Michael Barker, and Matt Warren, +and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166e b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166e new file mode 100644 index 00000000000000..0e259d42c99674 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166e @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166y b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166y new file mode 100644 index 00000000000000..b1c292b54cb2b2 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jsr166y @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuity of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jzlib b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jzlib new file mode 100644 index 00000000000000..29ad562af0a5ed --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.jzlib @@ -0,0 +1,26 @@ +Copyright (c) 2000,2001,2002,2003,2004 ymnk, JCraft,Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, +INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.webbit b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.webbit new file mode 100644 index 00000000000000..ec5f348998bd44 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch2/src/main/resources/META-INF/licenses/LICENSE.webbit @@ -0,0 +1,38 @@ +(BSD License: http://www.opensource.org/licenses/bsd-license) + +Copyright (c) 2011, Joe Walnes, Aslak Hellesøy and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the Webbit nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + From 6e774a2dcbf76a03910eba65fe717b60d36b97a7 Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 29 Nov 2018 12:04:24 +0100 Subject: [PATCH 014/147] [FLINK-11023] Add LICENSE & NOTICE files for flink-connector-elasticsearch5 +es5 --- .../flink-connector-elasticsearch5/pom.xml | 18 ++ .../src/main/resources/META-INF/NOTICE | 297 ++++++------------ .../META-INF/licenses/LICENSE.base64 | 26 ++ .../META-INF/licenses/LICENSE.hdrhistogram | 125 ++++++++ .../META-INF/licenses/LICENSE.joptsimple | 22 ++ .../META-INF/licenses/LICENSE.jsr166y | 26 ++ .../resources/META-INF/licenses/LICENSE.jzlib | 26 ++ .../META-INF/licenses/LICENSE.webbit | 38 +++ 8 files changed, 381 insertions(+), 197 deletions(-) create mode 100644 flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.base64 create mode 100644 flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram create mode 100644 flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.joptsimple create mode 100644 flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jsr166y create mode 100644 flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jzlib create mode 100644 flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.webbit diff --git a/flink-connectors/flink-connector-elasticsearch5/pom.xml b/flink-connectors/flink-connector-elasticsearch5/pom.xml index a4c7bfcc9caf78..3598e683cc8877 100644 --- a/flink-connectors/flink-connector-elasticsearch5/pom.xml +++ b/flink-connectors/flink-connector-elasticsearch5/pom.xml @@ -231,6 +231,13 @@ under the License. + + + *:* + + META-INF/LICENSE.txt + + * @@ -263,6 +270,17 @@ under the License. META-INF/log4j-provider.properties + + io.netty:netty + + + META-INF/license + + META-INF/NOTICE.txt + + diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/NOTICE index 19e96faa460031..9fe894149aed06 100644 --- a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/NOTICE +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/NOTICE @@ -1,201 +1,104 @@ -This project includes software developed at -The Apache Software Foundation (http://www.apache.org/). +flink-connector-elasticsearch5 +Copyright 2014-2018 The Apache Software Foundation -------------------------------------------------------------- - -This project bundles the following dependencies under -the Apache Software License 2.0 - - - org.apache.lucene : lucene-core version 6.3.0 - - org.apache.lucene : lucene-analyzers-common version 6.3.0 - - org.apache.lucene : lucene-backward-codecs version 6.3.0 - - org.apache.lucene : lucene-grouping version 6.3.0 - - org.apache.lucene : lucene-highlighter version 6.3.0 - - org.apache.lucene : lucene-join version 6.3.0 - - org.apache.lucene : lucene-memory version 6.3.0 - - org.apache.lucene : lucene-misc version 6.3.0 - - org.apache.lucene : lucene-queries version 6.3.0 - - org.apache.lucene : lucene-queryparser version 6.3.0 - - org.apache.lucene : lucene-sandbox version 6.3.0 - - org.apache.lucene : lucene-spatial version 6.3.0 - - org.apache.lucene : lucene-spatial-extras version 6.3.0 - - org.apache.lucene : lucene-spatial3d version 6.3.0 - - org.apache.lucene : lucene-suggest version 6.3.0 - - org.apache.httpcomponents : httpclient version 4.5.3 - - org.apache.httpcomponents : httpcore version 4.4.6 - - org.apache.httpcomponents : httpasynclcient version 4.1.2 - - org.apache.httpcomponents : httpcore-nio version 4.4.5 - - com.carrotsearch : hppc version 0.7.1 - - com.fasterxml.jackson.core : jackson-core version 2.8.1 - - com.fasterxml.jackson.dataformat : jackson-dataformat-smile version 2.8.1 - - com.fasterxml.jackson.dataformat : jackson-dataformat-yaml version 2.8.1 - - com.fasterxml.jackson.dataformat : jackson-dataformat-cbor version 2.8.1 - - com.tdunning : t-digest version 3.0 - - io.netty : netty version 3.10.6.Final - - io.netty : netty-buffer version 4.1.6.Final - - io.netty : netty-codec version 4.1.6.Final - - io.netty : netty-codec-http version 4.1.6.Final - - io.netty : netty-common version 4.1.6.Final - - io.netty : netty-handler version 4.1.6.Final - - io.netty : netty-resolver version 4.1.6.Final - - io.netty : netty-transport version 4.1.6.Final - - org.jboss.netty : netty version 3.2.0.Final - - joda-time : joda-time version 2.5 - - com.github.spullara.mustache.java : compiler version 0.9.3 - -=================================== - Notice for Yaml -=================================== - -This project bundles yaml (v. 1.15) under the Creative Commons License (CC-BY 2.0). - -Original project website: http://www.yaml.de - -Copyright (c) 2005-2013, Dirk Jesse - -YAML under Creative Commons License (CC-BY 2.0) -=============================================== - -The YAML framework is published under the Creative Commons Attribution 2.0 License (CC-BY 2.0), which permits -both private and commercial use (http://creativecommons.org/licenses/by/2.0/). - -Condition: For the free use of the YAML framework, a backlink to the YAML homepage (http://www.yaml.de) in a -suitable place (e.g.: footer of the website or in the imprint) is required. - -=================================== - Notice for Tartarus -=================================== - -This project bundles tartarus under the MIT License. - -Original source repository: https://github.com/sergiooramas/tartarus - -Copyright (c) 2017 Sergio Oramas and Oriol Nieto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -=================================== - Notice for joptsimple -=================================== - -This project bundles joptsimple under the MIT License. - -Original source repository: https://github.com/jopt-simple/jopt-simple - -Copyright (c) 2004-2016 Paul R. Holser, Jr. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -=================================== - Notice for scopt -=================================== - -This project bundles scopt (v. 3.5.0) underr the MIT License. - -Original source repository: https://github.com/scopt/scopt - -scopt - Copyright (c) scopt contributors - -See source files for details. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -``Software''), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.carrotsearch:hppc:0.7.1 +- com.fasterxml.jackson.core:jackson-core:2.8.1 +- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.1 +- com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.8.1 +- com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.8.1 +- com.github.spullara.mustache.java:compiler:0.9.3 +- com.tdunning:t-digest:3.0 +- commons-codec:commons-codec:1.10 +- commons-logging:commons-logging:1.1.3 +- io.netty:netty:3.10.6.Final +- io.netty:netty-buffer:4.1.6.Final +- io.netty:netty-codec:4.1.6.Final +- io.netty:netty-codec-http:4.1.6.Final +- io.netty:netty-common:4.1.6.Final +- io.netty:netty-handler:4.1.6.Final +- io.netty:netty-resolver:4.1.6.Final +- io.netty:netty-transport:4.1.6.Final +- joda.time:joda-time:2.5 +- net.java.dev.jna:jna:4.2.2 +- org.apache.httpcomponents:httpasynclcient:4.1.2 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 +- org.apache.httpcomponents:httpcore-nio:4.4.5 +- org.apache.logging.log4j:log4j-api:2.7 +- org.apache.logging.log4j:log4j-to-slf4j:2.7 +- org.apache.lucene:lucene-analyzers-common:6.3.0 +- org.apache.lucene:lucene-backward-codecs:6.3.0 +- org.apache.lucene:lucene-core:6.3.0 +- org.apache.lucene:lucene-grouping:6.3.0 +- org.apache.lucene:lucene-highlighter:6.3.0 +- org.apache.lucene:lucene-join:6.3.0 +- org.apache.lucene:lucene-memory:6.3.0 +- org.apache.lucene:lucene-misc:6.3.0 +- org.apache.lucene:lucene-queries:6.3.0 +- org.apache.lucene:lucene-queryparser:6.3.0 +- org.apache.lucene:lucene-sandbox:6.3.0 +- org.apache.lucene:lucene-spatial:6.3.0 +- org.apache.lucene:lucene-spatial-extras:6.3.0 +- org.apache.lucene:lucene-spatial3d:6.3.0 +- org.apache.lucene:lucene-suggest:6.3.0 +- org.elasticsearch:elasticsearch:5.1.2 +- org.elasticsearch:securesm:1.1 +- org.elasticsearch.client:rest:5.1.2 +- org.elasticsearch.client:transport:5.1.2 +- org.elasticsearch.plugin:lang-mustache-client:5.1.2 +- org.elasticsearch.plugin:percolator-client:5.1.2 +- org.elasticsearch.plugin:reindex-client:5.1.2 +- org.elasticsearch.plugin:transport-netty3-client:5.1.2 +- org.elasticsearch.plugin:transport-netty4-client:5.1.2 +- org.yaml:snakeyaml:1.15 + +This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT) + +- net.sf.jopt-simple:jopt-simple:5.0.2 + +This project bundles the following dependencies under the Creative Commons CC0 1.0 Universal Public Domain Dedication License (http://creativecommons.org/publicdomain/zero/1.0/) +See bundled license files for details. + +- org.hdrhistogram:HdrHistogram:2.1.6 + +This project bundles io.netty:netty:3.10.6.Final from which it inherits the following notices: + +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * licenses/LICENSE.jsr166y (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.base64 (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * licenses/LICENSE.jzlib (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * licenses/LICENSE.webbit (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit -THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -=================================== - Notice for HdrHistogram -=================================== - -This project bundles HdrHistogram (v. 2.1.9) under the BSD 2-Clause License - -Original source repository: https://github.com/HdrHistogram/HdrHistogram - -The code in this repository code was Written by Gil Tene, Michael Barker, -and Matt Warren, and released to the public domain, as explained at -http://creativecommons.org/publicdomain/zero/1.0/ - -For users of this code who wish to consume it under the "BSD" license -rather than under the public domain or CC0 contribution text mentioned -above, the code found under this directory is *also* provided under the -following license (commonly referred to as the BSD 2-Clause License). This -license does not detract from the above stated release of the code into -the public domain, and simply represents an additional license granted by -the Author. - ------------------------------------------------------------------------------ -** Beginning of "BSD 2-Clause License" text. ** - - Copyright (c) 2012, 2013, 2014, 2015, 2016 Gil Tene - Copyright (c) 2014 Michael Barker - Copyright (c) 2014 Matt Warren - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.base64 b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.base64 new file mode 100644 index 00000000000000..31ebc840539c1a --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.base64 @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuate of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram new file mode 100644 index 00000000000000..09c38ea51608e0 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.hdrhistogram @@ -0,0 +1,125 @@ +The code was Written by Gil Tene, Michael Barker, and Matt Warren, +and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.joptsimple b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.joptsimple new file mode 100644 index 00000000000000..ef77fc95e107e4 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.joptsimple @@ -0,0 +1,22 @@ +The MIT License + +Copyright (c) 2004-2015 Paul R. Holser, Jr. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jsr166y b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jsr166y new file mode 100644 index 00000000000000..b1c292b54cb2b2 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jsr166y @@ -0,0 +1,26 @@ +The person or persons who have associated work with this document (the +"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of +his knowledge, the work of authorship identified is in the public domain of +the country from which the work is published, or (b) hereby dedicates whatever +copyright the dedicators holds in the work of authorship identified below (the +"Work") to the public domain. A certifier, moreover, dedicates any copyright +interest he may have in the associated work, and for these purposes, is +described as a "dedicator" below. + +A certifier has taken reasonable steps to verify the copyright status of this +work. Certifier recognizes that his good faith efforts may not shield him from +liability if in fact the work certified is not in the public domain. + +Dedicator makes this dedication for the benefit of the public at large and to +the detriment of the Dedicator's heirs and successors. Dedicator intends this +dedication to be an overt act of relinquishment in perpetuity of all present +and future rights under copyright law, whether vested or contingent, in the +Work. Dedicator understands that such relinquishment of all rights includes +the relinquishment of all rights to enforce (by lawsuit or otherwise) those +copyrights in the Work. + +Dedicator recognizes that, once placed in the public domain, the Work may be +freely reproduced, distributed, transmitted, used, modified, built upon, or +otherwise exploited by anyone for any purpose, commercial or non-commercial, +and in any way, including by methods that have not yet been invented or +conceived. diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jzlib b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jzlib new file mode 100644 index 00000000000000..29ad562af0a5ed --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.jzlib @@ -0,0 +1,26 @@ +Copyright (c) 2000,2001,2002,2003,2004 ymnk, JCraft,Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, +INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.webbit b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.webbit new file mode 100644 index 00000000000000..ec5f348998bd44 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch5/src/main/resources/META-INF/licenses/LICENSE.webbit @@ -0,0 +1,38 @@ +(BSD License: http://www.opensource.org/licenses/bsd-license) + +Copyright (c) 2011, Joe Walnes, Aslak Hellesøy and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the Webbit nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + From 89f8f473eeb9d19aedf37a5c5ca63e64e633d85a Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 29 Nov 2018 12:04:42 +0100 Subject: [PATCH 015/147] [FLINK-11023] Add LICENSE & NOTICE files for flink-connector-kinesis --- .../src/main/resources/META-INF/NOTICE | 26 +++++++++++++ .../META-INF/licenses/LICENSE.amazon | 39 +++++++++++++++++++ .../META-INF/licenses/LICENSE.protobuf | 32 +++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/NOTICE create mode 100644 flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.amazon create mode 100644 flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.protobuf diff --git a/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/NOTICE new file mode 100644 index 00000000000000..a7dcc98cbadd0f --- /dev/null +++ b/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/NOTICE @@ -0,0 +1,26 @@ +flink-connector-kinesis +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.amazonaws:aws-java-sdk-core:1.11.319 +- com.amazonaws:aws-java-sdk-kinesis:1.11.319 +- com.amazonaws:aws-java-sdk-sts:1.11.319 +- com.amazonaws:jmespath-java:1.11.319 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 + +This project bundles the following dependencies under the BSD license. +See bundled license files for details. + +- com.google.protobuf:protobuf-java:2.6.1 + +This project bundles the following dependencies under the Amazon Softward License. +See bundled license files for details. + +- com.amazonaws:amazon-kinesis-client:1.9.0 +- com.amazonaws:amazon-kinesis-producer:0.12.9 + diff --git a/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.amazon b/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.amazon new file mode 100644 index 00000000000000..e933f6953c1cac --- /dev/null +++ b/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.amazon @@ -0,0 +1,39 @@ +Amazon Software License + +This Amazon Software License (“License”) governs your use, reproduction, and distribution of the accompanying software as specified below. +1. Definitions + +“Licensor” means any person or entity that distributes its Work. + +“Software” means the original work of authorship made available under this License. + +“Work” means the Software and any additions to or derivative works of the Software that are made available under this License. + +The terms “reproduce,” “reproduction,” “derivative works,” and “distribution” have the meaning as provided under U.S. copyright law; provided, however, that for the purposes of this License, derivative works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work. + +Works, including the Software, are “made available” under this License by including in or with the Work either (a) a copyright notice referencing the applicability of this License to the Work, or (b) a copy of this License. +2. License Grants + +2.1 Copyright Grant. Subject to the terms and conditions of this License, each Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free, copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense and distribute its Work and any resulting derivative works in any form. + +2.2 Patent Grant. Subject to the terms and conditions of this License, each Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free patent license to make, have made, use, sell, offer for sale, import, and otherwise transfer its Work, in whole or in part. The foregoing license applies only to the patent claims licensable by Licensor that would be infringed by Licensor’s Work (or portion thereof) individually and excluding any combinations with any other materials or technology. +3. Limitations + +3.1 Redistribution. You may reproduce or distribute the Work only if (a) you do so under this License, (b) you include a complete copy of this License with your distribution, and (c) you retain without modification any copyright, patent, trademark, or attribution notices that are present in the Work. + +3.2 Derivative Works. You may specify that additional or different terms apply to the use, reproduction, and distribution of your derivative works of the Work (“Your Terms”) only if (a) Your Terms provide that the use limitation in Section 3.3 applies to your derivative works, and (b) you identify the specific derivative works that are subject to Your Terms. Notwithstanding Your Terms, this License (including the redistribution requirements in Section 3.1) will continue to apply to the Work itself. + +3.3 Use Limitation. The Work and any derivative works thereof only may be used or intended for use with the web services, computing platforms or applications provided by Amazon.com, Inc. or its affiliates, including Amazon Web Services, Inc. + +3.4 Patent Claims. If you bring or threaten to bring a patent claim against any Licensor (including any claim, cross-claim or counterclaim in a lawsuit) to enforce any patents that you allege are infringed by any Work, then your rights under this License from such Licensor (including the grants in Sections 2.1 and 2.2) will terminate immediately. + +3.5 Trademarks. This License does not grant any rights to use any Licensor’s or its affiliates’ names, logos, or trademarks, except as necessary to reproduce the notices described in this License. + +3.6 Termination. If you violate any term of this License, then your rights under this License (including the grants in Sections 2.1 and 2.2) will terminate immediately. +4. Disclaimer of Warranty. + +THE WORK IS PROVIDED “AS IS” WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF M ERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON-INFRINGEMENT. YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER THIS LICENSE. SOME STATES’ CONSUMER LAWS DO NOT ALLOW EXCLUSION OF AN IMPLIED WARRANTY, SO THIS DISCLAIMER MAY NOT APPLY TO YOU. +5. Limitation of Liability. + +EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, LOST PROFITS OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER COMM ERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + diff --git a/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.protobuf b/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.protobuf new file mode 100644 index 00000000000000..19b305b00060a7 --- /dev/null +++ b/flink-connectors/flink-connector-kinesis/src/main/resources/META-INF/licenses/LICENSE.protobuf @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. From aa92438c4aa6218c5d7e67083152733271697089 Mon Sep 17 00:00:00 2001 From: Chesnay Schepler Date: Thu, 3 Jan 2019 18:13:48 +0100 Subject: [PATCH 016/147] [FLINK-10761][metrics] Do not acquire lock for getAllVariables() --- .../metrics/groups/AbstractMetricGroup.java | 16 ++-- .../groups/AbstractMetricGroupTest.java | 90 +++++++++++++++++++ 2 files changed, 96 insertions(+), 10 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroup.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroup.java index 4400b14a10d693..fb321303222c72 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroup.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroup.java @@ -111,17 +111,13 @@ public AbstractMetricGroup(MetricRegistry registry, String[] scope, A parent) { } public Map getAllVariables() { - if (variables == null) { // avoid synchronization for common case - synchronized (this) { - if (variables == null) { - Map tmpVariables = new HashMap<>(); - putVariables(tmpVariables); - if (parent != null) { // not true for Job-/TaskManagerMetricGroup - tmpVariables.putAll(parent.getAllVariables()); - } - variables = tmpVariables; - } + if (variables == null) { + Map tmpVariables = new HashMap<>(); + putVariables(tmpVariables); + if (parent != null) { // not true for Job-/TaskManagerMetricGroup + tmpVariables.putAll(parent.getAllVariables()); } + variables = tmpVariables; } return variables; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java index f3f8b42b85154b..d750f63cfd8694 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java @@ -21,17 +21,22 @@ import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.MetricOptions; +import org.apache.flink.core.testutils.BlockerSync; import org.apache.flink.metrics.CharacterFilter; import org.apache.flink.metrics.Metric; import org.apache.flink.metrics.MetricGroup; import org.apache.flink.metrics.reporter.MetricReporter; +import org.apache.flink.runtime.metrics.MetricRegistry; import org.apache.flink.runtime.metrics.MetricRegistryConfiguration; import org.apache.flink.runtime.metrics.MetricRegistryImpl; import org.apache.flink.runtime.metrics.dump.QueryScopeInfo; +import org.apache.flink.runtime.metrics.scope.ScopeFormats; import org.apache.flink.runtime.metrics.util.TestReporter; import org.junit.Test; +import javax.annotation.Nullable; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -252,4 +257,89 @@ public void testScopeGenerationWithoutReporters() throws Exception { testRegistry.shutdown().get(); } } + + @Test + public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException { + final TestMetricRegistry registry = new TestMetricRegistry(); + + final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent"); + final MetricGroup child = parent.addGroup("child"); + + final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter")); + final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter")); + + final BlockerSync parentSync = new BlockerSync(); + final BlockerSync childSync = new BlockerSync(); + + try { + // start both threads and have them block in the registry, so they acquire the lock of their respective group + registry.setOnRegistrationAction(childSync::blockNonInterruptible); + childRegisteringThread.start(); + childSync.awaitBlocker(); + + registry.setOnRegistrationAction(parentSync::blockNonInterruptible); + parentRegisteringThread.start(); + parentSync.awaitBlocker(); + + // the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter + // the child thread continues execution and calls getAllVariables() + // in the past this would block indefinitely since the method acquires the locks of all parent groups + childSync.releaseBlocker(); + // wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent + childRegisteringThread.join(1000 * 10); + + parentSync.releaseBlocker(); + parentRegisteringThread.join(); + } finally { + parentSync.releaseBlocker(); + childSync.releaseBlocker(); + parentRegisteringThread.join(); + childRegisteringThread.join(); + } + } + + private static final class TestMetricRegistry implements MetricRegistry { + + private Runnable onRegistrationAction; + + void setOnRegistrationAction(Runnable onRegistrationAction) { + this.onRegistrationAction = onRegistrationAction; + } + + @Override + public char getDelimiter() { + return 0; + } + + @Override + public char getDelimiter(int index) { + return 0; + } + + @Override + public int getNumberReporters() { + return 0; + } + + @Override + public void register(Metric metric, String metricName, AbstractMetricGroup group) { + onRegistrationAction.run(); + group.getAllVariables(); + } + + @Override + public void unregister(Metric metric, String metricName, AbstractMetricGroup group) { + } + + @Override + public ScopeFormats getScopeFormats() { + return null; + } + + @Nullable + @Override + public String getMetricQueryServicePath() { + return null; + } + } } From 9e0ea45645b242c1b13f59c684fbe9d47eaeda06 Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 20 Dec 2018 12:48:33 +0100 Subject: [PATCH 017/147] [FLINK-11079][storm] Skip deployment of storm-examples --- flink-contrib/flink-storm-examples/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/flink-contrib/flink-storm-examples/pom.xml b/flink-contrib/flink-storm-examples/pom.xml index 42038cd1e11b79..47323c05b706ec 100644 --- a/flink-contrib/flink-storm-examples/pom.xml +++ b/flink-contrib/flink-storm-examples/pom.xml @@ -111,6 +111,14 @@ under the License. + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + org.apache.maven.plugins From 7a0a525504ec4b5e82bbdb3138f6ee4170b48350 Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 19 Dec 2018 13:38:25 +0100 Subject: [PATCH 018/147] [FLINK-11194][hbase] Use type instead of classifier --- flink-connectors/flink-hbase/pom.xml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/flink-connectors/flink-hbase/pom.xml b/flink-connectors/flink-hbase/pom.xml index 75b9470f45c00c..2ab0ec4c2aa6a4 100644 --- a/flink-connectors/flink-hbase/pom.xml +++ b/flink-connectors/flink-hbase/pom.xml @@ -49,19 +49,6 @@ under the License. 1 - - org.apache.maven.plugins - maven-shade-plugin - - - - - - shade-flink - none - - - @@ -255,7 +242,7 @@ under the License. org.apache.hbase hbase-server ${hbase.version} - tests + test-jar test From c28250b576fa8b37679a9303223e952161e70584 Mon Sep 17 00:00:00 2001 From: Chesnay Schepler Date: Thu, 3 Jan 2019 19:35:49 +0100 Subject: [PATCH 019/147] [FLINK-11023][ES] Add NOTICE file --- .../pom.xml | 1 + .../src/main/resources/META-INF/NOTICE | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 flink-connectors/flink-sql-connector-elasticsearch6/src/main/resources/META-INF/NOTICE diff --git a/flink-connectors/flink-sql-connector-elasticsearch6/pom.xml b/flink-connectors/flink-sql-connector-elasticsearch6/pom.xml index 85b73a9c837a58..7438566105c6b9 100644 --- a/flink-connectors/flink-sql-connector-elasticsearch6/pom.xml +++ b/flink-connectors/flink-sql-connector-elasticsearch6/pom.xml @@ -115,6 +115,7 @@ under the License. META-INF/services/com.fasterxml.** META-INF/services/org.apache.lucene.** META-INF/services/org.elasticsearch.** + META-INF/LICENSE.txt diff --git a/flink-connectors/flink-sql-connector-elasticsearch6/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-sql-connector-elasticsearch6/src/main/resources/META-INF/NOTICE new file mode 100644 index 00000000000000..9880ee926b3cd7 --- /dev/null +++ b/flink-connectors/flink-sql-connector-elasticsearch6/src/main/resources/META-INF/NOTICE @@ -0,0 +1,45 @@ +flink-sql-connector-elasticsearch6 +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.fasterxml.jackson.core:jackson-core:2.8.10 +- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.10 +- com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.8.10 +- com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.8.10 +- commons-codec:commons-codec:1.10 +- commons-logging:commons-logging:1.1.3 +- org.apache.httpcomponents:httpasyncclient:4.1.2 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 +- org.apache.httpcomponents:httpcore-nio:4.4.5 +- org.apache.logging.log4j:log4j-api:2.9.1 +- org.apache.logging.log4j:log4j-to-slf4j:2.9.1 +- org.apache.lucene:lucene-analyzers-common:7.3.1 +- org.apache.lucene:lucene-backward-codecs:7.3.1 +- org.apache.lucene:lucene-core:7.3.1 +- org.apache.lucene:lucene-grouping:7.3.1 +- org.apache.lucene:lucene-highlighter:7.3.1 +- org.apache.lucene:lucene-join:7.3.1 +- org.apache.lucene:lucene-memory:7.3.1 +- org.apache.lucene:lucene-misc:7.3.1 +- org.apache.lucene:lucene-queries:7.3.1 +- org.apache.lucene:lucene-queryparser:7.3.1 +- org.apache.lucene:lucene-sandbox:7.3.1 +- org.apache.lucene:lucene-spatial:7.3.1 +- org.apache.lucene:lucene-spatial-extras:7.3.1 +- org.apache.lucene:lucene-spatial3d:7.3.1 +- org.apache.lucene:lucene-suggest:7.3.1 +- org.elasticsearch:elasticsearch:6.3.1 +- org.elasticsearch:elasticsearch-cli:6.3.1 +- org.elasticsearch:elasticsearch-core:6.3.1 +- org.elasticsearch:elasticsearch-secure-sm:6.3.1 +- org.elasticsearch:elasticsearch-x-content:6.3.1 +- org.elasticsearch.client:elasticsearch-rest-client:6.3.1 +- org.elasticsearch.client:elasticsearch-rest-high-level-client:6.3.1 +- org.elasticsearch.plugin:aggs-matrix-stats-client:6.3.1 +- org.elasticsearch.plugin:parent-join-client:6.3.1 +- org.elasticsearch.plugin:rank-eval-client:6.3.1 From 96d6979c3981b54e5230fb164acd64dd4a52ec3a Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 4 Jan 2019 17:32:26 +0800 Subject: [PATCH 020/147] [hotfix][docs][kafka] Fix typo --- docs/dev/connectors/kafka.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/connectors/kafka.md b/docs/dev/connectors/kafka.md index 351a4dc2d4126f..b7f65b5cd1c986 100644 --- a/docs/dev/connectors/kafka.md +++ b/docs/dev/connectors/kafka.md @@ -91,7 +91,7 @@ For most users, the `FlinkKafkaConsumer08` (part of `flink-connector-kafka`) is The version of the client it uses may change between Flink releases. Modern Kafka clients are backwards compatible with broker versions 0.10.0 or later. However for Kafka 0.11.x and 0.10.x versions, we recommend using dedicated - flink-connector-kafka-0.11{{ site.scala_version_suffix }} and link-connector-kafka-0.10{{ site.scala_version_suffix }} respectively. + flink-connector-kafka-0.11{{ site.scala_version_suffix }} and flink-connector-kafka-0.10{{ site.scala_version_suffix }} respectively.

Attention: as of Flink 1.7 the universal Kafka connector is considered to be in a BETA status and might not be as stable as the 0.11 connector. From 04035ce8b7927136c7f1e43f0514a055ce45e56a Mon Sep 17 00:00:00 2001 From: EronWright Date: Sun, 30 Dec 2018 19:59:47 -0800 Subject: [PATCH 021/147] [FLINK-11234] [table] Fix ExternalTableCatalogBuilder unable to build a batch-only table - fix the logic in supportsBatch to properly declare a batch-only table - adjust CommonTestData to provide batch-only or streaming-only tables This closes #7386. --- .../flink/table/catalog/ExternalCatalogTable.scala | 4 ++-- .../flink/table/runtime/utils/CommonTestData.scala | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala index 45414ee3ff6e7b..ce57070ac98d7e 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala @@ -270,8 +270,8 @@ class ExternalCatalogTableBuilder(private val connectorDescriptor: ConnectorDesc * Explicitly declares this external table for supporting only batch environments. */ def supportsBatch(): ExternalCatalogTableBuilder = { - isBatch = false - isStreaming = true + isBatch = true + isStreaming = false this } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/CommonTestData.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/CommonTestData.scala index 64fcc8ac7c4801..1209595837bf3d 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/CommonTestData.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/CommonTestData.scala @@ -85,7 +85,9 @@ object CommonTestData { .withSchema(schemaDesc1) if (isStreaming) { - externalTableBuilder1.inAppendMode() + externalTableBuilder1.supportsStreaming().inAppendMode() + } else { + externalTableBuilder1.supportsBatch() } val csvRecord2 = Seq( @@ -126,7 +128,9 @@ object CommonTestData { .withSchema(schemaDesc2) if (isStreaming) { - externalTableBuilder2.inAppendMode() + externalTableBuilder2.supportsStreaming().inAppendMode() + } else { + externalTableBuilder2.supportsBatch() } val tempFilePath3 = writeToTempFile("", "csv-test3", "tmp") @@ -145,7 +149,9 @@ object CommonTestData { .withSchema(schemaDesc3) if (isStreaming) { - externalTableBuilder3.inAppendMode() + externalTableBuilder3.supportsStreaming().inAppendMode() + } else { + externalTableBuilder3.supportsBatch() } val catalog = new InMemoryExternalCatalog("test") From 71f85640a4157a03c2f6788b89f16a419fe26be2 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 18 Dec 2018 13:39:33 +0100 Subject: [PATCH 022/147] [hotfix][ES6][SQL] Remove unused version property --- flink-connectors/flink-sql-connector-elasticsearch6/pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/flink-connectors/flink-sql-connector-elasticsearch6/pom.xml b/flink-connectors/flink-sql-connector-elasticsearch6/pom.xml index 7438566105c6b9..95432f448bed67 100644 --- a/flink-connectors/flink-sql-connector-elasticsearch6/pom.xml +++ b/flink-connectors/flink-sql-connector-elasticsearch6/pom.xml @@ -35,11 +35,6 @@ under the License. jar - - - 6.3.1 - - org.apache.flink From b6237688b27ee2ca2cdda49820c5d2d541e12bbc Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 18 Dec 2018 13:40:03 +0100 Subject: [PATCH 023/147] [FLINK-11026][kafka][SQL] Rework kafka sql-client jar creation --- docs/dev/table/connect.md | 8 +- .../flink-connector-kafka-0.10/pom.xml | 53 ----------- .../flink-connector-kafka-0.11/pom.xml | 54 ------------ .../flink-connector-kafka-0.9/pom.xml | 50 ----------- .../flink-connector-kafka/pom.xml | 52 ----------- .../flink-sql-connector-kafka-0.10/pom.xml | 87 ++++++++++++++++++ .../flink-sql-connector-kafka-0.11/pom.xml | 88 +++++++++++++++++++ .../flink-sql-connector-kafka-0.9/pom.xml | 86 ++++++++++++++++++ .../flink-sql-connector-kafka/pom.xml | 87 ++++++++++++++++++ flink-connectors/pom.xml | 4 + .../flink-sql-client-test/pom.xml | 21 ++--- 11 files changed, 363 insertions(+), 227 deletions(-) create mode 100644 flink-connectors/flink-sql-connector-kafka-0.10/pom.xml create mode 100644 flink-connectors/flink-sql-connector-kafka-0.11/pom.xml create mode 100644 flink-connectors/flink-sql-connector-kafka-0.9/pom.xml create mode 100644 flink-connectors/flink-sql-connector-kafka/pom.xml diff --git a/docs/dev/table/connect.md b/docs/dev/table/connect.md index c05102cf7a91f0..27c6920fcd73f5 100644 --- a/docs/dev/table/connect.md +++ b/docs/dev/table/connect.md @@ -45,10 +45,10 @@ The following tables list all available connectors and formats. Their mutual com | Filesystem | | Built-in | Built-in | | Elasticsearch | 6 | `flink-connector-elasticsearch6` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-sql-connector-elasticsearch6{{site.scala_version_suffix}}/{{site.version}}/flink-sql-connector-elasticsearch6{{site.scala_version_suffix}}-{{site.version}}.jar) | | Apache Kafka | 0.8 | `flink-connector-kafka-0.8` | Not available | -| Apache Kafka | 0.9 | `flink-connector-kafka-0.9` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-kafka-0.9{{site.scala_version_suffix}}/{{site.version}}/flink-connector-kafka-0.9{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) | -| Apache Kafka | 0.10 | `flink-connector-kafka-0.10` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-kafka-0.10{{site.scala_version_suffix}}/{{site.version}}/flink-connector-kafka-0.10{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) | -| Apache Kafka | 0.11 | `flink-connector-kafka-0.11` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-kafka-0.11{{site.scala_version_suffix}}/{{site.version}}/flink-connector-kafka-0.11{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) | -| Apache Kafka | 0.11+ (`universal`) | `flink-connector-kafka` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-connector-kafka{{site.scala_version_suffix}}/{{site.version}}/flink-connector-kafka{{site.scala_version_suffix}}-{{site.version}}-sql-jar.jar) | +| Apache Kafka | 0.9 | `flink-connector-kafka-0.9` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka-0.9{{site.scala_version_suffix}}/{{site.version}}/flink-sql-connector-kafka-0.9{{site.scala_version_suffix}}-{{site.version}}.jar) | +| Apache Kafka | 0.10 | `flink-connector-kafka-0.10` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka-0.10{{site.scala_version_suffix}}/{{site.version}}/flink-sql-connector-kafka-0.10{{site.scala_version_suffix}}-{{site.version}}.jar) | +| Apache Kafka | 0.11 | `flink-connector-kafka-0.11` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka-0.11{{site.scala_version_suffix}}/{{site.version}}/flink-sql-connector-kafka-0.11{{site.scala_version_suffix}}-{{site.version}}.jar) | +| Apache Kafka | 0.11+ (`universal`) | `flink-connector-kafka` | [Download](http://central.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka{{site.scala_version_suffix}}/{{site.version}}/flink-sql-connector-kafka{{site.scala_version_suffix}}-{{site.version}}.jar) | ### Formats diff --git a/flink-connectors/flink-connector-kafka-0.10/pom.xml b/flink-connectors/flink-connector-kafka-0.10/pom.xml index 75ced033e90430..8871d791e4f402 100644 --- a/flink-connectors/flink-connector-kafka-0.10/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.10/pom.xml @@ -174,59 +174,6 @@ under the License. - - - - sql-jars - - - !skipSqlJars - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - package - - shade - - - true - sql-jar - - - org.apache.kafka:* - org.apache.flink:flink-connector-kafka-base_${scala.binary.version} - org.apache.flink:flink-connector-kafka-0.9_${scala.binary.version} - - - - - *:* - - kafka/kafka-version.properties - - - - - - org.apache.kafka - org.apache.flink.kafka010.shaded.org.apache.kafka - - - - - - - - - - - diff --git a/flink-connectors/flink-connector-kafka-0.11/pom.xml b/flink-connectors/flink-connector-kafka-0.11/pom.xml index cf25ee933bb359..d44ab8d8ff7e07 100644 --- a/flink-connectors/flink-connector-kafka-0.11/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.11/pom.xml @@ -182,60 +182,6 @@ under the License. - - - - sql-jars - - - !skipSqlJars - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - package - - shade - - - true - sql-jar - - - org.apache.kafka:* - org.apache.flink:flink-connector-kafka-base_${scala.binary.version} - org.apache.flink:flink-connector-kafka-0.9_${scala.binary.version} - org.apache.flink:flink-connector-kafka-0.10_${scala.binary.version} - - - - - *:* - - kafka/kafka-version.properties - - - - - - org.apache.kafka - org.apache.flink.kafka011.shaded.org.apache.kafka - - - - - - - - - - - diff --git a/flink-connectors/flink-connector-kafka-0.9/pom.xml b/flink-connectors/flink-connector-kafka-0.9/pom.xml index 2881291955050a..3ac8554a7a8931 100644 --- a/flink-connectors/flink-connector-kafka-0.9/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.9/pom.xml @@ -174,56 +174,6 @@ under the License. - - - - sql-jars - - - !skipSqlJars - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - package - - shade - - - true - sql-jar - - - org.apache.kafka:* - org.apache.flink:flink-connector-kafka-base_${scala.binary.version} - - - - - *:* - - kafka/kafka-version.properties - - - - - - org.apache.kafka - org.apache.flink.kafka09.shaded.org.apache.kafka - - - - - - - - - diff --git a/flink-connectors/flink-connector-kafka/pom.xml b/flink-connectors/flink-connector-kafka/pom.xml index 53760089cb1a56..30f6a5fe28e230 100644 --- a/flink-connectors/flink-connector-kafka/pom.xml +++ b/flink-connectors/flink-connector-kafka/pom.xml @@ -165,58 +165,6 @@ under the License. - - - - sql-jars - - - !skipSqlJars - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - package - - shade - - - true - sql-jar - - - org.apache.kafka:* - org.apache.flink:flink-connector-kafka-base_${scala.binary.version} - - - - - *:* - - kafka/kafka-version.properties - - - - - - org.apache.kafka - org.apache.flink.kafka.shaded.org.apache.kafka - - - - - - - - - - - diff --git a/flink-connectors/flink-sql-connector-kafka-0.10/pom.xml b/flink-connectors/flink-sql-connector-kafka-0.10/pom.xml new file mode 100644 index 00000000000000..06027573b32a6f --- /dev/null +++ b/flink-connectors/flink-sql-connector-kafka-0.10/pom.xml @@ -0,0 +1,87 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-connectors + 1.8-SNAPSHOT + .. + + + flink-sql-connector-kafka-0.10_${scala.binary.version} + flink-sql-connector-kafka-0.10 + + jar + + + + org.apache.flink + flink-connector-kafka-0.10_${scala.binary.version} + ${project.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + + + org.apache.kafka:* + org.apache.flink:flink-connector-kafka-base_${scala.binary.version} + org.apache.flink:flink-connector-kafka-0.9_${scala.binary.version} + org.apache.flink:flink-connector-kafka-0.10_${scala.binary.version} + + + + + *:* + + kafka/kafka-version.properties + + + + + + org.apache.kafka + org.apache.flink.kafka010.shaded.org.apache.kafka + + + + + + + + + diff --git a/flink-connectors/flink-sql-connector-kafka-0.11/pom.xml b/flink-connectors/flink-sql-connector-kafka-0.11/pom.xml new file mode 100644 index 00000000000000..4b031a13a726bb --- /dev/null +++ b/flink-connectors/flink-sql-connector-kafka-0.11/pom.xml @@ -0,0 +1,88 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-connectors + 1.8-SNAPSHOT + .. + + + flink-sql-connector-kafka-0.11_${scala.binary.version} + flink-sql-connector-kafka-0.11 + + jar + + + + org.apache.flink + flink-connector-kafka-0.11_${scala.binary.version} + ${project.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + + + org.apache.kafka:* + org.apache.flink:flink-connector-kafka-base_${scala.binary.version} + org.apache.flink:flink-connector-kafka-0.9_${scala.binary.version} + org.apache.flink:flink-connector-kafka-0.10_${scala.binary.version} + org.apache.flink:flink-connector-kafka-0.11_${scala.binary.version} + + + + + *:* + + kafka/kafka-version.properties + + + + + + org.apache.kafka + org.apache.flink.kafka011.shaded.org.apache.kafka + + + + + + + + + diff --git a/flink-connectors/flink-sql-connector-kafka-0.9/pom.xml b/flink-connectors/flink-sql-connector-kafka-0.9/pom.xml new file mode 100644 index 00000000000000..4668ee5f64b3f8 --- /dev/null +++ b/flink-connectors/flink-sql-connector-kafka-0.9/pom.xml @@ -0,0 +1,86 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-connectors + 1.8-SNAPSHOT + .. + + + flink-sql-connector-kafka-0.9_${scala.binary.version} + flink-sql-connector-kafka-0.9 + + jar + + + + org.apache.flink + flink-connector-kafka-0.9_${scala.binary.version} + ${project.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + + + org.apache.kafka:* + org.apache.flink:flink-connector-kafka-0.9_${scala.binary.version} + org.apache.flink:flink-connector-kafka-base_${scala.binary.version} + + + + + *:* + + kafka/kafka-version.properties + + + + + + org.apache.kafka + org.apache.flink.kafka09.shaded.org.apache.kafka + + + + + + + + + diff --git a/flink-connectors/flink-sql-connector-kafka/pom.xml b/flink-connectors/flink-sql-connector-kafka/pom.xml new file mode 100644 index 00000000000000..8a2609ed6a7e08 --- /dev/null +++ b/flink-connectors/flink-sql-connector-kafka/pom.xml @@ -0,0 +1,87 @@ + + + + + 4.0.0 + + + flink-connectors + org.apache.flink + 1.8-SNAPSHOT + .. + + + flink-sql-connector-kafka_${scala.binary.version} + flink-sql-connector-kafka + + jar + + + + org.apache.flink + flink-connector-kafka_${scala.binary.version} + ${project.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + false + + + org.apache.kafka:* + org.apache.flink:flink-connector-kafka_${scala.binary.version} + org.apache.flink:flink-connector-kafka-base_${scala.binary.version} + + + + + *:* + + kafka/kafka-version.properties + + + + + + org.apache.kafka + org.apache.flink.kafka.shaded.org.apache.kafka + + + + + + + + + diff --git a/flink-connectors/pom.xml b/flink-connectors/pom.xml index 807f03d53436e0..80b0c257c61bea 100644 --- a/flink-connectors/pom.xml +++ b/flink-connectors/pom.xml @@ -93,6 +93,10 @@ under the License. flink-sql-connector-elasticsearch6 + flink-sql-connector-kafka-0.9 + flink-sql-connector-kafka-0.10 + flink-sql-connector-kafka-0.11 + flink-sql-connector-kafka diff --git a/flink-end-to-end-tests/flink-sql-client-test/pom.xml b/flink-end-to-end-tests/flink-sql-client-test/pom.xml index 3b04f8f61c9e8c..ad54b3a6625a32 100644 --- a/flink-end-to-end-tests/flink-sql-client-test/pom.xml +++ b/flink-end-to-end-tests/flink-sql-client-test/pom.xml @@ -63,33 +63,29 @@ under the License. org.apache.flink - flink-connector-kafka-0.9_${scala.binary.version} + flink-sql-connector-kafka-0.9_${scala.binary.version} ${project.version} - sql-jar provided org.apache.flink - flink-connector-kafka-0.10_${scala.binary.version} + flink-sql-connector-kafka-0.10_${scala.binary.version} ${project.version} - sql-jar provided org.apache.flink - flink-connector-kafka-0.11_${scala.binary.version} + flink-sql-connector-kafka-0.11_${scala.binary.version} ${project.version} - sql-jar provided org.apache.flink - flink-connector-kafka_${scala.binary.version} + flink-sql-connector-kafka_${scala.binary.version} ${project.version} - sql-jar provided @@ -173,23 +169,20 @@ under the License. --> org.apache.flink - flink-connector-kafka-0.10_${scala.binary.version} + flink-sql-connector-kafka-0.10_${scala.binary.version} ${project.version} - sql-jar jar org.apache.flink - flink-connector-kafka-0.11_${scala.binary.version} + flink-sql-connector-kafka-0.11_${scala.binary.version} ${project.version} - sql-jar jar org.apache.flink - flink-connector-kafka_${scala.binary.version} + flink-sql-connector-kafka_${scala.binary.version} ${project.version} - sql-jar jar From cef75fe87f1c457a0ceda20a9ca40c54d354c914 Mon Sep 17 00:00:00 2001 From: EronWright Date: Mon, 31 Dec 2018 09:02:17 -0800 Subject: [PATCH 024/147] [hotfix] [table] Fix typos in Table javadoc. This closes #7388. --- .../apache/flink/table/descriptors/ConnectorDescriptor.java | 2 +- .../table/descriptors/ConnectorDescriptorValidator.java | 5 +++++ .../apache/flink/table/factories/BatchTableSinkFactory.scala | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptor.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptor.java index a1d3e783cdc1cf..ed62b46b36b509 100644 --- a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptor.java +++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptor.java @@ -68,7 +68,7 @@ protected final boolean isFormatNeeded() { /** * Converts this descriptor into a set of connector properties. Usually prefixed with - * {@link FormatDescriptorValidator#FORMAT}. + * {@link ConnectorDescriptorValidator#CONNECTOR}. */ protected abstract Map toConnectorProperties(); } diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.java index 88cf34905f4529..cf9860ae69e2a0 100644 --- a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.java +++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.java @@ -26,6 +26,11 @@ @Internal public abstract class ConnectorDescriptorValidator implements DescriptorValidator { + /** + * Prefix for connector-related properties. + */ + public static final String CONNECTOR = "connector"; + /** * Key for describing the type of the connector. Usually used for factory discovery. */ diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/BatchTableSinkFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/BatchTableSinkFactory.scala index 6fd1f7afb16b58..0b685239e05aa3 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/BatchTableSinkFactory.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/BatchTableSinkFactory.scala @@ -23,7 +23,7 @@ import java.util import org.apache.flink.table.sinks.BatchTableSink /** - * A factory to create configured table sink instances in a streaming environment based on + * A factory to create configured table sink instances in a batch environment based on * string-based properties. See also [[TableFactory]] for more information. * * @tparam T type of records that the factory consumes From 1ee2a8875f959300dc0f930d0576b35f2007b542 Mon Sep 17 00:00:00 2001 From: Yangze Guo Date: Fri, 4 Jan 2019 19:51:51 +0800 Subject: [PATCH 025/147] [hotfix][docs] Fix incorrect example in cep doc --- docs/dev/libs/cep.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/dev/libs/cep.md b/docs/dev/libs/cep.md index 0519dbc366248d..ab44b5c99da01a 100644 --- a/docs/dev/libs/cep.md +++ b/docs/dev/libs/cep.md @@ -1334,7 +1334,7 @@ For example, for a given pattern `b+ c` and a data stream `b1 b2 b3 c`, the diff Have a look also at another example to better see the difference between NO_SKIP and SKIP_TO_FIRST: -Pattern: `(a | c) (b | c) c+.greedy d` and sequence: `a b c1 c2 c3 d` Then the results will be: +Pattern: `(a | b | c) (b | c) c+.greedy d` and sequence: `a b c1 c2 c3 d` Then the results will be: @@ -1349,12 +1349,11 @@ Pattern: `(a | c) (b | c) c+.greedy d` and sequence: `a b c1 c2 c3 d` Then the r a b c1 c2 c3 d
b c1 c2 c3 d
c1 c2 c3 d
- c2 c3 d
- + - + From d3f0075b041c8fd681a776b3278d12c4483a48ac Mon Sep 17 00:00:00 2001 From: wenhuitang Date: Sat, 29 Dec 2018 18:50:31 +0800 Subject: [PATCH 026/147] [FLINK-11119][docs] Correct Scala example for Table Function in User-defined Functions docs. This closes #7379. --- docs/dev/table/udfs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/table/udfs.md b/docs/dev/table/udfs.md index 44b8d431c8d7d0..d93b5c8c3f7972 100644 --- a/docs/dev/table/udfs.md +++ b/docs/dev/table/udfs.md @@ -190,7 +190,7 @@ tableEnv.sqlQuery("SELECT a, word, length FROM MyTable LEFT JOIN LATERAL TABLE(s class Split(separator: String) extends TableFunction[(String, Int)] { def eval(str: String): Unit = { // use collect(...) to emit a row. - str.split(separator).foreach(x -> collect((x, x.length)) + str.split(separator).foreach(x => collect((x, x.length))) } } @@ -210,7 +210,7 @@ tableEnv.registerFunction("split", new Split("#")) // CROSS JOIN a table function (equivalent to "join" in Table API) tableEnv.sqlQuery("SELECT a, word, length FROM MyTable, LATERAL TABLE(split(a)) as T(word, length)") // LEFT JOIN a table function (equivalent to "leftOuterJoin" in Table API) -tableEnv.sqlQuery("SELECT a, word, length FROM MyTable LEFT JOIN TABLE(split(a)) as T(word, length) ON TRUE") +tableEnv.sqlQuery("SELECT a, word, length FROM MyTable LEFT JOIN LATERAL TABLE(split(a)) as T(word, length) ON TRUE") {% endhighlight %} **IMPORTANT:** Do not implement TableFunction as a Scala object. Scala object is a singleton and will cause concurrency issues. From 1c3c663ba7d55fcd33af1502ca426274d51d2ba1 Mon Sep 17 00:00:00 2001 From: Xingcan Cui Date: Tue, 25 Dec 2018 12:03:39 +0800 Subject: [PATCH 027/147] [FLINK-11227] [table] Fix bound checking errors in DescriptorProperties This closes #7373. --- .../descriptors/DescriptorProperties.java | 4 +- .../DescriptorPropertiesTest.scala | 45 +++++++++++++++++-- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorProperties.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorProperties.java index c47698147284e7..76289128b5bca5 100644 --- a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorProperties.java +++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorProperties.java @@ -938,7 +938,7 @@ public void validateFixedIndexedProperties(String key, boolean allowEmpty, Map> subKey : subKeyValidation.entrySet()) { final String fullKey = key + '.' + i + '.' + subKey.getKey(); if (properties.containsKey(fullKey)) { @@ -1134,7 +1134,7 @@ public void validateArray(String key, Consumer elementValidation, int mi } // validate array elements - for (int i = 0; i < maxIndex; i++) { + for (int i = 0; i <= maxIndex; i++) { final String fullKey = key + '.' + i; if (properties.containsKey(fullKey)) { // run validation logic diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorPropertiesTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorPropertiesTest.scala index b2a8ec9f567d8d..fe7c75df6cf789 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorPropertiesTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorPropertiesTest.scala @@ -20,6 +20,7 @@ package org.apache.flink.table.descriptors import java.util import java.util.Collections +import java.util.function.Consumer import org.apache.flink.table.api.ValidationException import org.apache.flink.table.util.JavaScalaConversionUtil.toJava @@ -32,6 +33,9 @@ import org.junit.Test class DescriptorPropertiesTest { private val ARRAY_KEY = "my-array" + private val FIXED_INDEXED_PROPERTY_KEY = "my-fixed-indexed-property" + private val PROPERTY_1_KEY = "property-1" + private val PROPERTY_2_KEY = "property-2" @Test def testEquals(): Unit = { @@ -97,8 +101,8 @@ class DescriptorPropertiesTest { def testArrayInvalidValues(): Unit = { val properties = new DescriptorProperties() properties.putString(s"$ARRAY_KEY.0", "12") - properties.putString(s"$ARRAY_KEY.1", "INVALID") - properties.putString(s"$ARRAY_KEY.2", "66") + properties.putString(s"$ARRAY_KEY.1", "66") + properties.putString(s"$ARRAY_KEY.2", "INVALID") testArrayValidation(properties, 1, Integer.MAX_VALUE) } @@ -118,6 +122,19 @@ class DescriptorPropertiesTest { testArrayValidation(properties, 1, Integer.MAX_VALUE) } + @Test(expected = classOf[ValidationException]) + def testInvalidFixedIndexedProperties(): Unit = { + val property = new DescriptorProperties() + val list = new util.ArrayList[util.List[String]]() + list.add(util.Arrays.asList("1", "string")) + list.add(util.Arrays.asList("INVALID", "string")) + property.putIndexedFixedProperties( + FIXED_INDEXED_PROPERTY_KEY, + util.Arrays.asList(PROPERTY_1_KEY, PROPERTY_2_KEY), + list) + testFixedIndexedPropertiesValidation(property) + } + @Test def testRemoveKeys(): Unit = { val properties = new DescriptorProperties() @@ -155,7 +172,7 @@ class DescriptorPropertiesTest { minLength: Int, maxLength: Int) : Unit = { - val validator: (String) => Unit = (key: String) => { + val validator: String => Unit = (key: String) => { properties.validateInt(key, false) } @@ -165,4 +182,26 @@ class DescriptorPropertiesTest { minLength, maxLength) } + + private def testFixedIndexedPropertiesValidation(properties: DescriptorProperties): Unit = { + + val validatorMap = new util.HashMap[String, Consumer[String]]() + + // PROPERTY_1 should be Int + val validator1: String => Unit = (key: String) => { + properties.validateInt(key, false) + } + validatorMap.put(PROPERTY_1_KEY, toJava(validator1)) + // PROPERTY_2 should be String + val validator2: String => Unit = (key: String) => { + properties.validateString(key, false) + } + validatorMap.put(PROPERTY_2_KEY, toJava(validator2)) + + properties.validateFixedIndexedProperties( + FIXED_INDEXED_PROPERTY_KEY, + false, + validatorMap + ) + } } From c09dc96a460a4947a348639c10d52ed56360fb96 Mon Sep 17 00:00:00 2001 From: wenhuitang Date: Tue, 25 Dec 2018 20:16:27 +0800 Subject: [PATCH 028/147] [FLINK-11173][table] Fix the exception message of proctime attribute validation in TableSourceUtil#validateTableSource This closes #7374. --- .../org/apache/flink/table/sources/TableSourceUtil.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala index 15d68900d219d4..998fc1ff340833 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala @@ -114,12 +114,12 @@ object TableSourceUtil { val rowtimeIdx = schema.getFieldNames.indexOf(rowtimeAttribute) // ensure that field exists if (rowtimeIdx < 0) { - throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " + + throw new ValidationException(s"Found a rowtime attribute for field " + s"'$rowtimeAttribute' but field '$rowtimeAttribute' does not exist in table.") } // ensure that field is of type TIMESTAMP if (schema.getFieldTypes()(rowtimeIdx) != Types.SQL_TIMESTAMP) { - throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " + + throw new ValidationException(s"Found a rowtime attribute for field " + s"'$rowtimeAttribute' but field '$rowtimeAttribute' is not of type TIMESTAMP.") } // look up extractor input fields in return type @@ -138,12 +138,12 @@ object TableSourceUtil { val proctimeIdx = schema.getFieldNames.indexOf(proctimeAttribute) // ensure that field exists if (proctimeIdx < 0) { - throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " + + throw new ValidationException(s"Found a processing time attribute for field " + s"'$proctimeAttribute' but field '$proctimeAttribute' does not exist in table.") } // ensure that field is of type TIMESTAMP if (schema.getFieldTypes()(proctimeIdx) != Types.SQL_TIMESTAMP) { - throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " + + throw new ValidationException(s"Found a processing time attribute for field " + s"'$proctimeAttribute' but field '$proctimeAttribute' is not of type TIMESTAMP.") } case _ => // nothing to validate From 038dc0008f7b27c787e3ccba2204f3a2851dff7d Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Fri, 4 Jan 2019 15:46:06 +0100 Subject: [PATCH 029/147] [FLINK-11001] [table] Update exception message for time attributes on aliases --- .../org/apache/flink/table/api/StreamTableEnvironment.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala index 4fa501cde4d709..e0d32691d62745 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala @@ -683,9 +683,8 @@ abstract class StreamTableEnvironment( case (Alias(UnresolvedFieldReference(_), name, _), _) => fieldNames = name :: fieldNames case (e, _) => - throw new TableException(s"Time attributes can only be defined on field references or " + - s"aliases of valid field references. Rowtime attributes can replace existing fields, " + - s"proctime attributes can not. " + + throw new TableException(s"Time attributes can only be defined on field references. " + + s"Rowtime attributes can replace existing fields, proctime attributes can not. " + s"But was: $e") } From b82b24d720bdebd58ba507af98e68ab027790a4f Mon Sep 17 00:00:00 2001 From: hequn8128 Date: Tue, 11 Dec 2018 10:59:04 +0800 Subject: [PATCH 030/147] [FLINK-11124][table] Add private[flink] to TemporalTableFunction.create() This closes #7271. --- .../apache/flink/table/functions/TemporalTableFunction.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TemporalTableFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TemporalTableFunction.scala index bbb9911c63f309..bf668480a0ad7f 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TemporalTableFunction.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TemporalTableFunction.scala @@ -65,7 +65,7 @@ class TemporalTableFunction private( } object TemporalTableFunction { - def create( + private[flink] def create( table: Table, timeAttribute: Expression, primaryKey: String): TemporalTableFunction = { From c8675b84a36bd3a3e3fa3b9870059752ac5a02a8 Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Fri, 4 Jan 2019 16:18:06 +0100 Subject: [PATCH 031/147] [hotfix] [docs] Update docs regarding missing 'sql-jar' suffix --- docs/dev/table/sqlClient.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/table/sqlClient.md b/docs/dev/table/sqlClient.md index d4eb228c7bf192..97373f409f2fae 100644 --- a/docs/dev/table/sqlClient.md +++ b/docs/dev/table/sqlClient.md @@ -272,7 +272,7 @@ execution: ### Dependencies -The SQL Client does not require to setup a Java project using Maven or SBT. Instead, you can pass the dependencies as regular JAR files that get submitted to the cluster. You can either specify each JAR file separately (using `--jar`) or define entire library directories (using `--library`). For connectors to external systems (such as Apache Kafka) and corresponding data formats (such as JSON), Flink provides **ready-to-use JAR bundles**. These JAR files are suffixed with `sql-jar` and can be downloaded for each release from the Maven central repository. +The SQL Client does not require to setup a Java project using Maven or SBT. Instead, you can pass the dependencies as regular JAR files that get submitted to the cluster. You can either specify each JAR file separately (using `--jar`) or define entire library directories (using `--library`). For connectors to external systems (such as Apache Kafka) and corresponding data formats (such as JSON), Flink provides **ready-to-use JAR bundles**. These JAR files can be downloaded for each release from the Maven central repository. The full list of offered SQL JARs and documentation about how to use them can be found on the [connection to external systems page](connect.html). From e26d90fc86b266978b4bac84fe02ca34b62983fe Mon Sep 17 00:00:00 2001 From: Shuyi Chen Date: Sat, 10 Nov 2018 00:42:49 -0800 Subject: [PATCH 032/147] [FLINK-10848][YARN] properly remove YARN ContainerRequest upon container allocation success This closes #7078 --- .../test/java/org/apache/flink/yarn/YarnTestBase.java | 2 ++ .../org/apache/flink/yarn/YarnFlinkResourceManager.java | 2 ++ .../java/org/apache/flink/yarn/YarnResourceManager.java | 3 ++- .../apache/flink/yarn/YarnFlinkResourceManagerTest.java | 9 +++++++++ .../org/apache/flink/yarn/YarnResourceManagerTest.java | 4 ++++ 5 files changed, 19 insertions(+), 1 deletion(-) diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java index 3763f6592afeee..f1e6a3a767cd92 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java @@ -165,6 +165,8 @@ public abstract class YarnTestBase extends TestLogger { YARN_CONFIGURATION.setInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 3600); YARN_CONFIGURATION.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false); YARN_CONFIGURATION.setInt(YarnConfiguration.NM_VCORES, 666); // memory is overwritten in the MiniYARNCluster. + YARN_CONFIGURATION.set("yarn.scheduler.capacity.resource-calculator", + "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator"); // so we have to change the number of cores for testing. YARN_CONFIGURATION.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 20000); // 20 seconds expiry (to ensure we properly heartbeat with YARN). } diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java index 8e686bbbe34c10..3327505e32df56 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java @@ -438,6 +438,8 @@ private void containersAllocated(List containers) { numPendingContainerRequests = Math.max(0, numPendingContainerRequests - 1); LOG.info("Received new container: {} - Remaining pending container requests: {}", container.getId(), numPendingContainerRequests); + resourceManagerClient.removeContainerRequest(new AMRMClient.ContainerRequest( + container.getResource(), null, null, container.getPriority())); // decide whether to return the container, or whether to start a TaskManager if (numRegistered + containersInLaunch.size() < numRequired) { diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java index 6ff5cd66487112..6669f16fa405dd 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java @@ -361,7 +361,8 @@ public void onContainersAllocated(List containers) { "Received new container: {} - Remaining pending container requests: {}", container.getId(), numPendingContainerRequests); - + resourceManagerClient.removeContainerRequest(new AMRMClient.ContainerRequest( + container.getResource(), null, null, container.getPriority())); if (numPendingContainerRequests > 0) { numPendingContainerRequests--; diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java index 10b2ce97d6fe48..d665df6bc7ce07 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java @@ -43,6 +43,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.NMClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; @@ -69,8 +71,11 @@ import scala.concurrent.duration.FiniteDuration; import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -125,6 +130,8 @@ public void testYarnFlinkResourceManagerJobManagerLostLeadership() throws Except 1), i)); when(mockContainer.getNodeId()).thenReturn(NodeId.newInstance("container", 1234)); + when(mockContainer.getResource()).thenReturn(Resource.newInstance(200, 1)); + when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED); containerList.add(mockContainer); } @@ -233,6 +240,8 @@ public Object answer(InvocationOnMock invocation) throws Throwable { int numberOfRegisteredResources = (Integer) Await.result(numberOfRegisteredResourcesFuture, deadline.timeLeft()); + verify(resourceManagerClient, times(numInitialTaskManagers)).removeContainerRequest( + any(AMRMClient.ContainerRequest.class)); assertEquals(numInitialTaskManagers, numberOfRegisteredResources); } finally { if (resourceManager != null) { diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java index d41d42d7a05f97..ee325dad0f7b1d 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java @@ -401,6 +401,8 @@ public void testStopWorker() throws Exception { resourceManager.onContainersAllocated(ImmutableList.of(testingContainer)); verify(mockResourceManagerClient).addContainerRequest(any(AMRMClient.ContainerRequest.class)); + verify(mockResourceManagerClient).removeContainerRequest( + any(AMRMClient.ContainerRequest.class)); verify(mockNMClient).startContainer(eq(testingContainer), any(ContainerLaunchContext.class)); // Remote task executor registers with YarnResourceManager. @@ -496,6 +498,8 @@ public void testOnContainerCompleted() throws Exception { resourceManager.onContainersAllocated(ImmutableList.of(testingContainer)); verify(mockResourceManagerClient).addContainerRequest(any(AMRMClient.ContainerRequest.class)); + verify(mockResourceManagerClient).removeContainerRequest( + any(AMRMClient.ContainerRequest.class)); verify(mockNMClient).startContainer(eq(testingContainer), any(ContainerLaunchContext.class)); // Callback from YARN when container is Completed, pending request can not be fulfilled by pending From c0a37f831eeee600cba0f3aa5bc9fe7414667144 Mon Sep 17 00:00:00 2001 From: xueyu <278006819@qq.com> Date: Fri, 23 Nov 2018 23:10:28 +0800 Subject: [PATCH 033/147] [FLINK-10845] [table] Support multiple different DISTINCT aggregates for batch This closes #7079. --- .../flink/table/codegen/CodeGenerator.scala | 5 +++ .../table/codegen/calls/ScalarOperators.scala | 40 ++++++++++++++++++- .../expressions/ScalarOperatorsTest.scala | 4 ++ .../runtime/batch/sql/AggregateITCase.scala | 37 +++++++++++++++++ 4 files changed, 85 insertions(+), 1 deletion(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala index 6c05ff0f8015a3..54399379207f8f 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala @@ -838,6 +838,11 @@ abstract class CodeGenerator( val right = operands(1) generateEquals(nullCheck, left, right) + case IS_NOT_DISTINCT_FROM => + val left = operands.head + val right = operands(1) + generateIsNotDistinctFrom(nullCheck, left, right); + case NOT_EQUALS => val left = operands.head val right = operands(1) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala index 5a185da4922f5c..a9cd315a19fbce 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala @@ -191,7 +191,45 @@ object ScalarOperators { ) ) } - } + } + + def generateIsNotDistinctFrom( + nullCheck: Boolean, + left: GeneratedExpression, + right: GeneratedExpression) + : GeneratedExpression = { + + if (nullCheck) { + val resultTerm = newName("result") + val resultTypeTerm = primitiveTypeTermForTypeInfo(BOOLEAN_TYPE_INFO) + val equalExpression = generateEquals( + nullCheck = false, + left.copy(code = GeneratedExpression.NO_CODE), + right.copy(code = GeneratedExpression.NO_CODE)) + + val resultCode = + s""" + |${left.code} + |${right.code} + |$resultTypeTerm $resultTerm; + |if (${left.nullTerm}) { + | $resultTerm = ${right.nullTerm}; + |} else if (${right.nullTerm}) { + | $resultTerm = ${left.nullTerm}; + |} else { + | ${equalExpression.code} + | $resultTerm = ${equalExpression.resultTerm}; + |} + |""".stripMargin + + GeneratedExpression( + resultTerm, + GeneratedExpression.NEVER_NULL, + resultCode, BOOLEAN_TYPE_INFO) + } else { + generateEquals(nullCheck = false, left, right) + } + } def generateEquals( nullCheck: Boolean, diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarOperatorsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarOperatorsTest.scala index de4c804f075b09..d61627b2c41b0b 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarOperatorsTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarOperatorsTest.scala @@ -406,6 +406,10 @@ class ScalarOperatorsTest extends ScalarOperatorsTestBase { "((((true) === true) || false).cast(STRING) + 'X ').trim", "trueX") testTableApi(12.isNull, "12.isNull", "false") + + testSqlApi("f12 IS NOT DISTINCT FROM NULL", "true") + testSqlApi("f9 IS NOT DISTINCT FROM NULL", "false") + testSqlApi("f9 IS NOT DISTINCT FROM 10", "true") } @Test diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/sql/AggregateITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/sql/AggregateITCase.scala index 09ccfc47edabaa..fff252f4ea4f2d 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/sql/AggregateITCase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/sql/AggregateITCase.scala @@ -559,4 +559,41 @@ class AggregateITCase( TestBaseUtils.compareResultAsText(result.asJava, expected) } + + @Test + def testMultipleDistinctWithDiffParams(): Unit = { + val env = ExecutionEnvironment.getExecutionEnvironment + val tEnv = TableEnvironment.getTableEnvironment(env, config) + + val sqlWithNull = "SELECT a, " + + " CASE WHEN b = 2 THEN null ELSE b END AS b, " + + " c FROM MyTable" + + val sqlQuery = + "SELECT b, " + + " COUNT(DISTINCT b), " + + " SUM(DISTINCT (a / 3)), " + + " COUNT(DISTINCT SUBSTRING(c FROM 1 FOR 2))," + + " COUNT(DISTINCT c) " + + "FROM (" + + sqlWithNull + + ") GROUP BY b " + + "ORDER BY b" + + val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c) + tEnv.registerTable("MyTable", t) + + val result = tEnv.sqlQuery(sqlQuery).toDataSet[Row].collect() + + val expected = Seq( + "1,1,0,1,1", + "3,1,3,3,3", + "4,1,5,1,4", + "5,1,12,1,5", + "6,1,18,1,6", + "null,0,1,1,2" + ).mkString("\n") + + TestBaseUtils.compareResultAsText(result.asJava, expected) + } } From 28e0b83dd86cbf2a8d0eeda95391f206ba0d812b Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Wed, 6 Jun 2018 17:15:20 +0200 Subject: [PATCH 034/147] [hotfix] Improve performance of GenericArraySerializer.copy() --- .../base/GenericArraySerializer.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java index 6668c572930b2e..55ba8ab477f5a6 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.lang.reflect.Array; +import java.util.Arrays; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -86,16 +87,21 @@ public C[] createInstance() { @Override public C[] copy(C[] from) { - C[] copy = create(from.length); - for (int i = 0; i < copy.length; i++) { - C val = from[i]; - if (val != null) { - copy[i] = this.componentSerializer.copy(val); + final TypeSerializer serializer = this.componentSerializer; + + if (serializer.isImmutableType()) { + return Arrays.copyOf(from, from.length); + } else { + C[] copy = create(from.length); + for (int i = 0; i < copy.length; i++) { + C val = from[i]; + if (val != null) { + copy[i] = serializer.copy(val); + } } + return copy; } - - return copy; } @Override From 6f9a8840fc3cf58e3b43ae4ce1da5a9023888442 Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Tue, 11 Dec 2018 15:21:25 +0100 Subject: [PATCH 035/147] [hotfix] Remove unused generic parameter from RocksDB states --- .../streaming/state/AbstractRocksDBAppendingState.java | 5 ++--- .../flink/contrib/streaming/state/AbstractRocksDBState.java | 3 +-- .../contrib/streaming/state/RocksDBAggregatingState.java | 2 +- .../flink/contrib/streaming/state/RocksDBFoldingState.java | 2 +- .../contrib/streaming/state/RocksDBKeyedStateBackend.java | 2 +- .../flink/contrib/streaming/state/RocksDBListState.java | 2 +- .../flink/contrib/streaming/state/RocksDBMapState.java | 2 +- .../flink/contrib/streaming/state/RocksDBReducingState.java | 2 +- .../flink/contrib/streaming/state/RocksDBValueState.java | 2 +- 9 files changed, 10 insertions(+), 12 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBAppendingState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBAppendingState.java index 8c0f4d7da7b22e..04e665a6e87978 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBAppendingState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBAppendingState.java @@ -18,7 +18,6 @@ package org.apache.flink.contrib.streaming.state; -import org.apache.flink.api.common.state.State; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.runtime.state.internal.InternalAppendingState; import org.apache.flink.util.FlinkRuntimeException; @@ -28,8 +27,8 @@ import java.io.IOException; -abstract class AbstractRocksDBAppendingState - extends AbstractRocksDBState +abstract class AbstractRocksDBAppendingState + extends AbstractRocksDBState implements InternalAppendingState { /** diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java index 2218bc0d3e0ee3..b5ab996b1f6780 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java @@ -44,9 +44,8 @@ * @param The type of the key. * @param The type of the namespace. * @param The type of values kept internally in state. - * @param The type of {@link State}. */ -public abstract class AbstractRocksDBState implements InternalKvState, State { +public abstract class AbstractRocksDBState implements InternalKvState, State { /** Serializer for the namespace. */ final TypeSerializer namespaceSerializer; diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java index 2085fb86256a3d..770c558c8278f7 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java @@ -43,7 +43,7 @@ * @param The type of the value returned from the state */ class RocksDBAggregatingState - extends AbstractRocksDBAppendingState> + extends AbstractRocksDBAppendingState implements InternalAggregatingState { /** User-specified aggregation function. */ diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java index 4d6635704b934c..c5e830fcedf0a6 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java @@ -42,7 +42,7 @@ */ @Deprecated class RocksDBFoldingState - extends AbstractRocksDBAppendingState> + extends AbstractRocksDBAppendingState implements InternalFoldingState { /** User-specified fold function. */ diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 700c5468c8c40b..72445ca32dd4e1 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -1400,7 +1400,7 @@ private void migrateStateValues( } @SuppressWarnings("unchecked") - AbstractRocksDBState rocksDBState = (AbstractRocksDBState) state; + AbstractRocksDBState rocksDBState = (AbstractRocksDBState) state; Snapshot rocksDBSnapshot = db.getSnapshot(); try ( diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java index 6904c853fe310d..cedeed8753cb35 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java @@ -59,7 +59,7 @@ * @param The type of the values in the list state. */ class RocksDBListState - extends AbstractRocksDBState, ListState> + extends AbstractRocksDBState> implements InternalListState { /** Serializer for the values. */ diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java index cb656b53b1b152..7abbe3d2c457e7 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java @@ -64,7 +64,7 @@ * @param The type of the values in the map state. */ class RocksDBMapState - extends AbstractRocksDBState, MapState> + extends AbstractRocksDBState> implements InternalMapState { private static final Logger LOG = LoggerFactory.getLogger(RocksDBMapState.class); diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java index 138357b0d77c85..69736ae03f702e 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java @@ -41,7 +41,7 @@ * @param The type of value that the state state stores. */ class RocksDBReducingState - extends AbstractRocksDBAppendingState> + extends AbstractRocksDBAppendingState implements InternalReducingState { /** User-specified reduce function. */ diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java index 0ca90d4a5212f6..97b83dfa49c7a6 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java @@ -40,7 +40,7 @@ * @param The type of value that the state state stores. */ class RocksDBValueState - extends AbstractRocksDBState> + extends AbstractRocksDBState implements InternalValueState { /** From 54ef382439b1687a645a377a5f6a095746423109 Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Tue, 27 Nov 2018 13:51:29 +0800 Subject: [PATCH 036/147] [FLINK-9702] Improvement in (de)serialization of keys and values for RocksDB state This closes #7288. Co-authored-by: Stefan Richter Co-authored-by: klion26 --- .../memory/ByteArrayOutputStreamWithPos.java | 3 +- .../core/memory/DataOutputSerializer.java | 7 + .../streaming/state/AbstractRocksDBState.java | 140 +++++----- .../state/RocksDBAggregatingState.java | 14 +- .../state/RocksDBKeySerializationUtils.java | 12 +- .../state/RocksDBKeyedStateBackend.java | 14 + .../streaming/state/RocksDBListState.java | 90 +++---- .../streaming/state/RocksDBMapState.java | 83 ++---- .../streaming/state/RocksDBReducingState.java | 15 +- .../RocksDBSerializedCompositeKeyBuilder.java | 206 +++++++++++++++ .../streaming/state/RocksDBValueState.java | 14 +- ...ksDBSerializedCompositeKeyBuilderTest.java | 250 ++++++++++++++++++ 12 files changed, 643 insertions(+), 205 deletions(-) create mode 100644 flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBSerializedCompositeKeyBuilder.java create mode 100644 flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBSerializedCompositeKeyBuilderTest.java diff --git a/flink-core/src/main/java/org/apache/flink/core/memory/ByteArrayOutputStreamWithPos.java b/flink-core/src/main/java/org/apache/flink/core/memory/ByteArrayOutputStreamWithPos.java index 22330c5f360a15..2f4fdfe5b1e695 100644 --- a/flink-core/src/main/java/org/apache/flink/core/memory/ByteArrayOutputStreamWithPos.java +++ b/flink-core/src/main/java/org/apache/flink/core/memory/ByteArrayOutputStreamWithPos.java @@ -22,7 +22,6 @@ import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.util.Preconditions; -import java.io.IOException; import java.io.OutputStream; import java.util.Arrays; @@ -112,7 +111,7 @@ public void setPosition(int position) { } @Override - public void close() throws IOException { + public void close() { } public byte[] getBuf() { diff --git a/flink-core/src/main/java/org/apache/flink/core/memory/DataOutputSerializer.java b/flink-core/src/main/java/org/apache/flink/core/memory/DataOutputSerializer.java index 01feae03380ba2..b749c84711caaa 100644 --- a/flink-core/src/main/java/org/apache/flink/core/memory/DataOutputSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/core/memory/DataOutputSerializer.java @@ -18,6 +18,8 @@ package org.apache.flink.core.memory; +import org.apache.flink.util.Preconditions; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -350,6 +352,11 @@ public void write(DataInputView source, int numBytes) throws IOException { this.position += numBytes; } + public void setPosition(int position) { + Preconditions.checkArgument(position >= 0 && position <= this.position, "Position out of bounds."); + this.position = position; + } + // ------------------------------------------------------------------------ // Utilities // ------------------------------------------------------------------------ diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java index b5ab996b1f6780..fb19a8a2816703 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java @@ -34,6 +34,7 @@ import org.rocksdb.WriteOptions; import java.io.IOException; +import java.util.List; /** * Base class for {@link State} implementations that store state in a RocksDB database. @@ -70,7 +71,7 @@ public abstract class AbstractRocksDBState implements InternalKvState sharedKeyNamespaceSerializer; /** * Creates a new RocksDB backed state. @@ -99,8 +100,7 @@ protected AbstractRocksDBState( this.dataOutputView = new DataOutputSerializer(128); this.dataInputView = new DataInputDeserializer(); - this.ambiguousKeyPossible = - RocksDBKeySerializationUtils.isAmbiguousKeyPossible(backend.getKeySerializer(), namespaceSerializer); + this.sharedKeyNamespaceSerializer = backend.getSharedRocksKeyBuilder(); } // ------------------------------------------------------------------------ @@ -108,17 +108,15 @@ protected AbstractRocksDBState( @Override public void clear() { try { - writeCurrentKeyWithGroupAndNamespace(); - byte[] key = dataOutputView.getCopyOfBuffer(); - backend.db.delete(columnFamily, writeOptions, key); - } catch (IOException | RocksDBException e) { + backend.db.delete(columnFamily, writeOptions, serializeCurrentKeyWithGroupAndNamespace()); + } catch (RocksDBException e) { throw new FlinkRuntimeException("Error while removing entry from RocksDB", e); } } @Override public void setCurrentNamespace(N namespace) { - this.currentNamespace = Preconditions.checkNotNull(namespace, "Namespace"); + this.currentNamespace = namespace; } @Override @@ -128,30 +126,78 @@ public byte[] getSerializedValue( final TypeSerializer safeNamespaceSerializer, final TypeSerializer safeValueSerializer) throws Exception { - Preconditions.checkNotNull(serializedKeyAndNamespace); - Preconditions.checkNotNull(safeKeySerializer); - Preconditions.checkNotNull(safeNamespaceSerializer); - Preconditions.checkNotNull(safeValueSerializer); - //TODO make KvStateSerializer key-group aware to save this round trip and key-group computation Tuple2 keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace( serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer); int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(keyAndNamespace.f0, backend.getNumberOfKeyGroups()); - // we cannot reuse the keySerializationStream member since this method - // is called concurrently to the other ones and it may thus contain garbage - DataOutputSerializer tmpKeySerializationView = new DataOutputSerializer(128); + RocksDBSerializedCompositeKeyBuilder keyBuilder = + new RocksDBSerializedCompositeKeyBuilder<>( + safeKeySerializer, + backend.getKeyGroupPrefixBytes(), + 32 + ); + keyBuilder.setKeyAndKeyGroup(keyAndNamespace.f0, keyGroup); + byte[] key = keyBuilder.buildCompositeKeyNamespace(keyAndNamespace.f1, namespaceSerializer); + return backend.db.get(columnFamily, key); + } + + byte[] serializeCurrentKeyWithGroupAndNamespacePlusUserKey( + UK userKey, + TypeSerializer userKeySerializer) throws IOException { + return sharedKeyNamespaceSerializer.buildCompositeKeyNamesSpaceUserKey( + currentNamespace, + namespaceSerializer, + userKey, + userKeySerializer + ); + } + + private byte[] serializeValueInternal(T value, TypeSerializer serializer) throws IOException { + serializer.serialize(value, dataOutputView); + return dataOutputView.getCopyOfBuffer(); + } + + byte[] serializeCurrentKeyWithGroupAndNamespace() { + return sharedKeyNamespaceSerializer.buildCompositeKeyNamespace(currentNamespace, namespaceSerializer); + } + + byte[] serializeValue(V value) throws IOException { + return serializeValue(value, valueSerializer); + } + + byte[] serializeValueNullSensitive(T value, TypeSerializer serializer) throws IOException { + dataOutputView.clear(); + dataOutputView.writeBoolean(value == null); + return serializeValueInternal(value, serializer); + } + + byte[] serializeValue(T value, TypeSerializer serializer) throws IOException { + dataOutputView.clear(); + return serializeValueInternal(value, serializer); + } + + byte[] serializeValueList( + List valueList, + TypeSerializer elementSerializer, + byte delimiter) throws IOException { + + dataOutputView.clear(); + boolean first = true; - writeKeyWithGroupAndNamespace( - keyGroup, - keyAndNamespace.f0, - safeKeySerializer, - keyAndNamespace.f1, - safeNamespaceSerializer, - tmpKeySerializationView); + for (T value : valueList) { + Preconditions.checkNotNull(value, "You cannot add null to a value list."); + + if (first) { + first = false; + } else { + dataOutputView.write(delimiter); + } + elementSerializer.serialize(value, dataOutputView); + } - return backend.db.get(columnFamily, tmpKeySerializationView.getCopyOfBuffer()); + return dataOutputView.getCopyOfBuffer(); } public void migrateSerializedValue( @@ -169,12 +215,7 @@ public void migrateSerializedValue( } byte[] getKeyBytes() { - try { - writeCurrentKeyWithGroupAndNamespace(); - return dataOutputView.getCopyOfBuffer(); - } catch (IOException e) { - throw new FlinkRuntimeException("Error while serializing key", e); - } + return serializeCurrentKeyWithGroupAndNamespace(); } byte[] getValueBytes(V value) { @@ -187,45 +228,6 @@ byte[] getValueBytes(V value) { } } - protected void writeCurrentKeyWithGroupAndNamespace() throws IOException { - writeKeyWithGroupAndNamespace( - backend.getCurrentKeyGroupIndex(), - backend.getCurrentKey(), - currentNamespace, - dataOutputView); - } - - protected void writeKeyWithGroupAndNamespace( - int keyGroup, K key, N namespace, - DataOutputSerializer keySerializationDataOutputView) throws IOException { - - writeKeyWithGroupAndNamespace( - keyGroup, - key, - backend.getKeySerializer(), - namespace, - namespaceSerializer, - keySerializationDataOutputView); - } - - protected void writeKeyWithGroupAndNamespace( - final int keyGroup, - final K key, - final TypeSerializer keySerializer, - final N namespace, - final TypeSerializer namespaceSerializer, - final DataOutputSerializer keySerializationDataOutputView) throws IOException { - - Preconditions.checkNotNull(key, "No key set. This method should not be called outside of a keyed context."); - Preconditions.checkNotNull(keySerializer); - Preconditions.checkNotNull(namespaceSerializer); - - keySerializationDataOutputView.clear(); - RocksDBKeySerializationUtils.writeKeyGroup(keyGroup, backend.getKeyGroupPrefixBytes(), keySerializationDataOutputView); - RocksDBKeySerializationUtils.writeKey(key, keySerializer, keySerializationDataOutputView, ambiguousKeyPossible); - RocksDBKeySerializationUtils.writeNameSpace(namespace, namespaceSerializer, keySerializationDataOutputView, ambiguousKeyPossible); - } - protected V getDefaultValue() { if (defaultValue != null) { return valueSerializer.copy(defaultValue); diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java index 770c558c8278f7..fbd2979feb1b53 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java @@ -109,19 +109,14 @@ public void mergeNamespaces(N target, Collection sources) { return; } - // cache key and namespace - final K key = backend.getCurrentKey(); - final int keyGroup = backend.getCurrentKeyGroupIndex(); - try { ACC current = null; // merge the sources to the target for (N source : sources) { if (source != null) { - writeKeyWithGroupAndNamespace(keyGroup, key, source, dataOutputView); - - final byte[] sourceKey = dataOutputView.getCopyOfBuffer(); + setCurrentNamespace(source); + final byte[] sourceKey = serializeCurrentKeyWithGroupAndNamespace(); final byte[] valueBytes = backend.db.get(columnFamily, sourceKey); backend.db.delete(columnFamily, writeOptions, sourceKey); @@ -141,10 +136,9 @@ public void mergeNamespaces(N target, Collection sources) { // if something came out of merging the sources, merge it or write it to the target if (current != null) { + setCurrentNamespace(target); // create the target full-binary-key - writeKeyWithGroupAndNamespace(keyGroup, key, target, dataOutputView); - - final byte[] targetKey = dataOutputView.getCopyOfBuffer(); + final byte[] targetKey = serializeCurrentKeyWithGroupAndNamespace(); final byte[] targetValueBytes = backend.db.get(columnFamily, targetKey); if (targetValueBytes != null) { diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeySerializationUtils.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeySerializationUtils.java index d8844bfece12a1..02b4ffcecfd538 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeySerializationUtils.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeySerializationUtils.java @@ -23,6 +23,8 @@ import org.apache.flink.core.memory.DataOutputSerializer; import org.apache.flink.core.memory.DataOutputView; +import javax.annotation.Nonnull; + import java.io.IOException; /** @@ -80,8 +82,12 @@ public static void writeNameSpace( } } + public static boolean isSerializerTypeVariableSized(@Nonnull TypeSerializer serializer) { + return serializer.getLength() < 0; + } + public static boolean isAmbiguousKeyPossible(TypeSerializer keySerializer, TypeSerializer namespaceSerializer) { - return (keySerializer.getLength() < 0) && (namespaceSerializer.getLength() < 0); + return (isSerializerTypeVariableSized(keySerializer) && isSerializerTypeVariableSized(namespaceSerializer)); } public static void writeKeyGroup( @@ -108,7 +114,7 @@ public static void writeKey( } } - private static void readVariableIntBytes(DataInputView inputView, int value) throws IOException { + public static void readVariableIntBytes(DataInputView inputView, int value) throws IOException { do { inputView.readByte(); value >>>= 8; @@ -122,7 +128,7 @@ private static void writeLengthFrom( writeVariableIntBytes(length, keySerializationDateDataOutputView); } - private static void writeVariableIntBytes( + public static void writeVariableIntBytes( int value, DataOutputView keySerializationDateDataOutputView) throws IOException { diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 72445ca32dd4e1..6b0039342de95c 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -242,6 +242,9 @@ IS createState( /** The native metrics monitor. */ private RocksDBNativeMetricMonitor nativeMetricMonitor; + /** Helper to build the byte arrays of composite keys to address data in RocksDB. Shared across all states.*/ + private final RocksDBSerializedCompositeKeyBuilder sharedRocksKeyBuilder; + public RocksDBKeyedStateBackend( String operatorIdentifier, ClassLoader userCodeClassLoader, @@ -294,6 +297,7 @@ public RocksDBKeyedStateBackend( this.kvStateInformation = new LinkedHashMap<>(); this.writeOptions = new WriteOptions().setDisableWAL(true); + this.sharedRocksKeyBuilder = new RocksDBSerializedCompositeKeyBuilder<>(keySerializer, keyGroupPrefixBytes, 32); this.metricOptions = metricOptions; this.metricGroup = metricGroup; @@ -373,6 +377,12 @@ private void registerKvStateInformation(String columnFamilyName, Tuple2 getSharedRocksKeyBuilder() { + return sharedRocksKeyBuilder; + } + /** * Triggers an asynchronous snapshot of the keyed state backend from RocksDB. This snapshot can be canceled and * is also stopped when the backend is closed through {@link #dispose()}. For each backend, this method must always diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java index cedeed8753cb35..13f5559405e8bf 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java @@ -115,11 +115,10 @@ public Iterable get() { @Override public List getInternal() { try { - writeCurrentKeyWithGroupAndNamespace(); - byte[] key = dataOutputView.getCopyOfBuffer(); + byte[] key = serializeCurrentKeyWithGroupAndNamespace(); byte[] valueBytes = backend.db.get(columnFamily, key); return deserializeList(valueBytes); - } catch (IOException | RocksDBException e) { + } catch (RocksDBException e) { throw new FlinkRuntimeException("Error while retrieving data from RocksDB", e); } } @@ -160,11 +159,12 @@ public void add(V value) { Preconditions.checkNotNull(value, "You cannot add null to a ListState."); try { - writeCurrentKeyWithGroupAndNamespace(); - byte[] key = dataOutputView.getCopyOfBuffer(); - dataOutputView.clear(); - elementSerializer.serialize(value, dataOutputView); - backend.db.merge(columnFamily, writeOptions, key, dataOutputView.getCopyOfBuffer()); + backend.db.merge( + columnFamily, + writeOptions, + serializeCurrentKeyWithGroupAndNamespace(), + serializeValue(value, elementSerializer) + ); } catch (Exception e) { throw new FlinkRuntimeException("Error while adding data to RocksDB", e); } @@ -176,21 +176,17 @@ public void mergeNamespaces(N target, Collection sources) { return; } - // cache key and namespace - final K key = backend.getCurrentKey(); - final int keyGroup = backend.getCurrentKeyGroupIndex(); - try { // create the target full-binary-key - writeKeyWithGroupAndNamespace(keyGroup, key, target, dataOutputView); - final byte[] targetKey = dataOutputView.getCopyOfBuffer(); + setCurrentNamespace(target); + final byte[] targetKey = serializeCurrentKeyWithGroupAndNamespace(); // merge the sources to the target for (N source : sources) { if (source != null) { - writeKeyWithGroupAndNamespace(keyGroup, key, source, dataOutputView); + setCurrentNamespace(source); + final byte[] sourceKey = serializeCurrentKeyWithGroupAndNamespace(); - byte[] sourceKey = dataOutputView.getCopyOfBuffer(); byte[] valueBytes = backend.db.get(columnFamily, sourceKey); backend.db.delete(columnFamily, writeOptions, sourceKey); @@ -218,10 +214,11 @@ public void updateInternal(List values) { if (!values.isEmpty()) { try { - writeCurrentKeyWithGroupAndNamespace(); - byte[] key = dataOutputView.getCopyOfBuffer(); - byte[] premerge = getPreMergedValue(values, elementSerializer, dataOutputView); - backend.db.put(columnFamily, writeOptions, key, premerge); + backend.db.put( + columnFamily, + writeOptions, + serializeCurrentKeyWithGroupAndNamespace(), + serializeValueList(values, elementSerializer, DELIMITER)); } catch (IOException | RocksDBException e) { throw new FlinkRuntimeException("Error while updating data to RocksDB", e); } @@ -234,10 +231,11 @@ public void addAll(List values) { if (!values.isEmpty()) { try { - writeCurrentKeyWithGroupAndNamespace(); - byte[] key = dataOutputView.getCopyOfBuffer(); - byte[] premerge = getPreMergedValue(values, elementSerializer, dataOutputView); - backend.db.merge(columnFamily, writeOptions, key, premerge); + backend.db.merge( + columnFamily, + writeOptions, + serializeCurrentKeyWithGroupAndNamespace(), + serializeValueList(values, elementSerializer, DELIMITER)); } catch (IOException | RocksDBException e) { throw new FlinkRuntimeException("Error while updating data to RocksDB", e); } @@ -273,26 +271,6 @@ public void migrateSerializedValue( } } - private static byte[] getPreMergedValue( - List values, - TypeSerializer elementSerializer, - DataOutputSerializer keySerializationStream) throws IOException { - - keySerializationStream.clear(); - boolean first = true; - for (V value : values) { - Preconditions.checkNotNull(value, "You cannot add null to a ListState."); - if (first) { - first = false; - } else { - keySerializationStream.write(DELIMITER); - } - elementSerializer.serialize(value, keySerializationStream); - } - - return keySerializationStream.getCopyOfBuffer(); - } - @SuppressWarnings("unchecked") static IS create( StateDescriptor stateDesc, @@ -343,10 +321,32 @@ public byte[] filterOrTransform(@Nullable byte[] value) { prevPosition = in.getPosition(); } try { - return result.isEmpty() ? null : getPreMergedValue(result, elementSerializer, out); + return result.isEmpty() ? null : serializeValueList(result, elementSerializer, DELIMITER); } catch (IOException e) { throw new FlinkRuntimeException("Failed to serialize transformed list", e); } } + + byte[] serializeValueList( + List valueList, + TypeSerializer elementSerializer, + byte delimiter) throws IOException { + + out.clear(); + boolean first = true; + + for (T value : valueList) { + Preconditions.checkNotNull(value, "You cannot add null to a value list."); + + if (first) { + first = false; + } else { + out.write(delimiter); + } + elementSerializer.serialize(value, out); + } + + return out.getCopyOfBuffer(); + } } } diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java index 7abbe3d2c457e7..13cbdedb1d91e4 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java @@ -119,7 +119,7 @@ public TypeSerializer> getValueSerializer() { @Override public UV get(UK userKey) throws IOException, RocksDBException { - byte[] rawKeyBytes = serializeUserKeyWithCurrentKeyAndNamespace(userKey); + byte[] rawKeyBytes = serializeCurrentKeyWithGroupAndNamespacePlusUserKey(userKey, userKeySerializer); byte[] rawValueBytes = backend.db.get(columnFamily, rawKeyBytes); return (rawValueBytes == null ? null : deserializeUserValue(dataInputView, rawValueBytes, userValueSerializer)); @@ -128,8 +128,8 @@ public UV get(UK userKey) throws IOException, RocksDBException { @Override public void put(UK userKey, UV userValue) throws IOException, RocksDBException { - byte[] rawKeyBytes = serializeUserKeyWithCurrentKeyAndNamespace(userKey); - byte[] rawValueBytes = serializeUserValue(userValue, userValueSerializer, dataOutputView); + byte[] rawKeyBytes = serializeCurrentKeyWithGroupAndNamespacePlusUserKey(userKey, userKeySerializer); + byte[] rawValueBytes = serializeValueNullSensitive(userValue, userValueSerializer); backend.db.put(columnFamily, writeOptions, rawKeyBytes, rawValueBytes); } @@ -142,8 +142,8 @@ public void putAll(Map map) throws IOException, RocksDBException { try (RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(backend.db, writeOptions)) { for (Map.Entry entry : map.entrySet()) { - byte[] rawKeyBytes = serializeUserKeyWithCurrentKeyAndNamespace(entry.getKey()); - byte[] rawValueBytes = serializeUserValue(entry.getValue(), userValueSerializer, dataOutputView); + byte[] rawKeyBytes = serializeCurrentKeyWithGroupAndNamespacePlusUserKey(entry.getKey(), userKeySerializer); + byte[] rawValueBytes = serializeValueNullSensitive(entry.getValue(), userValueSerializer); writeBatchWrapper.put(columnFamily, rawKeyBytes, rawValueBytes); } } @@ -151,21 +151,21 @@ public void putAll(Map map) throws IOException, RocksDBException { @Override public void remove(UK userKey) throws IOException, RocksDBException { - byte[] rawKeyBytes = serializeUserKeyWithCurrentKeyAndNamespace(userKey); + byte[] rawKeyBytes = serializeCurrentKeyWithGroupAndNamespacePlusUserKey(userKey, userKeySerializer); backend.db.delete(columnFamily, writeOptions, rawKeyBytes); } @Override public boolean contains(UK userKey) throws IOException, RocksDBException { - byte[] rawKeyBytes = serializeUserKeyWithCurrentKeyAndNamespace(userKey); + byte[] rawKeyBytes = serializeCurrentKeyWithGroupAndNamespacePlusUserKey(userKey, userKeySerializer); byte[] rawValueBytes = backend.db.get(columnFamily, rawKeyBytes); return (rawValueBytes != null); } @Override - public Iterable> entries() throws IOException { + public Iterable> entries() { final Iterator> iterator = iterator(); // Return null to make the behavior consistent with other states. @@ -177,10 +177,11 @@ public Iterable> entries() throws IOException { } @Override - public Iterable keys() throws IOException { - final byte[] prefixBytes = serializeCurrentKeyAndNamespace(); + public Iterable keys() { + final byte[] prefixBytes = serializeCurrentKeyWithGroupAndNamespace(); return () -> new RocksDBMapIterator(backend.db, prefixBytes, userKeySerializer, userValueSerializer, dataInputView) { + @Nullable @Override public UK next() { RocksDBMapEntry entry = nextEntry(); @@ -190,8 +191,8 @@ public UK next() { } @Override - public Iterable values() throws IOException { - final byte[] prefixBytes = serializeCurrentKeyAndNamespace(); + public Iterable values() { + final byte[] prefixBytes = serializeCurrentKeyWithGroupAndNamespace(); return () -> new RocksDBMapIterator(backend.db, prefixBytes, userKeySerializer, userValueSerializer, dataInputView) { @Override @@ -203,8 +204,8 @@ public UV next() { } @Override - public Iterator> iterator() throws IOException { - final byte[] prefixBytes = serializeCurrentKeyAndNamespace(); + public Iterator> iterator() { + final byte[] prefixBytes = serializeCurrentKeyWithGroupAndNamespace(); return new RocksDBMapIterator>(backend.db, prefixBytes, userKeySerializer, userValueSerializer, dataInputView) { @Override @@ -220,7 +221,7 @@ public void clear() { try (RocksIteratorWrapper iterator = RocksDBKeyedStateBackend.getRocksIterator(backend.db, columnFamily); WriteBatch writeBatch = new WriteBatch(128)) { - final byte[] keyPrefixBytes = serializeCurrentKeyAndNamespace(); + final byte[] keyPrefixBytes = serializeCurrentKeyWithGroupAndNamespace(); iterator.seek(keyPrefixBytes); while (iterator.isValid()) { @@ -259,18 +260,15 @@ public byte[] getSerializedValue( int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(keyAndNamespace.f0, backend.getNumberOfKeyGroups()); - DataOutputSerializer outputView = new DataOutputSerializer(128); - DataInputDeserializer inputView = new DataInputDeserializer(); - - writeKeyWithGroupAndNamespace( - keyGroup, - keyAndNamespace.f0, + RocksDBSerializedCompositeKeyBuilder keyBuilder = + new RocksDBSerializedCompositeKeyBuilder<>( safeKeySerializer, - keyAndNamespace.f1, - safeNamespaceSerializer, - outputView); + backend.getKeyGroupPrefixBytes(), + 32); + + keyBuilder.setKeyAndKeyGroup(keyAndNamespace.f0, keyGroup); - final byte[] keyPrefixBytes = outputView.getCopyOfBuffer(); + final byte[] keyPrefixBytes = keyBuilder.buildCompositeKeyNamespace(keyAndNamespace.f1, namespaceSerializer); final MapSerializer serializer = (MapSerializer) safeValueSerializer; @@ -282,7 +280,8 @@ public byte[] getSerializedValue( keyPrefixBytes, dupUserKeySerializer, dupUserValueSerializer, - inputView) { + dataInputView + ) { @Override public Map.Entry next() { @@ -302,36 +301,6 @@ public Map.Entry next() { // Serialization Methods // ------------------------------------------------------------------------ - private byte[] serializeCurrentKeyAndNamespace() throws IOException { - writeCurrentKeyWithGroupAndNamespace(); - - return dataOutputView.getCopyOfBuffer(); - } - - private byte[] serializeUserKeyWithCurrentKeyAndNamespace(UK userKey) throws IOException { - serializeCurrentKeyAndNamespace(); - userKeySerializer.serialize(userKey, dataOutputView); - - return dataOutputView.getCopyOfBuffer(); - } - - private static byte[] serializeUserValue( - UV userValue, - TypeSerializer valueSerializer, - DataOutputSerializer dataOutputView) throws IOException { - - dataOutputView.clear(); - - if (userValue == null) { - dataOutputView.writeBoolean(true); - } else { - dataOutputView.writeBoolean(false); - valueSerializer.serialize(userValue, dataOutputView); - } - - return dataOutputView.getCopyOfBuffer(); - } - private static UK deserializeUserKey( DataInputDeserializer dataInputView, int userKeyOffset, @@ -474,7 +443,7 @@ public UV setValue(UV value) { try { userValue = value; - rawValueBytes = serializeUserValue(value, valueSerializer, dataOutputView); + rawValueBytes = serializeValueNullSensitive(value, valueSerializer); db.put(columnFamily, writeOptions, rawKeyBytes, rawValueBytes); } catch (IOException | RocksDBException e) { diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java index 69736ae03f702e..a5eb46eace6e0a 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java @@ -102,20 +102,14 @@ public void mergeNamespaces(N target, Collection sources) { return; } - // cache key and namespace - final K key = backend.getCurrentKey(); - final int keyGroup = backend.getCurrentKeyGroupIndex(); - try { V current = null; // merge the sources to the target for (N source : sources) { if (source != null) { - - writeKeyWithGroupAndNamespace(keyGroup, key, source, dataOutputView); - - final byte[] sourceKey = dataOutputView.getCopyOfBuffer(); + setCurrentNamespace(source); + final byte[] sourceKey = serializeCurrentKeyWithGroupAndNamespace(); final byte[] valueBytes = backend.db.get(columnFamily, sourceKey); backend.db.delete(columnFamily, writeOptions, sourceKey); @@ -136,9 +130,8 @@ public void mergeNamespaces(N target, Collection sources) { // if something came out of merging the sources, merge it or write it to the target if (current != null) { // create the target full-binary-key - writeKeyWithGroupAndNamespace(keyGroup, key, target, dataOutputView); - - final byte[] targetKey = dataOutputView.getCopyOfBuffer(); + setCurrentNamespace(target); + final byte[] targetKey = serializeCurrentKeyWithGroupAndNamespace(); final byte[] targetValueBytes = backend.db.get(columnFamily, targetKey); if (targetValueBytes != null) { diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBSerializedCompositeKeyBuilder.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBSerializedCompositeKeyBuilder.java new file mode 100644 index 00000000000000..8e83e292820953 --- /dev/null +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBSerializedCompositeKeyBuilder.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.contrib.streaming.state; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.memory.DataOutputSerializer; +import org.apache.flink.util.FlinkRuntimeException; + +import javax.annotation.Nonnegative; +import javax.annotation.Nonnull; +import javax.annotation.concurrent.NotThreadSafe; + +import java.io.IOException; + +/** + * Responsible for serialization of currentKey, currentGroup and namespace. + * Will reuse the previous serialized currentKeyed if possible. + * @param type of the key. + */ +@NotThreadSafe +@Internal +class RocksDBSerializedCompositeKeyBuilder { + + /** The serializer for the key. */ + @Nonnull + private final TypeSerializer keySerializer; + + /** The output to write the key into. */ + @Nonnull + private final DataOutputSerializer keyOutView; + + /** The number of Key-group-prefix bytes for the key. */ + @Nonnegative + private final int keyGroupPrefixBytes; + + /** This flag indicates whether the key type has a variable byte size in serialization. */ + private final boolean keySerializerTypeVariableSized; + + /** Mark for the position after the serialized key. */ + @Nonnegative + private int afterKeyMark; + + public RocksDBSerializedCompositeKeyBuilder( + @Nonnull TypeSerializer keySerializer, + @Nonnegative int keyGroupPrefixBytes, + @Nonnegative int initialSize) { + this( + keySerializer, + new DataOutputSerializer(initialSize), + keyGroupPrefixBytes, + RocksDBKeySerializationUtils.isSerializerTypeVariableSized(keySerializer), + 0); + } + + @VisibleForTesting + RocksDBSerializedCompositeKeyBuilder( + @Nonnull TypeSerializer keySerializer, + @Nonnull DataOutputSerializer keyOutView, + @Nonnegative int keyGroupPrefixBytes, + boolean keySerializerTypeVariableSized, + @Nonnegative int afterKeyMark) { + this.keySerializer = keySerializer; + this.keyOutView = keyOutView; + this.keyGroupPrefixBytes = keyGroupPrefixBytes; + this.keySerializerTypeVariableSized = keySerializerTypeVariableSized; + this.afterKeyMark = afterKeyMark; + } + + /** + * Sets the key and key-group as prefix. This will serialize them into the buffer and the will be used to create + * composite keys with provided namespaces. + * + * @param key the key. + * @param keyGroupId the key-group id for the key. + */ + public void setKeyAndKeyGroup(@Nonnull K key, @Nonnegative int keyGroupId) { + try { + serializeKeyGroupAndKey(key, keyGroupId); + } catch (IOException shouldNeverHappen) { + throw new FlinkRuntimeException(shouldNeverHappen); + } + } + + /** + * Returns a serialized composite key, from the key and key-group provided in a previous call to + * {@link #setKeyAndKeyGroup(Object, int)} and the given namespace. + * + * @param namespace the namespace to concatenate for the serialized composite key bytes. + * @param namespaceSerializer the serializer to obtain the serialized form of the namespace. + * @param the type of the namespace. + * @return the bytes for the serialized composite key of key-group, key, namespace. + */ + @Nonnull + public byte[] buildCompositeKeyNamespace(@Nonnull N namespace, @Nonnull TypeSerializer namespaceSerializer) { + try { + serializeNamespace(namespace, namespaceSerializer); + final byte[] result = keyOutView.getCopyOfBuffer(); + resetToKey(); + return result; + } catch (IOException shouldNeverHappen) { + throw new FlinkRuntimeException(shouldNeverHappen); + } + } + + /** + * Returns a serialized composite key, from the key and key-group provided in a previous call to + * {@link #setKeyAndKeyGroup(Object, int)} and the given namespace, folloed by the given user-key. + * + * @param namespace the namespace to concatenate for the serialized composite key bytes. + * @param namespaceSerializer the serializer to obtain the serialized form of the namespace. + * @param userKey the user-key to concatenate for the serialized composite key, after the namespace. + * @param userKeySerializer the serializer to obtain the serialized form of the user-key. + * @param the type of the namespace. + * @param the type of the user-key. + * @return the bytes for the serialized composite key of key-group, key, namespace. + */ + @Nonnull + public byte[] buildCompositeKeyNamesSpaceUserKey( + @Nonnull N namespace, + @Nonnull TypeSerializer namespaceSerializer, + @Nonnull UK userKey, + @Nonnull TypeSerializer userKeySerializer) throws IOException { + serializeNamespace(namespace, namespaceSerializer); + userKeySerializer.serialize(userKey, keyOutView); + byte[] result = keyOutView.getCopyOfBuffer(); + resetToKey(); + return result; + } + + private void serializeKeyGroupAndKey(K key, int keyGroupId) throws IOException { + + // clear buffer and mark + resetFully(); + + // write key-group + RocksDBKeySerializationUtils.writeKeyGroup( + keyGroupId, + keyGroupPrefixBytes, + keyOutView); + // write key + keySerializer.serialize(key, keyOutView); + afterKeyMark = keyOutView.length(); + } + + private void serializeNamespace( + @Nonnull N namespace, + @Nonnull TypeSerializer namespaceSerializer) throws IOException { + + // this should only be called when there is already a key written so that we build the composite. + assert isKeyWritten(); + + final boolean ambiguousCompositeKeyPossible = isAmbiguousCompositeKeyPossible(namespaceSerializer); + if (ambiguousCompositeKeyPossible) { + RocksDBKeySerializationUtils.writeVariableIntBytes( + afterKeyMark - keyGroupPrefixBytes, + keyOutView); + } + RocksDBKeySerializationUtils.writeNameSpace( + namespace, + namespaceSerializer, + keyOutView, + ambiguousCompositeKeyPossible); + } + + private void resetFully() { + afterKeyMark = 0; + keyOutView.clear(); + } + + private void resetToKey() { + keyOutView.setPosition(afterKeyMark); + } + + private boolean isKeyWritten() { + return afterKeyMark > 0; + } + + @VisibleForTesting + boolean isAmbiguousCompositeKeyPossible(TypeSerializer namespaceSerializer) { + return keySerializerTypeVariableSized & + RocksDBKeySerializationUtils.isSerializerTypeVariableSized(namespaceSerializer); + } + + @VisibleForTesting + boolean isKeySerializerTypeVariableSized() { + return keySerializerTypeVariableSized; + } +} diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java index 97b83dfa49c7a6..a8f5163a7fd4c8 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java @@ -80,9 +80,9 @@ public TypeSerializer getValueSerializer() { @Override public V value() { try { - writeCurrentKeyWithGroupAndNamespace(); - byte[] key = dataOutputView.getCopyOfBuffer(); - byte[] valueBytes = backend.db.get(columnFamily, key); + byte[] valueBytes = backend.db.get(columnFamily, + serializeCurrentKeyWithGroupAndNamespace()); + if (valueBytes == null) { return getDefaultValue(); } @@ -101,11 +101,9 @@ public void update(V value) { } try { - writeCurrentKeyWithGroupAndNamespace(); - byte[] key = dataOutputView.getCopyOfBuffer(); - dataOutputView.clear(); - valueSerializer.serialize(value, dataOutputView); - backend.db.put(columnFamily, writeOptions, key, dataOutputView.getCopyOfBuffer()); + backend.db.put(columnFamily, writeOptions, + serializeCurrentKeyWithGroupAndNamespace(), + serializeValue(value)); } catch (Exception e) { throw new FlinkRuntimeException("Error while adding data to RocksDB", e); } diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBSerializedCompositeKeyBuilderTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBSerializedCompositeKeyBuilderTest.java new file mode 100644 index 00000000000000..70a4c3cfef79f6 --- /dev/null +++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBSerializedCompositeKeyBuilderTest.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.contrib.streaming.state; + +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.base.IntSerializer; +import org.apache.flink.api.common.typeutils.base.StringSerializer; +import org.apache.flink.core.memory.DataInputDeserializer; +import org.apache.flink.core.memory.DataOutputSerializer; +import org.apache.flink.runtime.state.KeyGroupRangeAssignment; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +/** + * Test for @{@link RocksDBSerializedCompositeKeyBuilder}. + */ +public class RocksDBSerializedCompositeKeyBuilderTest { + + private final DataOutputSerializer dataOutputSerializer = new DataOutputSerializer(128); + + private static final int[] TEST_PARALLELISMS = new int[]{64, 4096}; + private static final Collection TEST_INTS = Arrays.asList(42, 4711); + private static final Collection TEST_STRINGS = Arrays.asList("test123", "abc"); + + @Before + public void before() { + dataOutputSerializer.clear(); + } + + @Test + public void testSetKey() throws IOException { + for (int parallelism : TEST_PARALLELISMS) { + testSetKeyInternal(IntSerializer.INSTANCE, TEST_INTS, parallelism); + testSetKeyInternal(StringSerializer.INSTANCE, TEST_STRINGS, parallelism); + } + } + + @Test + public void testSetKeyNamespace() throws IOException { + for (int parallelism : TEST_PARALLELISMS) { + testSetKeyNamespaceInternal(IntSerializer.INSTANCE, IntSerializer.INSTANCE, TEST_INTS, TEST_INTS, parallelism); + testSetKeyNamespaceInternal(IntSerializer.INSTANCE, StringSerializer.INSTANCE, TEST_INTS, TEST_STRINGS, parallelism); + testSetKeyNamespaceInternal(StringSerializer.INSTANCE, IntSerializer.INSTANCE, TEST_STRINGS, TEST_INTS, parallelism); + testSetKeyNamespaceInternal(StringSerializer.INSTANCE, StringSerializer.INSTANCE, TEST_STRINGS, TEST_STRINGS, parallelism); + } + } + + @Test + public void testSetKeyNamespaceUserKey() throws IOException { + for (int parallelism : TEST_PARALLELISMS) { + testSetKeyNamespaceUserKeyInternal(IntSerializer.INSTANCE, IntSerializer.INSTANCE, IntSerializer.INSTANCE, TEST_INTS, TEST_INTS, TEST_INTS, parallelism); + testSetKeyNamespaceUserKeyInternal(IntSerializer.INSTANCE, StringSerializer.INSTANCE, IntSerializer.INSTANCE, TEST_INTS, TEST_STRINGS, TEST_INTS, parallelism); + testSetKeyNamespaceUserKeyInternal(StringSerializer.INSTANCE, IntSerializer.INSTANCE, IntSerializer.INSTANCE, TEST_STRINGS, TEST_INTS, TEST_INTS, parallelism); + testSetKeyNamespaceUserKeyInternal(StringSerializer.INSTANCE, StringSerializer.INSTANCE, IntSerializer.INSTANCE, TEST_STRINGS, TEST_STRINGS, TEST_INTS, parallelism); + testSetKeyNamespaceUserKeyInternal(IntSerializer.INSTANCE, IntSerializer.INSTANCE, StringSerializer.INSTANCE, TEST_INTS, TEST_INTS, TEST_STRINGS, parallelism); + testSetKeyNamespaceUserKeyInternal(IntSerializer.INSTANCE, StringSerializer.INSTANCE, StringSerializer.INSTANCE, TEST_INTS, TEST_STRINGS, TEST_STRINGS, parallelism); + testSetKeyNamespaceUserKeyInternal(StringSerializer.INSTANCE, IntSerializer.INSTANCE, StringSerializer.INSTANCE, TEST_STRINGS, TEST_INTS, TEST_STRINGS, parallelism); + testSetKeyNamespaceUserKeyInternal(StringSerializer.INSTANCE, StringSerializer.INSTANCE, StringSerializer.INSTANCE, TEST_STRINGS, TEST_STRINGS, TEST_STRINGS, parallelism); + } + } + + private void testSetKeyInternal(TypeSerializer serializer, Collection testKeys, int maxParallelism) throws IOException { + final int prefixBytes = maxParallelism > Byte.MAX_VALUE ? 2 : 1; + RocksDBSerializedCompositeKeyBuilder keyBuilder = + createRocksDBSerializedCompositeKeyBuilder(serializer, prefixBytes); + + final DataInputDeserializer deserializer = new DataInputDeserializer(); + for (K testKey : testKeys) { + int keyGroup = setKeyAndReturnKeyGroup(keyBuilder, testKey, maxParallelism); + byte[] result = dataOutputSerializer.getCopyOfBuffer(); + deserializer.setBuffer(result); + assertKeyKeyGroupBytes(testKey, keyGroup, prefixBytes, serializer, deserializer, false); + Assert.assertEquals(0, deserializer.available()); + } + } + + private void testSetKeyNamespaceInternal( + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + Collection testKeys, + Collection testNamespaces, + int maxParallelism) throws IOException { + final int prefixBytes = maxParallelism > Byte.MAX_VALUE ? 2 : 1; + + RocksDBSerializedCompositeKeyBuilder keyBuilder = + createRocksDBSerializedCompositeKeyBuilder(keySerializer, prefixBytes); + + final DataInputDeserializer deserializer = new DataInputDeserializer(); + + final boolean ambiguousPossible = keyBuilder.isAmbiguousCompositeKeyPossible(namespaceSerializer); + + for (K testKey : testKeys) { + int keyGroup = setKeyAndReturnKeyGroup(keyBuilder, testKey, maxParallelism); + for (N testNamespace : testNamespaces) { + byte[] compositeBytes = keyBuilder.buildCompositeKeyNamespace(testNamespace, namespaceSerializer); + deserializer.setBuffer(compositeBytes); + assertKeyGroupKeyNamespaceBytes( + testKey, + keyGroup, + prefixBytes, + keySerializer, + testNamespace, + namespaceSerializer, + deserializer, + ambiguousPossible); + Assert.assertEquals(0, deserializer.available()); + } + } + } + + private void testSetKeyNamespaceUserKeyInternal( + TypeSerializer keySerializer, + TypeSerializer namespaceSerializer, + TypeSerializer userKeySerializer, + Collection testKeys, + Collection testNamespaces, + Collection testUserKeys, + int maxParallelism) throws IOException { + final int prefixBytes = maxParallelism > Byte.MAX_VALUE ? 2 : 1; + + RocksDBSerializedCompositeKeyBuilder keyBuilder = + createRocksDBSerializedCompositeKeyBuilder(keySerializer, prefixBytes); + + final DataInputDeserializer deserializer = new DataInputDeserializer(); + + final boolean ambiguousPossible = keyBuilder.isAmbiguousCompositeKeyPossible(namespaceSerializer); + + for (K testKey : testKeys) { + int keyGroup = setKeyAndReturnKeyGroup(keyBuilder, testKey, maxParallelism); + for (N testNamespace : testNamespaces) { + for (U testUserKey : testUserKeys) { + byte[] compositeBytes = keyBuilder.buildCompositeKeyNamesSpaceUserKey( + testNamespace, + namespaceSerializer, + testUserKey, + userKeySerializer); + + deserializer.setBuffer(compositeBytes); + assertKeyGroupKeyNamespaceUserKeyBytes( + testKey, + keyGroup, + prefixBytes, + keySerializer, + testNamespace, + namespaceSerializer, + testUserKey, + userKeySerializer, + deserializer, + ambiguousPossible); + + Assert.assertEquals(0, deserializer.available()); + } + } + } + } + + private RocksDBSerializedCompositeKeyBuilder createRocksDBSerializedCompositeKeyBuilder( + TypeSerializer serializer, + int prefixBytes) { + final boolean variableSize = RocksDBKeySerializationUtils.isSerializerTypeVariableSized(serializer); + return new RocksDBSerializedCompositeKeyBuilder<>( + serializer, + dataOutputSerializer, + prefixBytes, + variableSize, + 0); + } + + private int setKeyAndReturnKeyGroup( + RocksDBSerializedCompositeKeyBuilder compositeKeyBuilder, + K key, + int maxParallelism) { + + int keyGroup = KeyGroupRangeAssignment.assignKeyToParallelOperator(key, maxParallelism, maxParallelism); + compositeKeyBuilder.setKeyAndKeyGroup(key, keyGroup); + return keyGroup; + } + + private void assertKeyKeyGroupBytes( + K key, + int keyGroup, + int prefixBytes, + TypeSerializer typeSerializer, + DataInputDeserializer deserializer, + boolean ambiguousCompositeKeyPossible) throws IOException { + + Assert.assertEquals(keyGroup, RocksDBKeySerializationUtils.readKeyGroup(prefixBytes, deserializer)); + Assert.assertEquals(key, RocksDBKeySerializationUtils.readKey(typeSerializer, deserializer, ambiguousCompositeKeyPossible)); + } + + private void assertKeyGroupKeyNamespaceBytes( + K key, + int keyGroup, + int prefixBytes, + TypeSerializer keySerializer, + N namespace, + TypeSerializer namespaceSerializer, + DataInputDeserializer deserializer, + boolean ambiguousCompositeKeyPossible) throws IOException { + assertKeyKeyGroupBytes(key, keyGroup, prefixBytes, keySerializer, deserializer, ambiguousCompositeKeyPossible); + N readNamespace = + RocksDBKeySerializationUtils.readNamespace(namespaceSerializer, deserializer, ambiguousCompositeKeyPossible); + Assert.assertEquals(namespace, readNamespace); + } + + private void assertKeyGroupKeyNamespaceUserKeyBytes( + K key, + int keyGroup, + int prefixBytes, + TypeSerializer keySerializer, + N namespace, + TypeSerializer namespaceSerializer, + U userKey, + TypeSerializer userKeySerializer, + DataInputDeserializer deserializer, + boolean ambiguousCompositeKeyPossible) throws IOException { + assertKeyGroupKeyNamespaceBytes( + key, + keyGroup, + prefixBytes, + keySerializer, + namespace, + namespaceSerializer, + deserializer, + ambiguousCompositeKeyPossible); + Assert.assertEquals(userKey, userKeySerializer.deserialize(deserializer)); + } +} From ec48beaa2c86b6e1d6c09d4be8bc3564dcdd0ead Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 7 Jan 2019 13:47:32 +0100 Subject: [PATCH 037/147] =?UTF-8?q?[FLINK-11265][docs]=20AvroSinkWriter=20?= =?UTF-8?q?=E2=86=92=20AvroKeyValueSinkWriter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../flink/streaming/connectors/fs/AvroKeyValueSinkWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/AvroKeyValueSinkWriter.java b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/AvroKeyValueSinkWriter.java index 0f73e8c5c0bf3c..3294b55a0e0185 100644 --- a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/AvroKeyValueSinkWriter.java +++ b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/AvroKeyValueSinkWriter.java @@ -59,7 +59,7 @@ properties.put(AvroKeyValueSinkWriter.CONF_COMPRESS, Boolean.toString(true)); properties.put(AvroKeyValueSinkWriter.CONF_COMPRESS_CODEC, DataFileConstants.SNAPPY_CODEC); - sink.setWriter(new AvroSinkWriter(properties)); + sink.setWriter(new AvroKeyValueSinkWriter(properties)); sink.setBatchSize(1024 * 1024 * 64); // this is 64 MB, } From 11aa9b6d74f3ff8951b3dd8f4a33a87810cb1649 Mon Sep 17 00:00:00 2001 From: Tzu-Li Chen Date: Mon, 7 Jan 2019 21:08:38 +0800 Subject: [PATCH 038/147] [hotfix][runtime] Remove redundant suppression --- .../flink/runtime/registration/RegisteredRpcConnection.java | 1 - 1 file changed, 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/registration/RegisteredRpcConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/registration/RegisteredRpcConnection.java index 7d2c35a7a4b908..8c78ede8e11798 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/registration/RegisteredRpcConnection.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/registration/RegisteredRpcConnection.java @@ -86,7 +86,6 @@ public RegisteredRpcConnection(Logger log, String targetAddress, F fencingToken, // Life cycle // ------------------------------------------------------------------------ - @SuppressWarnings("unchecked") public void start() { checkState(!closed, "The RPC connection is already closed"); checkState(!isConnected() && pendingRegistration == null, "The RPC connection is already started"); From 1794ea06bd351a5b3c4219d4097b2516c0298447 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Thu, 29 Nov 2018 16:32:01 +0800 Subject: [PATCH 039/147] [FLINK-11073] [core] Add COMPATIBLE_WITH_RECONFIGURED_SERIALIZER to TypeSerializerSchemaCompatibility --- .../TypeSerializerSchemaCompatibility.java | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSchemaCompatibility.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSchemaCompatibility.java index d488799cfce5c5..4bb4aa0d8a261e 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSchemaCompatibility.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSchemaCompatibility.java @@ -56,6 +56,12 @@ enum Type { */ COMPATIBLE_AFTER_MIGRATION, + /** + * This indicates that a reconfigured version of the new serializer + * is compatible, and should be used instead of the original new serializer. + */ + COMPATIBLE_WITH_RECONFIGURED_SERIALIZER, + /** * This indicates that the new serializer is incompatible, even with migration. * This normally implies that the deserialized Java class can not be commonly recognized @@ -69,6 +75,8 @@ enum Type { */ private final Type resultType; + private final TypeSerializer reconfiguredNewSerializer; + /** * Returns a result that indicates that the new serializer is compatible and no migration is required. * The new serializer can continued to be used as is. @@ -89,6 +97,20 @@ public static TypeSerializerSchemaCompatibility compatibleAfterMigration( return new TypeSerializerSchemaCompatibility<>(Type.COMPATIBLE_AFTER_MIGRATION, null); } + /** + * Returns a result that indicates a reconfigured version of the new serializer is compatible, and should be + * used instead of the original new serializer. + * + * @param reconfiguredSerializer the reconfigured version of the new serializer. + * @return a result that indicates a reconfigured version of the new serializer is compatible, and should be + * used instead of the original new serializer. + */ + public static TypeSerializerSchemaCompatibility compatibleWithReconfiguredSerializer(TypeSerializer reconfiguredSerializer) { + return new TypeSerializerSchemaCompatibility<>( + Type.COMPATIBLE_WITH_RECONFIGURED_SERIALIZER, + Preconditions.checkNotNull(reconfiguredSerializer)); + } + /** * Returns a result that indicates there is no possible way for the new serializer to be use-able. * This normally indicates that there is no common Java class between what the previous bytes can be @@ -105,6 +127,7 @@ public static TypeSerializerSchemaCompatibility incompatible() { private TypeSerializerSchemaCompatibility(Type resultType, @Nullable TypeSerializer reconfiguredNewSerializer) { this.resultType = Preconditions.checkNotNull(resultType); + this.reconfiguredNewSerializer = reconfiguredNewSerializer; } /** @@ -125,6 +148,27 @@ public boolean isCompatibleAfterMigration() { return resultType == Type.COMPATIBLE_AFTER_MIGRATION; } + /** + * Returns whether or not the type of the compatibility is {@link Type#COMPATIBLE_WITH_RECONFIGURED_SERIALIZER}. + * + * @return whether or not the type of the compatibility is {@link Type#COMPATIBLE_WITH_RECONFIGURED_SERIALIZER}. + */ + public boolean isCompatibleWithReconfiguredSerializer() { + return resultType == Type.COMPATIBLE_WITH_RECONFIGURED_SERIALIZER; + } + + /** + * Gets the reconfigured serializer. This throws an exception if + * {@link #isCompatibleWithReconfiguredSerializer()} is {@code false}. + */ + public TypeSerializer getReconfiguredSerializer() { + Preconditions.checkState( + isCompatibleWithReconfiguredSerializer(), + "It is only possible to get a reconfigured serializer if the compatibility type is %s, but the type is %s", + Type.COMPATIBLE_WITH_RECONFIGURED_SERIALIZER, resultType); + return reconfiguredNewSerializer; + } + /** * Returns whether or not the type of the compatibility is {@link Type#INCOMPATIBLE}. * From e03cea5431dc63f603469bdbe8c499f8819e3cbd Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Thu, 13 Dec 2018 19:22:46 +0800 Subject: [PATCH 040/147] [FLINK-11073] [state backends] Respect serializer reconfiguration in state backends This commit adapts the StateSerializerProvider to respect the COMPATIBLE_WITH_RECONFIGURED_SERIALIZER case, which effectively lets all state backends respect serializer reconfiguration because state serializers are always obtained via StateSerializerProviders. It also makes StateSerializerProvider work for both eagerly / lazily registered serializers. This is required so that the StateSerializerProvider can be used for the key serializer in AbstractKeyedStateBackend. For the key serializer, regardless of whether or not we're restoring old state, we always first get the new key serializer, and then maybe get the previous key serializer's snapshot in the restore phase. Therefore, after this commit, access the key serializer in AbstractKeyedStateBackend is also governed by a StateSerializerProvider. For a more fine-grained explaination of all the changes that build up to this, please refer to #7329. This closes #7329. --- .../state/AbstractKeyedStateBackend.java | 18 +- ...gisteredBroadcastStateBackendMetaInfo.java | 8 +- ...egisteredKeyValueStateBackendMetaInfo.java | 12 +- ...egisteredOperatorStateBackendMetaInfo.java | 4 +- ...eredPriorityQueueStateBackendMetaInfo.java | 4 +- .../state/StateSerializerProvider.java | 253 ++++-- .../state/heap/HeapKeyedStateBackend.java | 11 +- .../state/StateBackendMigrationTestBase.java | 753 ++++++++++-------- .../state/StateSerializerProviderTest.java | 136 +++- .../testutils/statemigration/TestType.java | 23 + .../V1TestTypeSerializerSnapshot.java | 7 +- .../V2TestTypeSerializerSnapshot.java | 11 +- .../state/RocksDBKeyedStateBackend.java | 43 +- 13 files changed, 820 insertions(+), 463 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java index 19de2102a35532..7422e654465be8 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/AbstractKeyedStateBackend.java @@ -23,6 +23,8 @@ import org.apache.flink.api.common.state.State; import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; import org.apache.flink.core.fs.CloseableRegistry; import org.apache.flink.runtime.checkpoint.CheckpointOptions; import org.apache.flink.runtime.query.TaskKvStateRegistry; @@ -52,8 +54,8 @@ public abstract class AbstractKeyedStateBackend implements Closeable, CheckpointListener { - /** {@link TypeSerializer} for our key. */ - protected final TypeSerializer keySerializer; + /** {@link StateSerializerProvider} for our key serializer. */ + private final StateSerializerProvider keySerializerProvider; /** The currently active key. */ private K currentKey; @@ -104,7 +106,7 @@ public AbstractKeyedStateBackend( Preconditions.checkArgument(numberOfKeyGroups >= keyGroupRange.getNumberOfKeyGroups(), "The total number of key groups must be at least the number in the key group range assigned to this backend"); this.kvStateRegistry = kvStateRegistry; - this.keySerializer = Preconditions.checkNotNull(keySerializer); + this.keySerializerProvider = StateSerializerProvider.fromNewRegisteredSerializer(keySerializer); this.numberOfKeyGroups = numberOfKeyGroups; this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader); this.keyGroupRange = Preconditions.checkNotNull(keyGroupRange); @@ -156,7 +158,13 @@ public void setCurrentKey(K newKey) { */ @Override public TypeSerializer getKeySerializer() { - return keySerializer; + return keySerializerProvider.currentSchemaSerializer(); + } + + public TypeSerializerSchemaCompatibility checkKeySerializerSchemaCompatibility( + TypeSerializerSnapshot previousKeySerializerSnapshot) { + + return keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(previousKeySerializerSnapshot); } /** @@ -230,7 +238,7 @@ public S getOrCreateKeyedState( final TypeSerializer namespaceSerializer, StateDescriptor stateDescriptor) throws Exception { checkNotNull(namespaceSerializer, "Namespace serializer"); - checkNotNull(keySerializer, "State key serializer has not been configured in the config. " + + checkNotNull(keySerializerProvider, "State key serializer has not been configured in the config. " + "This operation cannot use partitioned state."); InternalKvState kvState = keyValueStatesByName.get(stateDescriptor.getName()); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastStateBackendMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastStateBackendMetaInfo.java index ecc13faa43d229..e44559a2571092 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastStateBackendMetaInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastStateBackendMetaInfo.java @@ -55,8 +55,8 @@ public RegisteredBroadcastStateBackendMetaInfo( this( name, assignmentMode, - StateSerializerProvider.fromNewState(keySerializer), - StateSerializerProvider.fromNewState(valueSerializer)); + StateSerializerProvider.fromNewRegisteredSerializer(keySerializer), + StateSerializerProvider.fromNewRegisteredSerializer(valueSerializer)); } public RegisteredBroadcastStateBackendMetaInfo(@Nonnull RegisteredBroadcastStateBackendMetaInfo copy) { @@ -73,10 +73,10 @@ public RegisteredBroadcastStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot sn snapshot.getName(), OperatorStateHandle.Mode.valueOf( snapshot.getOption(StateMetaInfoSnapshot.CommonOptionsKeys.OPERATOR_STATE_DISTRIBUTION_MODE)), - StateSerializerProvider.fromRestoredState( + StateSerializerProvider.fromPreviousSerializerSnapshot( (TypeSerializerSnapshot) Preconditions.checkNotNull( snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.KEY_SERIALIZER))), - StateSerializerProvider.fromRestoredState( + StateSerializerProvider.fromPreviousSerializerSnapshot( (TypeSerializerSnapshot) Preconditions.checkNotNull( snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER)))); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyValueStateBackendMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyValueStateBackendMetaInfo.java index b37c79de0264eb..b2d1cdc49ff701 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyValueStateBackendMetaInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyValueStateBackendMetaInfo.java @@ -60,8 +60,8 @@ public RegisteredKeyValueStateBackendMetaInfo( this( stateType, name, - StateSerializerProvider.fromNewState(namespaceSerializer), - StateSerializerProvider.fromNewState(stateSerializer), + StateSerializerProvider.fromNewRegisteredSerializer(namespaceSerializer), + StateSerializerProvider.fromNewRegisteredSerializer(stateSerializer), null); } @@ -75,8 +75,8 @@ public RegisteredKeyValueStateBackendMetaInfo( this( stateType, name, - StateSerializerProvider.fromNewState(namespaceSerializer), - StateSerializerProvider.fromNewState(stateSerializer), + StateSerializerProvider.fromNewRegisteredSerializer(namespaceSerializer), + StateSerializerProvider.fromNewRegisteredSerializer(stateSerializer), snapshotTransformer); } @@ -85,10 +85,10 @@ public RegisteredKeyValueStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot sna this( StateDescriptor.Type.valueOf(snapshot.getOption(StateMetaInfoSnapshot.CommonOptionsKeys.KEYED_STATE_TYPE)), snapshot.getName(), - StateSerializerProvider.fromRestoredState( + StateSerializerProvider.fromPreviousSerializerSnapshot( (TypeSerializerSnapshot) Preconditions.checkNotNull( snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER))), - StateSerializerProvider.fromRestoredState( + StateSerializerProvider.fromPreviousSerializerSnapshot( (TypeSerializerSnapshot) Preconditions.checkNotNull( snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))), null); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorStateBackendMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorStateBackendMetaInfo.java index 921947a4dd0bce..6b83ca2693d4b2 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorStateBackendMetaInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorStateBackendMetaInfo.java @@ -56,7 +56,7 @@ public RegisteredOperatorStateBackendMetaInfo( @Nonnull OperatorStateHandle.Mode assignmentMode) { this( name, - StateSerializerProvider.fromNewState(partitionStateSerializer), + StateSerializerProvider.fromNewRegisteredSerializer(partitionStateSerializer), assignmentMode); } @@ -71,7 +71,7 @@ private RegisteredOperatorStateBackendMetaInfo(@Nonnull RegisteredOperatorStateB public RegisteredOperatorStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot snapshot) { this( snapshot.getName(), - StateSerializerProvider.fromRestoredState( + StateSerializerProvider.fromPreviousSerializerSnapshot( (TypeSerializerSnapshot) Preconditions.checkNotNull( snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))), OperatorStateHandle.Mode.valueOf( diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredPriorityQueueStateBackendMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredPriorityQueueStateBackendMetaInfo.java index 961d96fa40577d..691e74c0288643 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredPriorityQueueStateBackendMetaInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredPriorityQueueStateBackendMetaInfo.java @@ -42,14 +42,14 @@ public RegisteredPriorityQueueStateBackendMetaInfo( @Nonnull String name, @Nonnull TypeSerializer elementSerializer) { - this(name, StateSerializerProvider.fromNewState(elementSerializer)); + this(name, StateSerializerProvider.fromNewRegisteredSerializer(elementSerializer)); } @SuppressWarnings("unchecked") public RegisteredPriorityQueueStateBackendMetaInfo(StateMetaInfoSnapshot snapshot) { this( snapshot.getName(), - StateSerializerProvider.fromRestoredState( + StateSerializerProvider.fromPreviousSerializerSnapshot( (TypeSerializerSnapshot) Preconditions.checkNotNull( snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER)))); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java index a24f12e42fbee9..bad37ff87078c4 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java @@ -35,6 +35,21 @@ * A {@link StateSerializerProvider} wraps logic on how to obtain serializers for registered state, * either with the previous schema of state in checkpoints or the current schema of state. * + *

A provider can be created from either a registered state serializer, or the snapshot + * of the previous state serializer. For the former case, if the state was restored and a + * snapshot of the previous state serializer was retrieved later on, the snapshot can be set + * on the provider which also additionally checks the compatibility of the initially registered + * serializer. Similarly for the latter case, if a new state serializer is registered later on, + * it can be set on the provider, which then also checks the compatibility of the new registered + * serializer. + * + *

Simply put, the provider works both directions - either creating it first with a registered + * serializer or the previous serializer's snapshot, and then setting the previous serializer's + * snapshot (if the provider was created with a registered serializer) or a new registered state + * serializer (if the provider was created with a serializer snapshot). Either way, + * the new registered serializer is checked for schema compatibility once both the new serializer + * and the previous serializer snapshot is present. + * * @param the type of the state. */ @Internal @@ -44,12 +59,36 @@ public abstract class StateSerializerProvider { * The registered serializer for the state. * *

In the case that this provider was created from a restored serializer snapshot via - * {@link #fromRestoredState(TypeSerializerSnapshot)}, but a new serializer was never registered + * {@link #fromPreviousSerializerSnapshot(TypeSerializerSnapshot)}, but a new serializer was never registered * for the state (i.e., this is the case if a restored state was never accessed), this would be {@code null}. */ @Nullable TypeSerializer registeredSerializer; + /** + * The state's previous serializer's snapshot. + * + *

In the case that this provider was created from a registered state serializer instance via + * {@link #fromNewRegisteredSerializer(TypeSerializer)}, but a serializer snapshot was never supplied to this + * provider (i.e. because the registered serializer was for a new state, not a restored one), this + * would be {@code null}. + */ + @Nullable + TypeSerializerSnapshot previousSerializerSnapshot; + + /** + * The restore serializer, lazily created only when the restore serializer is accessed. + * + *

NOTE: It is important to only create this lazily, so that off-heap + * state do not fail eagerly when restoring state that has a + * {@link UnloadableDummyTypeSerializer} as the previous serializer. This should + * be relevant only for restores from Flink versions prior to 1.7.x. + */ + @Nullable + private TypeSerializer cachedRestoredSerializer; + + private boolean isRegisteredWithIncompatibleSerializer = false; + /** * Creates a {@link StateSerializerProvider} for restored state from the previous serializer's snapshot. * @@ -59,26 +98,36 @@ public abstract class StateSerializerProvider { * @param stateSerializerSnapshot the previous serializer's snapshot. * @param the type of the state. * - * @return a new {@link StateSerializerProvider} for restored state. + * @return a new {@link StateSerializerProvider}. */ - public static StateSerializerProvider fromRestoredState(TypeSerializerSnapshot stateSerializerSnapshot) { - return new RestoredStateSerializerProvider<>(stateSerializerSnapshot); + public static StateSerializerProvider fromPreviousSerializerSnapshot(TypeSerializerSnapshot stateSerializerSnapshot) { + return new LazilyRegisteredStateSerializerProvider<>(stateSerializerSnapshot); } /** - * Creates a {@link StateSerializerProvider} for new state from the registered state serializer. + * Creates a {@link StateSerializerProvider} from the registered state serializer. + * + *

If the state is a restored one, and the previous serializer's snapshot is + * obtained later on, is should be supplied via the + * {@link #setPreviousSerializerSnapshotForRestoredState(TypeSerializerSnapshot)} method. * * @param registeredStateSerializer the new state's registered serializer. * @param the type of the state. * - * @return a new {@link StateSerializerProvider} for new state. + * @return a new {@link StateSerializerProvider}. */ - public static StateSerializerProvider fromNewState(TypeSerializer registeredStateSerializer) { - return new NewStateSerializerProvider<>(registeredStateSerializer); + public static StateSerializerProvider fromNewRegisteredSerializer(TypeSerializer registeredStateSerializer) { + return new EagerlyRegisteredStateSerializerProvider<>(registeredStateSerializer); } - private StateSerializerProvider(@Nullable TypeSerializer stateSerializer) { + private StateSerializerProvider(@Nonnull TypeSerializer stateSerializer) { this.registeredSerializer = stateSerializer; + this.previousSerializerSnapshot = null; + } + + private StateSerializerProvider(@Nonnull TypeSerializerSnapshot previousSerializerSnapshot) { + this.previousSerializerSnapshot = previousSerializerSnapshot; + this.registeredSerializer = null; } /** @@ -92,36 +141,63 @@ private StateSerializerProvider(@Nullable TypeSerializer stateSerializer) { * identical. Therefore, in this case, it is guaranteed that the serializer returned by * this method is the same as the one returned by {@link #previousSchemaSerializer()}. * - *

If this provider was created from new state, then this always returns the - * serializer that the new state was registered with. + *

If this provider was created from a serializer instance, then this always returns the + * that same serializer instance. If later on a snapshot of the previous serializer is supplied + * via {@link #setPreviousSerializerSnapshotForRestoredState(TypeSerializerSnapshot)}, then + * the initially supplied serializer instance will be checked for compatibility. * * @return a serializer that reads and writes in the current schema of the state. */ @Nonnull - public abstract TypeSerializer currentSchemaSerializer(); + public final TypeSerializer currentSchemaSerializer() { + if (registeredSerializer != null) { + checkState( + !isRegisteredWithIncompatibleSerializer, + "Unable to provide a serializer with the current schema, because the restored state was " + + "registered with a new serializer that has incompatible schema."); + + return registeredSerializer; + } + + // if we are not yet registered with a new serializer, + // we can just use the restore serializer to read / write the state. + return previousSchemaSerializer(); + }; /** * Gets the serializer that recognizes the previous serialization schema of the state. * This is the serializer that should be used for restoring the state, i.e. when the state * is still in the previous serialization schema. * - *

This method can only be used if this provider was created from a restored state's serializer - * snapshot. If this provider was created from new state, then this method is - * irrelevant, since there doesn't exist any previous version of the state schema. + *

This method only returns a serializer if this provider has the previous serializer's + * snapshot. Otherwise, trying to access the previous schema serializer will fail + * with an exception. * * @return a serializer that reads and writes in the previous schema of the state. */ @Nonnull - public abstract TypeSerializer previousSchemaSerializer(); + public final TypeSerializer previousSchemaSerializer() { + if (cachedRestoredSerializer != null) { + return cachedRestoredSerializer; + } + + if (previousSerializerSnapshot == null) { + throw new UnsupportedOperationException( + "This provider does not contain the state's previous serializer's snapshot. Cannot provider a serializer for previous schema."); + } + + this.cachedRestoredSerializer = previousSerializerSnapshot.restoreSerializer(); + return cachedRestoredSerializer; + }; /** * For restored state, register a new serializer that potentially has a new serialization schema. * *

Users are allowed to register serializers for state only once. Therefore, this method - * is irrelevant if this provider was created from new state, since a state serializer had + * is irrelevant if this provider was created with a serializer instance, since a state serializer had * been registered already. * - *

For the case where this provider was created from restored state, then this method should + *

For the case where this provider was created from a serializer snapshot, then this method should * be called at most once. The new serializer will be checked for its schema compatibility with the * previous serializer's schema, and returned to the caller. The caller is responsible for * checking the result and react appropriately to it, as follows: @@ -143,52 +219,61 @@ private StateSerializerProvider(@Nullable TypeSerializer stateSerializer) { public abstract TypeSerializerSchemaCompatibility registerNewSerializerForRestoredState(TypeSerializer newSerializer); /** - * Implementation of the {@link StateSerializerProvider} for the restored state case. + * For restored state, set the state's previous serializer's snapshot. + * + *

Users are allowed to set the previous serializer's snapshot once. Therefore, this method + * is irrelevant if this provider was created with a serializer snapshot, since the serializer + * snapshot had been set already. + * + *

For the case where this provider was created from a serializer instance, then this method should + * be called at most once. The initially registered state serializer will be checked for its + * schema compatibility with the previous serializer's schema, and returned to the caller. + * The caller is responsible for checking the result and react appropriately to it, as follows: + *

    + *
  • {@link TypeSerializerSchemaCompatibility#isCompatibleAsIs()}: nothing needs to be done. + * {@link #currentSchemaSerializer()} remains to return the initially registered serializer.
    • + *
  • {@link TypeSerializerSchemaCompatibility#isCompatibleAfterMigration()} ()}: state needs to be + * migrated before the serializer returned by {@link #currentSchemaSerializer()} can be used. + * The migration should be performed by reading the state with {@link #previousSchemaSerializer()}, + * and then writing it again with {@link #currentSchemaSerializer()}.
    • + *
  • {@link TypeSerializerSchemaCompatibility#isIncompatible()}: the registered serializer is + * incompatible. {@link #currentSchemaSerializer()} can no longer return a serializer for + * the state, and therefore this provider shouldn't be used anymore.
    • + *
+ * + * @param previousSerializerSnapshot the state's previous serializer's snapshot + * + * @return the schema compatibility of the initially registered serializer, with respect to the previous serializer. */ - private static class RestoredStateSerializerProvider extends StateSerializerProvider { - - /** - * The snapshot of the previous serializer of the state. - */ - @Nonnull - private final TypeSerializerSnapshot previousSerializerSnapshot; - - private boolean isRegisteredWithIncompatibleSerializer = false; - - RestoredStateSerializerProvider(TypeSerializerSnapshot previousSerializerSnapshot) { - super(null); - this.previousSerializerSnapshot = Preconditions.checkNotNull(previousSerializerSnapshot); - } - - /** - * The restore serializer, lazily created only when the restore serializer is accessed. - * - *

NOTE: It is important to only create this lazily, so that off-heap - * state do not fail eagerly when restoring state that has a - * {@link UnloadableDummyTypeSerializer} as the previous serializer. This should - * be relevant only for restores from Flink versions prior to 1.7.x. - */ - @Nullable - private TypeSerializer cachedRestoredSerializer; + @Nonnull + public abstract TypeSerializerSchemaCompatibility setPreviousSerializerSnapshotForRestoredState(TypeSerializerSnapshot previousSerializerSnapshot); - @Override - @Nonnull - public TypeSerializer currentSchemaSerializer() { - if (registeredSerializer != null) { - checkState( - !isRegisteredWithIncompatibleSerializer, - "Unable to provide a serializer with the current schema, because the restored state was " + - "registered with a new serializer that has incompatible schema."); + /** + * Invalidates access to the current schema serializer. This lets {@link #currentSchemaSerializer()} + * fail when invoked. + * + *

Access to the current schema serializer should be invalidated by the methods + * {@link #registerNewSerializerForRestoredState(TypeSerializer)} or + * {@link #setPreviousSerializerSnapshotForRestoredState(TypeSerializerSnapshot)} + * once the registered serializer is determined to be incompatible. + */ + protected final void invalidateCurrentSchemaSerializerAccess() { + this.isRegisteredWithIncompatibleSerializer = true; + } - return registeredSerializer; - } + /** + * Implementation of the {@link StateSerializerProvider} for the case where a snapshot of the + * previous serializer is obtained before a new state serializer is registered (hence, the naming "lazily" registered). + */ + private static class LazilyRegisteredStateSerializerProvider extends StateSerializerProvider { - // if we are not yet registered with a new serializer, - // we can just use the restore serializer to read / write the state. - return previousSchemaSerializer(); + LazilyRegisteredStateSerializerProvider(TypeSerializerSnapshot previousSerializerSnapshot) { + super(Preconditions.checkNotNull(previousSerializerSnapshot)); } @Nonnull + @Override + @SuppressWarnings("ConstantConditions") public TypeSerializerSchemaCompatibility registerNewSerializerForRestoredState(TypeSerializer newSerializer) { checkNotNull(newSerializer); if (registeredSerializer != null) { @@ -197,49 +282,59 @@ public TypeSerializerSchemaCompatibility registerNewSerializerForRestoredStat TypeSerializerSchemaCompatibility result = previousSerializerSnapshot.resolveSchemaCompatibility(newSerializer); if (result.isIncompatible()) { - this.isRegisteredWithIncompatibleSerializer = true; + invalidateCurrentSchemaSerializerAccess(); + } + if (result.isCompatibleWithReconfiguredSerializer()) { + this.registeredSerializer = result.getReconfiguredSerializer(); + } else { + this.registeredSerializer = newSerializer; } - this.registeredSerializer = newSerializer; return result; } @Nonnull - public final TypeSerializer previousSchemaSerializer() { - if (cachedRestoredSerializer != null) { - return cachedRestoredSerializer; - } - - this.cachedRestoredSerializer = previousSerializerSnapshot.restoreSerializer(); - return cachedRestoredSerializer; + @Override + public TypeSerializerSchemaCompatibility setPreviousSerializerSnapshotForRestoredState( + TypeSerializerSnapshot previousSerializerSnapshot) { + throw new UnsupportedOperationException("The snapshot of the state's previous serializer has already been set; cannot reset."); } } /** - * Implementation of the {@link StateSerializerProvider} for the new state case. + * Implementation of the {@link StateSerializerProvider} for the case where a new state + * serializer instance is registered first, before any snapshots of the previous state serializer + * is obtained (hence, the naming "eagerly" registered). */ - private static class NewStateSerializerProvider extends StateSerializerProvider { + private static class EagerlyRegisteredStateSerializerProvider extends StateSerializerProvider { - NewStateSerializerProvider(TypeSerializer registeredStateSerializer) { + EagerlyRegisteredStateSerializerProvider(TypeSerializer registeredStateSerializer) { super(Preconditions.checkNotNull(registeredStateSerializer)); } - @Override @Nonnull - @SuppressWarnings("ConstantConditions") - public TypeSerializer currentSchemaSerializer() { - return registeredSerializer; - } - @Override - @Nonnull public TypeSerializerSchemaCompatibility registerNewSerializerForRestoredState(TypeSerializer newSerializer) { throw new UnsupportedOperationException("A serializer has already been registered for the state; re-registration is not allowed."); } - @Override @Nonnull - public TypeSerializer previousSchemaSerializer() { - throw new UnsupportedOperationException("This is a NewStateSerializerProvider; you cannot get a restore serializer because there was no restored state."); + @Override + public TypeSerializerSchemaCompatibility setPreviousSerializerSnapshotForRestoredState(TypeSerializerSnapshot previousSerializerSnapshot) { + checkNotNull(previousSerializerSnapshot); + if (this.previousSerializerSnapshot != null) { + throw new UnsupportedOperationException("The snapshot of the state's previous serializer has already been set; cannot reset."); + } + + this.previousSerializerSnapshot = previousSerializerSnapshot; + + TypeSerializerSchemaCompatibility result = previousSerializerSnapshot.resolveSchemaCompatibility(registeredSerializer); + if (result.isIncompatible()) { + invalidateCurrentSchemaSerializerAccess(); + } + if (result.isCompatibleWithReconfiguredSerializer()) { + this.registeredSerializer = result.getReconfiguredSerializer(); + } + return result; } } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java index 3f8761b657a3fe..c95741429d85e0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java @@ -243,7 +243,7 @@ private StateTable tryRegisterStateTable( TypeSerializerSchemaCompatibility namespaceCompatibility = restoredKvMetaInfo.updateNamespaceSerializer(namespaceSerializer); - if (!namespaceCompatibility.isCompatibleAsIs()) { + if (namespaceCompatibility.isCompatibleAfterMigration() || namespaceCompatibility.isIncompatible()) { throw new StateMigrationException("For heap backends, the new namespace serializer must be compatible."); } @@ -302,7 +302,7 @@ public IS createInternalState( } StateTable stateTable = tryRegisterStateTable( namespaceSerializer, stateDesc, getStateSnapshotTransformer(stateDesc, snapshotTransformFactory)); - return stateFactory.createState(stateDesc, stateTable, keySerializer); + return stateFactory.createState(stateDesc, stateTable, getKeySerializer()); } @SuppressWarnings("unchecked") @@ -394,8 +394,9 @@ private void restorePartitionedState(Collection state) throws if (!keySerializerRestored) { // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible - if (!serializationProxy.getKeySerializerConfigSnapshot() - .resolveSchemaCompatibility(keySerializer).isCompatibleAsIs()) { + TypeSerializerSchemaCompatibility keySerializerSchemaCompat = + checkKeySerializerSchemaCompatibility(serializationProxy.getKeySerializerConfigSnapshot()); + if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) { throw new StateMigrationException("The new key serializer must be compatible."); } @@ -700,7 +701,7 @@ public RunnableFuture> snapshot( new KeyedBackendSerializationProxy<>( // TODO: this code assumes that writing a serializer is threadsafe, we should support to // get a serialized form already at state registration time in the future - keySerializer, + getKeySerializer(), metaInfoSnapshots, !Objects.equals(UncompressedStreamCompressionDecorator.INSTANCE, keyGroupCompressionDecorator)); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java index 5511792673e9b7..8d85e74aca7eca 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java @@ -40,6 +40,7 @@ import org.apache.flink.util.TestLogger; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -69,23 +70,70 @@ public abstract class StateBackendMigrationTestBase( + stateName, + new TestType.V1TestTypeSerializer()), + new ValueStateDescriptor<>( + stateName, + // restore with a V2 serializer that has a different schema + new TestType.V2TestTypeSerializer())); + } + + @Test + public void testKeyedValueStateSerializerReconfiguration() throws Exception { + final String stateName = "test-name"; + + testKeyedValueStateUpgrade( + new ValueStateDescriptor<>( + stateName, + new TestType.V1TestTypeSerializer()), + new ValueStateDescriptor<>( + stateName, + // the test fails if this serializer is used instead of a reconfigured new serializer + new TestType.ReconfigurationRequiringTestTypeSerializer())); + } + + @Test + public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() throws Exception { + final String stateName = "test-name"; + + try { + testKeyedValueStateUpgrade( + new ValueStateDescriptor<>( + stateName, + new TestType.V1TestTypeSerializer()), + new ValueStateDescriptor<>( + stateName, + new TestType.IncompatibleTestTypeSerializer())); + + Assert.fail("should have failed"); + } catch (Exception expected) { + Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent()); + } + } + + private void testKeyedValueStateUpgrade( + ValueStateDescriptor initialAccessDescriptor, + ValueStateDescriptor newAccessDescriptorAfterRestore) throws Exception { + CheckpointStreamFactory streamFactory = createStreamFactory(); SharedStateRegistry sharedStateRegistry = new SharedStateRegistry(); - final String stateName = "test-name"; AbstractKeyedStateBackend backend = createKeyedBackend(IntSerializer.INSTANCE); try { - ValueStateDescriptor kvId = new ValueStateDescriptor<>( - stateName, - new TestType.V1TestTypeSerializer()); - ValueState valueState = backend - .getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); + ValueState valueState = backend.getPartitionedState( + VoidNamespace.INSTANCE, + CustomVoidNamespaceSerializer.INSTANCE, + initialAccessDescriptor); backend.setCurrentKey(1); valueState.update(new TestType("foo", 1456)); @@ -101,16 +149,13 @@ public void testKeyedValueStateMigration() throws Exception { backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot); - // the new serializer is V2, and has a completely new serialization schema. - kvId = new ValueStateDescriptor<>( - stateName, - new TestType.V2TestTypeSerializer()); - valueState = backend - .getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); + valueState = backend.getPartitionedState( + VoidNamespace.INSTANCE, + CustomVoidNamespaceSerializer.INSTANCE, + newAccessDescriptorAfterRestore); snapshot.discardState(); - // the state backend should have decided whether or not it needs to perform state migration; // make sure that reading and writing each key state works with the new serializer backend.setCurrentKey(1); Assert.assertEquals(new TestType("foo", 1456), valueState.value()); @@ -128,20 +173,72 @@ public void testKeyedValueStateMigration() throws Exception { } } + // ------------------------------------------------------------------------------- + // Tests for keyed ListState + // ------------------------------------------------------------------------------- + @Test public void testKeyedListStateMigration() throws Exception { + final String stateName = "test-name"; + + testKeyedListStateUpgrade( + new ListStateDescriptor<>( + stateName, + new TestType.V1TestTypeSerializer()), + new ListStateDescriptor<>( + stateName, + // restore with a V2 serializer that has a different schema + new TestType.V2TestTypeSerializer())); + } + + @Test + @Ignore("This currently doesn't pass because the ListSerializer doesn't respect the reconfigured case, yet.") + public void testKeyedListStateSerializerReconfiguration() throws Exception { + final String stateName = "test-name"; + + testKeyedListStateUpgrade( + new ListStateDescriptor<>( + stateName, + new TestType.V1TestTypeSerializer()), + new ListStateDescriptor<>( + stateName, + // the test fails if this serializer is used instead of a reconfigured new serializer + new TestType.ReconfigurationRequiringTestTypeSerializer())); + } + + @Test + public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() throws Exception { + final String stateName = "test-name"; + + try { + testKeyedListStateUpgrade( + new ListStateDescriptor<>( + stateName, + new TestType.V1TestTypeSerializer()), + new ListStateDescriptor<>( + stateName, + new TestType.IncompatibleTestTypeSerializer())); + + Assert.fail("should have failed"); + } catch (Exception expected) { + Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent()); + } + } + + private void testKeyedListStateUpgrade( + ListStateDescriptor initialAccessDescriptor, + ListStateDescriptor newAccessDescriptorAfterRestore) throws Exception { + CheckpointStreamFactory streamFactory = createStreamFactory(); SharedStateRegistry sharedStateRegistry = new SharedStateRegistry(); - final String stateName = "test-name"; AbstractKeyedStateBackend backend = createKeyedBackend(IntSerializer.INSTANCE); try { - ListStateDescriptor kvId = new ListStateDescriptor<>( - stateName, - new TestType.V1TestTypeSerializer()); - ListState listState = backend - .getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); + ListState listState = backend.getPartitionedState( + VoidNamespace.INSTANCE, + CustomVoidNamespaceSerializer.INSTANCE, + initialAccessDescriptor); backend.setCurrentKey(1); listState.add(new TestType("key-1", 1)); @@ -162,16 +259,13 @@ public void testKeyedListStateMigration() throws Exception { backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot); - // the new serializer is V2, and has a completely new serialization schema. - kvId = new ListStateDescriptor<>( - stateName, - new TestType.V2TestTypeSerializer()); - listState = backend - .getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); + listState = backend.getPartitionedState( + VoidNamespace.INSTANCE, + CustomVoidNamespaceSerializer.INSTANCE, + newAccessDescriptorAfterRestore); snapshot.discardState(); - // the state backend should have decided whether or not it needs to perform state migration; // make sure that reading and writing each key state works with the new serializer backend.setCurrentKey(1); Iterator iterable1 = listState.get().iterator(); @@ -198,27 +292,24 @@ public void testKeyedListStateMigration() throws Exception { } } + // ------------------------------------------------------------------------------- + // Tests for keyed priority queue state + // ------------------------------------------------------------------------------- + @Test - public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() throws Exception { + public void testPriorityQueueStateCreationFailsIfNewSerializerIsNotCompatible() throws Exception { CheckpointStreamFactory streamFactory = createStreamFactory(); SharedStateRegistry sharedStateRegistry = new SharedStateRegistry(); - final String stateName = "test-name"; AbstractKeyedStateBackend backend = createKeyedBackend(IntSerializer.INSTANCE); try { - ValueStateDescriptor kvId = new ValueStateDescriptor<>( - stateName, - new TestType.V1TestTypeSerializer()); - ValueState valueState = backend - .getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); + InternalPriorityQueue internalPriorityQueue = backend.create( + "testPriorityQueue", new TestType.V1TestTypeSerializer()); - backend.setCurrentKey(1); - valueState.update(new TestType("foo", 1456)); - backend.setCurrentKey(2); - valueState.update(new TestType("bar", 478)); - backend.setCurrentKey(3); - valueState.update(new TestType("hello", 189)); + internalPriorityQueue.add(new TestType("key-1", 123)); + internalPriorityQueue.add(new TestType("key-2", 346)); + internalPriorityQueue.add(new TestType("key-1", 777)); KeyedStateHandle snapshot = runSnapshot( backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), @@ -226,110 +317,64 @@ public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatib backend.dispose(); backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot); - - kvId = new ValueStateDescriptor<>( - stateName, - new TestType.IncompatibleTestTypeSerializer()); - - // the new serializer is INCOMPATIBLE, so registering the state should fail - backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); + backend.create( + "testPriorityQueue", new TestType.IncompatibleTestTypeSerializer()); Assert.fail("should have failed"); } catch (Exception e) { Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); - }finally { + } finally { backend.dispose(); } } - @Test - public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() throws Exception { - CheckpointStreamFactory streamFactory = createStreamFactory(); - SharedStateRegistry sharedStateRegistry = new SharedStateRegistry(); - - final String stateName = "test-name"; - AbstractKeyedStateBackend backend = createKeyedBackend(IntSerializer.INSTANCE); + // ------------------------------------------------------------------------------- + // Tests for key serializer in keyed state backends + // ------------------------------------------------------------------------------- + @Test + public void testStateBackendRestoreFailsIfNewKeySerializerRequiresMigration() throws Exception { try { - ListStateDescriptor kvId = new ListStateDescriptor<>( - stateName, - new TestType.V1TestTypeSerializer()); - ListState listState = backend - .getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); - - backend.setCurrentKey(1); - listState.add(new TestType("key-1", 1)); - listState.add(new TestType("key-1", 2)); - listState.add(new TestType("key-1", 3)); - - backend.setCurrentKey(2); - listState.add(new TestType("key-2", 1)); - - backend.setCurrentKey(3); - listState.add(new TestType("key-3", 1)); - listState.add(new TestType("key-3", 2)); - - KeyedStateHandle snapshot = runSnapshot( - backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), - sharedStateRegistry); - backend.dispose(); - - backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot); - - kvId = new ListStateDescriptor<>( - stateName, - new TestType.IncompatibleTestTypeSerializer()); - - // the new serializer is INCOMPATIBLE, so registering the state should fail - backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); + testKeySerializerUpgrade( + new TestType.V1TestTypeSerializer(), + new TestType.V2TestTypeSerializer()); Assert.fail("should have failed"); - } catch (Exception e) { - Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); - } finally { - backend.dispose(); + } catch (Exception expected) { + // the new key serializer requires migration; this should fail the restore + Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent()); } } @Test - public void testPriorityQueueStateCreationFailsIfNewSerializerIsNotCompatible() throws Exception { - CheckpointStreamFactory streamFactory = createStreamFactory(); - SharedStateRegistry sharedStateRegistry = new SharedStateRegistry(); - - AbstractKeyedStateBackend backend = createKeyedBackend(IntSerializer.INSTANCE); + public void testStateBackendRestoreSucceedsIfNewKeySerializerRequiresReconfiguration() throws Exception { + testKeySerializerUpgrade( + new TestType.V1TestTypeSerializer(), + new TestType.ReconfigurationRequiringTestTypeSerializer()); + } + @Test + public void testStateBackendRestoreFailsIfNewKeySerializerIsIncompatible() throws Exception { try { - InternalPriorityQueue internalPriorityQueue = backend.create( - "testPriorityQueue", new TestType.V1TestTypeSerializer()); - - internalPriorityQueue.add(new TestType("key-1", 123)); - internalPriorityQueue.add(new TestType("key-2", 346)); - internalPriorityQueue.add(new TestType("key-1", 777)); - - KeyedStateHandle snapshot = runSnapshot( - backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), - sharedStateRegistry); - backend.dispose(); - - backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot); - backend.create( - "testPriorityQueue", new TestType.IncompatibleTestTypeSerializer()); + testKeySerializerUpgrade( + new TestType.V1TestTypeSerializer(), + new TestType.IncompatibleTestTypeSerializer()); Assert.fail("should have failed"); - } catch (Exception e) { - Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); - } finally { - backend.dispose(); + } catch (Exception expected) { + // the new key serializer is incompatible; this should fail the restore + Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent()); } } - @Test - public void testStateBackendCreationFailsIfNewKeySerializerIsNotCompatible() throws Exception { + private void testKeySerializerUpgrade( + TypeSerializer initialKeySerializer, + TypeSerializer newKeySerializer) throws Exception { + CheckpointStreamFactory streamFactory = createStreamFactory(); SharedStateRegistry sharedStateRegistry = new SharedStateRegistry(); - AbstractKeyedStateBackend backend = createKeyedBackend( - new TestType.V1TestTypeSerializer()); + AbstractKeyedStateBackend backend = createKeyedBackend(initialKeySerializer); final String stateName = "test-name"; try { @@ -347,30 +392,66 @@ public void testStateBackendCreationFailsIfNewKeySerializerIsNotCompatible() thr sharedStateRegistry); backend.dispose(); - try { - // the new key serializer is incompatible; this should fail the restore - restoreKeyedBackend(new TestType.IncompatibleTestTypeSerializer(), snapshot); + backend = restoreKeyedBackend(newKeySerializer, snapshot); - Assert.fail("should have failed"); - } catch (Exception e) { - Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); - } + valueState = backend + .getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId); - try { - // the new key serializer requires migration; this should fail the restore - restoreKeyedBackend(new TestType.V2TestTypeSerializer(), snapshot); + // access and check previous state + backend.setCurrentKey(new TestType("foo", 123)); + Assert.assertEquals(1, valueState.value().intValue()); + backend.setCurrentKey(new TestType("bar", 456)); + Assert.assertEquals(5, valueState.value().intValue()); - Assert.fail("should have failed"); - } catch (Exception e) { - Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); - } + snapshot.discardState(); } finally { backend.dispose(); } } + // ------------------------------------------------------------------------------- + // Tests for namespace serializer in keyed state backends + // ------------------------------------------------------------------------------- + @Test - public void testKeyedStateRegistrationFailsIfNewNamespaceSerializerIsNotCompatible() throws Exception { + public void testKeyedStateRegistrationFailsIfNewNamespaceSerializerRequiresMigration() throws Exception { + try { + testNamespaceSerializerUpgrade( + new TestType.V1TestTypeSerializer(), + new TestType.V2TestTypeSerializer()); + + Assert.fail("should have failed"); + } catch (Exception expected) { + // the new namespace serializer requires migration; this should fail the restore + Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent()); + } + } + + @Test + public void testKeyedStateRegistrationSucceedsIfNewNamespaceSerializerRequiresReconfiguration() throws Exception { + testNamespaceSerializerUpgrade( + new TestType.V1TestTypeSerializer(), + new TestType.ReconfigurationRequiringTestTypeSerializer()); + } + + @Test + public void testKeyedStateRegistrationFailsIfNewNamespaceSerializerIsIncompatible() throws Exception { + try { + testNamespaceSerializerUpgrade( + new TestType.V1TestTypeSerializer(), + new TestType.IncompatibleTestTypeSerializer()); + + Assert.fail("should have failed"); + } catch (Exception expected) { + // the new namespace serializer is incompatible; this should fail the restore + Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent()); + } + } + + private void testNamespaceSerializerUpgrade( + TypeSerializer initialNamespaceSerializer, + TypeSerializer newNamespaceSerializerAfterRestore) throws Exception { + CheckpointStreamFactory streamFactory = createStreamFactory(); SharedStateRegistry sharedStateRegistry = new SharedStateRegistry(); @@ -382,7 +463,7 @@ public void testKeyedStateRegistrationFailsIfNewNamespaceSerializerIsNotCompatib ValueState valueState = backend .getPartitionedState( new TestType("namespace", 123), - new TestType.V1TestTypeSerializer(), + initialNamespaceSerializer, kvId); backend.setCurrentKey(1); @@ -397,53 +478,87 @@ public void testKeyedStateRegistrationFailsIfNewNamespaceSerializerIsNotCompatib // test incompatible namespace serializer; start with a freshly restored backend backend.dispose(); backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot); - try { - // the new namespace serializer is incompatible; this should fail the restore - backend.getPartitionedState( - new TestType("namespace", 123), - new TestType.IncompatibleTestTypeSerializer(), - kvId); - Assert.fail("should have failed"); - } catch (Exception e) { - Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); - } + valueState = backend.getPartitionedState( + new TestType("namespace", 123), + newNamespaceSerializerAfterRestore, + kvId); - // test namespace serializer that requires migration; start with a freshly restored backend - backend.dispose(); - backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot); - try { - // the new namespace serializer requires migration; this should fail the restore - backend.getPartitionedState( - new TestType("namespace", 123), - new TestType.V2TestTypeSerializer(), - kvId); + // access and check previous state + backend.setCurrentKey(1); + Assert.assertEquals(10, valueState.value().intValue()); + valueState.update(10); + backend.setCurrentKey(5); + Assert.assertEquals(50, valueState.value().intValue()); - Assert.fail("should have failed"); - } catch (Exception e) { - Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); - } + snapshot.discardState(); } finally { backend.dispose(); } } // ------------------------------------------------------------------------------- - // Operator state backend migration tests + // Operator state backend partitionable list state tests // ------------------------------------------------------------------------------- @Test public void testOperatorParitionableListStateMigration() throws Exception { + final String stateName = "partitionable-list-state"; + + testOperatorPartitionableListStateUpgrade( + new ListStateDescriptor<>( + stateName, + new TestType.V1TestTypeSerializer()), + new ListStateDescriptor<>( + stateName, + // restore with a V2 serializer that has a different schema + new TestType.V2TestTypeSerializer())); + } + + @Test + public void testOperatorParitionableListStateSerializerReconfiguration() throws Exception { + final String stateName = "partitionable-list-state"; + + testOperatorPartitionableListStateUpgrade( + new ListStateDescriptor<>( + stateName, + new TestType.V1TestTypeSerializer()), + new ListStateDescriptor<>( + stateName, + // restore with a new serializer that requires reconfiguration + new TestType.ReconfigurationRequiringTestTypeSerializer())); + } + + @Test + public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception { + final String stateName = "partitionable-list-state"; + + try { + testOperatorPartitionableListStateUpgrade( + new ListStateDescriptor<>( + stateName, + new TestType.V1TestTypeSerializer()), + new ListStateDescriptor<>( + stateName, + // restore with a new incompatible serializer + new TestType.IncompatibleTestTypeSerializer())); + + Assert.fail("should have failed."); + } catch (Exception e) { + Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); + } + } + + private void testOperatorPartitionableListStateUpgrade( + ListStateDescriptor initialAccessDescriptor, + ListStateDescriptor newAccessDescriptorAfterRestore) throws Exception { + CheckpointStreamFactory streamFactory = createStreamFactory(); OperatorStateBackend backend = createOperatorStateBackend(); - final String stateName = "partitionable-list-state"; try { - ListStateDescriptor descriptor = new ListStateDescriptor<>( - stateName, - new TestType.V1TestTypeSerializer()); - ListState state = backend.getListState(descriptor); + ListState state = backend.getListState(initialAccessDescriptor); state.add(new TestType("foo", 13)); state.add(new TestType("bar", 278)); @@ -454,12 +569,8 @@ public void testOperatorParitionableListStateMigration() throws Exception { backend = restoreOperatorStateBackend(snapshot); - descriptor = new ListStateDescriptor<>( - stateName, - new TestType.V2TestTypeSerializer()); - state = backend.getListState(descriptor); + state = backend.getListState(newAccessDescriptorAfterRestore); - // the state backend should have decided whether or not it needs to perform state migration; // make sure that reading and writing each state partition works with the new serializer Iterator iterator = state.get().iterator(); Assert.assertEquals(new TestType("foo", 13), iterator.next()); @@ -471,18 +582,69 @@ public void testOperatorParitionableListStateMigration() throws Exception { } } + // ------------------------------------------------------------------------------- + // Operator state backend union list state tests + // ------------------------------------------------------------------------------- + + @Test + public void testOperatorUnionListStateMigration() throws Exception { + final String stateName = "union-list-state"; + + testOperatorUnionListStateUpgrade( + new ListStateDescriptor<>( + stateName, + new TestType.V1TestTypeSerializer()), + new ListStateDescriptor<>( + stateName, + // restore with a V2 serializer that has a different schema + new TestType.V2TestTypeSerializer())); + } + @Test - public void testUnionListStateMigration() throws Exception { + public void testOperatorUnionListStateSerializerReconfiguration() throws Exception { + final String stateName = "union-list-state"; + + testOperatorUnionListStateUpgrade( + new ListStateDescriptor<>( + stateName, + new TestType.V1TestTypeSerializer()), + new ListStateDescriptor<>( + stateName, + // restore with a new serializer that requires reconfiguration + new TestType.ReconfigurationRequiringTestTypeSerializer())); + } + + + @Test + public void testOperatorUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception { + final String stateName = "union-list-state"; + + try { + testOperatorUnionListStateUpgrade( + new ListStateDescriptor<>( + stateName, + new TestType.V1TestTypeSerializer()), + new ListStateDescriptor<>( + stateName, + // restore with a new incompatible serializer + new TestType.IncompatibleTestTypeSerializer())); + + Assert.fail("should have failed."); + } catch (Exception e) { + Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); + } + } + + private void testOperatorUnionListStateUpgrade( + ListStateDescriptor initialAccessDescriptor, + ListStateDescriptor newAccessDescriptorAfterRestore) throws Exception { + CheckpointStreamFactory streamFactory = createStreamFactory(); OperatorStateBackend backend = createOperatorStateBackend(); - final String stateName = "union-list-state"; try { - ListStateDescriptor descriptor = new ListStateDescriptor<>( - stateName, - new TestType.V1TestTypeSerializer()); - ListState state = backend.getUnionListState(descriptor); + ListState state = backend.getUnionListState(initialAccessDescriptor); state.add(new TestType("foo", 13)); state.add(new TestType("bar", 278)); @@ -493,10 +655,7 @@ public void testUnionListStateMigration() throws Exception { backend = restoreOperatorStateBackend(snapshot); - descriptor = new ListStateDescriptor<>( - stateName, - new TestType.V2TestTypeSerializer()); - state = backend.getUnionListState(descriptor); + state = backend.getUnionListState(newAccessDescriptorAfterRestore); // the state backend should have decided whether or not it needs to perform state migration; // make sure that reading and writing each state partition works with the new serializer @@ -510,171 +669,128 @@ public void testUnionListStateMigration() throws Exception { } } + // ------------------------------------------------------------------------------- + // Operator state backend broadcast state tests + // ------------------------------------------------------------------------------- + @Test public void testBroadcastStateValueMigration() throws Exception { - CheckpointStreamFactory streamFactory = createStreamFactory(); - - OperatorStateBackend backend = createOperatorStateBackend(); - final String stateName = "broadcast-state"; - try { - MapStateDescriptor descriptor = new MapStateDescriptor<>( + + testBroadcastStateValueUpgrade( + new MapStateDescriptor<>( stateName, IntSerializer.INSTANCE, - new TestType.V1TestTypeSerializer()); - BroadcastState state = backend.getBroadcastState(descriptor); - - state.put(3, new TestType("foo", 13)); - state.put(5, new TestType("bar", 278)); - - OperatorStateHandle snapshot = runSnapshot( - backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation())); - backend.dispose(); - - backend = restoreOperatorStateBackend(snapshot); - - descriptor = new MapStateDescriptor<>( + new TestType.V1TestTypeSerializer()), + new MapStateDescriptor<>( stateName, IntSerializer.INSTANCE, - new TestType.V2TestTypeSerializer()); - state = backend.getBroadcastState(descriptor); - - // the state backend should have decided whether or not it needs to perform state migration; - // make sure that reading and writing each broadcast entry works with the new serializer - Assert.assertEquals(new TestType("foo", 13), state.get(3)); - Assert.assertEquals(new TestType("bar", 278), state.get(5)); - state.put(17, new TestType("new-entry", 777)); - } finally { - backend.dispose(); - } + // new value serializer is a V2 serializer with a different schema + new TestType.V2TestTypeSerializer())); } + @Test public void testBroadcastStateKeyMigration() throws Exception { - CheckpointStreamFactory streamFactory = createStreamFactory(); - - OperatorStateBackend backend = createOperatorStateBackend(); - final String stateName = "broadcast-state"; - try { - MapStateDescriptor descriptor = new MapStateDescriptor<>( + + testBroadcastStateKeyUpgrade( + new MapStateDescriptor<>( stateName, new TestType.V1TestTypeSerializer(), - IntSerializer.INSTANCE); - BroadcastState state = backend.getBroadcastState(descriptor); - - state.put(new TestType("foo", 13), 3); - state.put(new TestType("bar", 278), 5); - - OperatorStateHandle snapshot = runSnapshot( - backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation())); - backend.dispose(); - - backend = restoreOperatorStateBackend(snapshot); - - descriptor = new MapStateDescriptor<>( + IntSerializer.INSTANCE), + new MapStateDescriptor<>( stateName, + // new key serializer is a V2 serializer with a different schema new TestType.V2TestTypeSerializer(), - IntSerializer.INSTANCE); - state = backend.getBroadcastState(descriptor); - - // the state backend should have decided whether or not it needs to perform state migration; - // make sure that reading and writing each broadcast entry works with the new serializer - Assert.assertEquals((Integer) 3, state.get(new TestType("foo", 13))); - Assert.assertEquals((Integer) 5, state.get(new TestType("bar", 278))); - state.put(new TestType("new-entry", 777), 17); - } finally { - backend.dispose(); - } + IntSerializer.INSTANCE)); } @Test - public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception { - CheckpointStreamFactory streamFactory = createStreamFactory(); - - OperatorStateBackend backend = createOperatorStateBackend(); + public void testBroadcastStateValueSerializerReconfiguration() throws Exception { + final String stateName = "broadcast-state"; - final String stateName = "partitionable-list-state"; - try { - ListStateDescriptor descriptor = new ListStateDescriptor<>( + testBroadcastStateValueUpgrade( + new MapStateDescriptor<>( stateName, - new TestType.V1TestTypeSerializer()); - ListState state = backend.getListState(descriptor); - - state.add(new TestType("foo", 13)); - state.add(new TestType("bar", 278)); - - OperatorStateHandle snapshot = runSnapshot( - backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation())); - backend.dispose(); + IntSerializer.INSTANCE, + new TestType.V1TestTypeSerializer()), + new MapStateDescriptor<>( + stateName, + IntSerializer.INSTANCE, + // new value serializer is a new serializer that requires reconfiguration + new TestType.ReconfigurationRequiringTestTypeSerializer())); + } - backend = restoreOperatorStateBackend(snapshot); + @Test + public void testBroadcastStateKeySerializerReconfiguration() throws Exception { + final String stateName = "broadcast-state"; - descriptor = new ListStateDescriptor<>( + testBroadcastStateKeyUpgrade( + new MapStateDescriptor<>( stateName, - new TestType.IncompatibleTestTypeSerializer()); + new TestType.V1TestTypeSerializer(), + IntSerializer.INSTANCE), + new MapStateDescriptor<>( + stateName, + // new key serializer is a new serializer that requires reconfiguration + new TestType.ReconfigurationRequiringTestTypeSerializer(), + IntSerializer.INSTANCE)); + } - // the new serializer is INCOMPATIBLE, so registering the state should fail - backend.getListState(descriptor); + @Test + public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatible() throws Exception { + final String stateName = "broadcast-state"; + + try { + testBroadcastStateValueUpgrade( + new MapStateDescriptor<>( + stateName, + IntSerializer.INSTANCE, + new TestType.V1TestTypeSerializer()), + new MapStateDescriptor<>( + stateName, + IntSerializer.INSTANCE, + // new value serializer is incompatible + new TestType.IncompatibleTestTypeSerializer())); Assert.fail("should have failed."); } catch (Exception e) { Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); - } finally { - backend.dispose(); } } @Test - public void testUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception { - CheckpointStreamFactory streamFactory = createStreamFactory(); - - OperatorStateBackend backend = createOperatorStateBackend(); + public void testBroadcastStateRegistrationFailsIfNewKeySerializerIsIncompatible() throws Exception { + final String stateName = "broadcast-state"; - final String stateName = "union-list-state"; try { - ListStateDescriptor descriptor = new ListStateDescriptor<>( - stateName, - new TestType.V1TestTypeSerializer()); - ListState state = backend.getUnionListState(descriptor); - - state.add(new TestType("foo", 13)); - state.add(new TestType("bar", 278)); - - OperatorStateHandle snapshot = runSnapshot( - backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation())); - backend.dispose(); - - backend = restoreOperatorStateBackend(snapshot); - - descriptor = new ListStateDescriptor<>( - stateName, - new TestType.IncompatibleTestTypeSerializer()); - - // the new serializer is INCOMPATIBLE, so registering the state should fail - backend.getUnionListState(descriptor); + testBroadcastStateKeyUpgrade( + new MapStateDescriptor<>( + stateName, + new TestType.V1TestTypeSerializer(), + IntSerializer.INSTANCE), + new MapStateDescriptor<>( + stateName, + // new key serializer is incompatible + new TestType.IncompatibleTestTypeSerializer(), + IntSerializer.INSTANCE)); Assert.fail("should have failed."); } catch (Exception e) { Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); - } finally { - backend.dispose(); } } - @Test - public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatible() throws Exception { + private void testBroadcastStateValueUpgrade( + MapStateDescriptor initialAccessDescriptor, + MapStateDescriptor newAccessDescriptorAfterRestore) throws Exception { CheckpointStreamFactory streamFactory = createStreamFactory(); OperatorStateBackend backend = createOperatorStateBackend(); - final String stateName = "broadcast-state"; try { - MapStateDescriptor descriptor = new MapStateDescriptor<>( - stateName, - IntSerializer.INSTANCE, - new TestType.V1TestTypeSerializer()); - BroadcastState state = backend.getBroadcastState(descriptor); + BroadcastState state = backend.getBroadcastState(initialAccessDescriptor); state.put(3, new TestType("foo", 13)); state.put(5, new TestType("bar", 278)); @@ -685,35 +801,28 @@ public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatibl backend = restoreOperatorStateBackend(snapshot); - descriptor = new MapStateDescriptor<>( - stateName, - IntSerializer.INSTANCE, - new TestType.IncompatibleTestTypeSerializer()); + state = backend.getBroadcastState(newAccessDescriptorAfterRestore); - // the new value serializer is INCOMPATIBLE, so registering the state should fail - backend.getBroadcastState(descriptor); - - Assert.fail("should have failed."); - } catch (Exception e) { - Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); + // the state backend should have decided whether or not it needs to perform state migration; + // make sure that reading and writing each broadcast entry works with the new serializer + Assert.assertEquals(new TestType("foo", 13), state.get(3)); + Assert.assertEquals(new TestType("bar", 278), state.get(5)); + state.put(17, new TestType("new-entry", 777)); } finally { backend.dispose(); } } - @Test - public void testBroadcastStateRegistrationFailsIfNewKeySerializerIsIncompatible() throws Exception { + private void testBroadcastStateKeyUpgrade( + MapStateDescriptor initialAccessDescriptor, + MapStateDescriptor newAccessDescriptorAfterRestore) throws Exception { + CheckpointStreamFactory streamFactory = createStreamFactory(); OperatorStateBackend backend = createOperatorStateBackend(); - final String stateName = "broadcast-state"; try { - MapStateDescriptor descriptor = new MapStateDescriptor<>( - stateName, - new TestType.V1TestTypeSerializer(), - IntSerializer.INSTANCE); - BroadcastState state = backend.getBroadcastState(descriptor); + BroadcastState state = backend.getBroadcastState(initialAccessDescriptor); state.put(new TestType("foo", 13), 3); state.put(new TestType("bar", 278), 5); @@ -724,17 +833,13 @@ public void testBroadcastStateRegistrationFailsIfNewKeySerializerIsIncompatible( backend = restoreOperatorStateBackend(snapshot); - descriptor = new MapStateDescriptor<>( - stateName, - new TestType.IncompatibleTestTypeSerializer(), - IntSerializer.INSTANCE); - - // the new key serializer is INCOMPATIBLE, so registering the state should fail - backend.getBroadcastState(descriptor); + state = backend.getBroadcastState(newAccessDescriptorAfterRestore); - Assert.fail("should have failed."); - } catch (Exception e) { - Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent()); + // the state backend should have decided whether or not it needs to perform state migration; + // make sure that reading and writing each broadcast entry works with the new serializer + Assert.assertEquals((Integer) 3, state.get(new TestType("foo", 13))); + Assert.assertEquals((Integer) 5, state.get(new TestType("bar", 278))); + state.put(new TestType("new-entry", 777), 17); } finally { backend.dispose(); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateSerializerProviderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateSerializerProviderTest.java index de1f2bd962cd21..91ebc958ae60a4 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateSerializerProviderTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateSerializerProviderTest.java @@ -42,15 +42,15 @@ public class StateSerializerProviderTest { // -------------------------------------------------------------------------------- @Test - public void testCurrentSchemaSerializerForNewStateSerializerProvider() { - StateSerializerProvider testProvider = StateSerializerProvider.fromNewState(new TestType.V1TestTypeSerializer()); + public void testCurrentSchemaSerializerForEagerlyRegisteredStateSerializerProvider() { + StateSerializerProvider testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.V1TestTypeSerializer()); assertTrue(testProvider.currentSchemaSerializer() instanceof TestType.V1TestTypeSerializer); } @Test - public void testCurrentSchemaSerializerForRestoredStateSerializerProvider() { + public void testCurrentSchemaSerializerForLazilyRegisteredStateSerializerProvider() { TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); - StateSerializerProvider testProvider = StateSerializerProvider.fromRestoredState(serializer.snapshotConfiguration()); + StateSerializerProvider testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration()); assertTrue(testProvider.currentSchemaSerializer() instanceof TestType.V1TestTypeSerializer); } @@ -59,17 +59,17 @@ public void testCurrentSchemaSerializerForRestoredStateSerializerProvider() { // -------------------------------------------------------------------------------- @Test(expected = UnsupportedOperationException.class) - public void testPreviousSchemaSerializerForNewStateSerializerProvider() { - StateSerializerProvider testProvider = StateSerializerProvider.fromNewState(new TestType.V1TestTypeSerializer()); + public void testPreviousSchemaSerializerForEagerlyRegisteredStateSerializerProvider() { + StateSerializerProvider testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.V1TestTypeSerializer()); // this should fail with an exception testProvider.previousSchemaSerializer(); } @Test - public void testPreviousSchemaSerializerForRestoredStateSerializerProvider() { + public void testPreviousSchemaSerializerForLazilyRegisteredStateSerializerProvider() { TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); - StateSerializerProvider testProvider = StateSerializerProvider.fromRestoredState(serializer.snapshotConfiguration()); + StateSerializerProvider testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration()); assertTrue(testProvider.previousSchemaSerializer() instanceof TestType.V1TestTypeSerializer); } @@ -78,7 +78,7 @@ public void testLazyInstantiationOfPreviousSchemaSerializer() { // create the provider with an exception throwing snapshot; // this would throw an exception if the restore serializer was eagerly accessed StateSerializerProvider testProvider = - StateSerializerProvider.fromRestoredState(new ExceptionThrowingSerializerSnapshot()); + StateSerializerProvider.fromPreviousSerializerSnapshot(new ExceptionThrowingSerializerSnapshot()); try { // if we fail here, that means the restore serializer was indeed lazily accessed @@ -94,15 +94,15 @@ public void testLazyInstantiationOfPreviousSchemaSerializer() { // -------------------------------------------------------------------------------- @Test(expected = UnsupportedOperationException.class) - public void testRegisterNewSerializerWithNewStateSerializerProviderShouldFail() { - StateSerializerProvider testProvider = StateSerializerProvider.fromNewState(new TestType.V1TestTypeSerializer()); + public void testRegisterNewSerializerWithEagerlyRegisteredStateSerializerProviderShouldFail() { + StateSerializerProvider testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.V1TestTypeSerializer()); testProvider.registerNewSerializerForRestoredState(new TestType.V2TestTypeSerializer()); } @Test(expected = UnsupportedOperationException.class) - public void testRegisterNewSerializerTwiceWithNewStateSerializerProviderShouldFail() { + public void testRegisterNewSerializerTwiceWithLazilyRegisteredStateSerializerProviderShouldFail() { TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); - StateSerializerProvider testProvider = StateSerializerProvider.fromRestoredState(serializer.snapshotConfiguration()); + StateSerializerProvider testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration()); testProvider.registerNewSerializerForRestoredState(new TestType.V2TestTypeSerializer()); @@ -111,9 +111,9 @@ public void testRegisterNewSerializerTwiceWithNewStateSerializerProviderShouldFa } @Test - public void testRegisterNewCompatibleAsIsSerializer() { + public void testLazilyRegisterNewCompatibleAsIsSerializer() { TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); - StateSerializerProvider testProvider = StateSerializerProvider.fromRestoredState(serializer.snapshotConfiguration()); + StateSerializerProvider testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration()); // register compatible serializer for state TypeSerializerSchemaCompatibility schemaCompatibility = @@ -125,20 +125,36 @@ public void testRegisterNewCompatibleAsIsSerializer() { } @Test - public void testRegisterNewCompatibleAfterMigrationSerializer() { + public void testLazilyRegisterNewCompatibleAfterMigrationSerializer() { TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); - StateSerializerProvider testProvider = StateSerializerProvider.fromRestoredState(serializer.snapshotConfiguration()); + StateSerializerProvider testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration()); // register serializer that requires migration for state TypeSerializerSchemaCompatibility schemaCompatibility = testProvider.registerNewSerializerForRestoredState(new TestType.V2TestTypeSerializer()); assertTrue(schemaCompatibility.isCompatibleAfterMigration()); + + assertTrue(testProvider.currentSchemaSerializer() instanceof TestType.V2TestTypeSerializer); + assertTrue(testProvider.previousSchemaSerializer() instanceof TestType.V1TestTypeSerializer); + } + + @Test + public void testLazilyRegisterNewSerializerRequiringReconfiguration() { + TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); + StateSerializerProvider testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration()); + + // register serializer that requires reconfiguration, and verify that + // the resulting current schema serializer is the reconfigured one + TypeSerializerSchemaCompatibility schemaCompatibility = + testProvider.registerNewSerializerForRestoredState(new TestType.ReconfigurationRequiringTestTypeSerializer()); + assertTrue(schemaCompatibility.isCompatibleWithReconfiguredSerializer()); + assertTrue(testProvider.currentSchemaSerializer().getClass() == TestType.V1TestTypeSerializer.class); } @Test - public void testRegisterIncompatibleSerializer() { + public void testLazilyRegisterIncompatibleSerializer() { TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); - StateSerializerProvider testProvider = StateSerializerProvider.fromRestoredState(serializer.snapshotConfiguration()); + StateSerializerProvider testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration()); // register serializer that requires migration for state TypeSerializerSchemaCompatibility schemaCompatibility = @@ -148,6 +164,88 @@ public void testRegisterIncompatibleSerializer() { try { // a serializer for the current schema will no longer be accessible testProvider.currentSchemaSerializer(); + + fail(); + } catch (Exception excepted) { + // success + } + } + + // -------------------------------------------------------------------------------- + // Tests for #setPreviousSerializerSnapshotForRestoredState(TypeSerializerSnapshot) + // -------------------------------------------------------------------------------- + + @Test(expected = UnsupportedOperationException.class) + public void testSetSerializerSnapshotWithLazilyRegisteredSerializerProviderShouldFail() { + TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); + StateSerializerProvider testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration()); + + testProvider.setPreviousSerializerSnapshotForRestoredState(serializer.snapshotConfiguration()); + } + + @Test(expected = UnsupportedOperationException.class) + public void testSetSerializerSnapshotTwiceWithEagerlyRegisteredSerializerProviderShouldFail() { + TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); + StateSerializerProvider testProvider = StateSerializerProvider.fromNewRegisteredSerializer(serializer); + + testProvider.setPreviousSerializerSnapshotForRestoredState(serializer.snapshotConfiguration()); + + // second registration should fail + testProvider.setPreviousSerializerSnapshotForRestoredState(serializer.snapshotConfiguration()); + } + + @Test + public void testEagerlyRegisterNewCompatibleAsIsSerializer() { + StateSerializerProvider testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.V1TestTypeSerializer()); + + // set previous serializer snapshot for state, which should let the new serializer be considered compatible as is + TypeSerializerSchemaCompatibility schemaCompatibility = + testProvider.setPreviousSerializerSnapshotForRestoredState(new TestType.V1TestTypeSerializer().snapshotConfiguration()); + assertTrue(schemaCompatibility.isCompatibleAsIs()); + + assertTrue(testProvider.currentSchemaSerializer() instanceof TestType.V1TestTypeSerializer); + assertTrue(testProvider.previousSchemaSerializer() instanceof TestType.V1TestTypeSerializer); + } + + @Test + public void testEagerlyRegisterCompatibleAfterMigrationSerializer() { + StateSerializerProvider testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.V2TestTypeSerializer()); + + // set previous serializer snapshot for state, which should let the new serializer be considered compatible after migration + TypeSerializerSchemaCompatibility schemaCompatibility = + testProvider.setPreviousSerializerSnapshotForRestoredState(new TestType.V1TestTypeSerializer().snapshotConfiguration()); + assertTrue(schemaCompatibility.isCompatibleAfterMigration()); + + assertTrue(testProvider.currentSchemaSerializer() instanceof TestType.V2TestTypeSerializer); + assertTrue(testProvider.previousSchemaSerializer() instanceof TestType.V1TestTypeSerializer); + } + + @Test + public void testEagerlyRegisterNewSerializerRequiringReconfiguration() { + StateSerializerProvider testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.ReconfigurationRequiringTestTypeSerializer()); + + // set previous serializer snapshot, which should let the new serializer be considered to require reconfiguration, + // and verify that the resulting current schema serializer is the reconfigured one + TypeSerializerSchemaCompatibility schemaCompatibility = + testProvider.setPreviousSerializerSnapshotForRestoredState(new TestType.V1TestTypeSerializer().snapshotConfiguration()); + assertTrue(schemaCompatibility.isCompatibleWithReconfiguredSerializer()); + assertTrue(testProvider.currentSchemaSerializer().getClass() == TestType.V1TestTypeSerializer.class); + } + + @Test + public void testEagerlyRegisterIncompatibleSerializer() { + StateSerializerProvider testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.IncompatibleTestTypeSerializer()); + + // set previous serializer snapshot for state, which should let the new serializer be considered incompatible + TypeSerializerSchemaCompatibility schemaCompatibility = + testProvider.setPreviousSerializerSnapshotForRestoredState(new TestType.V1TestTypeSerializer().snapshotConfiguration()); + assertTrue(schemaCompatibility.isIncompatible()); + + try { + // a serializer for the current schema will no longer be accessible + testProvider.currentSchemaSerializer(); + + fail(); } catch (Exception excepted) { // success } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/TestType.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/TestType.java index e3b0a066e5158a..52d5521aae8376 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/TestType.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/TestType.java @@ -148,6 +148,29 @@ public TypeSerializerSnapshot snapshotConfiguration() { } } + /** + * A serializer that is meant to be compatible with any of the serializers only ofter being reconfigured as a new instance. + */ + public static class ReconfigurationRequiringTestTypeSerializer extends TestTypeSerializerBase { + + private static final long serialVersionUID = -7254527815207212324L; + + @Override + public void serialize(TestType record, DataOutputView target) throws IOException { + throw new UnsupportedOperationException("The serializer should have been reconfigured as a new instance; shouldn't be used."); + } + + @Override + public TestType deserialize(DataInputView source) throws IOException { + throw new UnsupportedOperationException("The serializer should have been reconfigured as a new instance; shouldn't be used."); + } + + @Override + public TypeSerializerSnapshot snapshotConfiguration() { + throw new UnsupportedOperationException("The serializer should have been reconfigured as a new instance; shouldn't be used."); + } + } + /** * A serializer that is meant to be incompatible with any of the serializers. */ diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V1TestTypeSerializerSnapshot.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V1TestTypeSerializerSnapshot.java index b2b802a30c2c54..9f78ce32fdd9e9 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V1TestTypeSerializerSnapshot.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V1TestTypeSerializerSnapshot.java @@ -42,8 +42,13 @@ public TypeSerializerSchemaCompatibility resolveSchemaCompatibility(Ty return TypeSerializerSchemaCompatibility.compatibleAsIs(); } else if (newSerializer instanceof TestType.V2TestTypeSerializer) { return TypeSerializerSchemaCompatibility.compatibleAfterMigration(); - } else { + } else if (newSerializer instanceof TestType.ReconfigurationRequiringTestTypeSerializer) { + // we mimic the reconfiguration by just re-instantiating the correct serializer + return TypeSerializerSchemaCompatibility.compatibleWithReconfiguredSerializer(new TestType.V1TestTypeSerializer()); + } else if (newSerializer instanceof TestType.IncompatibleTestTypeSerializer) { return TypeSerializerSchemaCompatibility.incompatible(); + } else { + throw new IllegalStateException("Unknown serializer class for TestType."); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V2TestTypeSerializerSnapshot.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V2TestTypeSerializerSnapshot.java index 3cd4fff8d86631..983a2ae88c58d6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V2TestTypeSerializerSnapshot.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/statemigration/V2TestTypeSerializerSnapshot.java @@ -40,8 +40,17 @@ public int getCurrentVersion() { public TypeSerializerSchemaCompatibility resolveSchemaCompatibility(TypeSerializer newSerializer) { if (newSerializer instanceof TestType.V2TestTypeSerializer) { return TypeSerializerSchemaCompatibility.compatibleAsIs(); - } else { + } else if (newSerializer instanceof TestType.ReconfigurationRequiringTestTypeSerializer) { + // we mimic the reconfiguration by just re-instantiating the correct serializer + return TypeSerializerSchemaCompatibility.compatibleWithReconfiguredSerializer(new TestType.V2TestTypeSerializer()); + } else if ( + // migrating from V2 -> V1 is not supported + newSerializer instanceof TestType.V1TestTypeSerializer + || newSerializer instanceof TestType.IncompatibleTestTypeSerializer) { + return TypeSerializerSchemaCompatibility.incompatible(); + } else { + throw new IllegalStateException("Unknown serializer class for TestType."); } } diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 6b0039342de95c..9335762d63b713 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -340,7 +340,7 @@ public Stream getKeys(String state, N namespace) { final TypeSerializer namespaceSerializer = registeredKeyValueStateBackendMetaInfo.getNamespaceSerializer(); final DataOutputSerializer namespaceOutputView = new DataOutputSerializer(8); - boolean ambiguousKeyPossible = RocksDBKeySerializationUtils.isAmbiguousKeyPossible(keySerializer, namespaceSerializer); + boolean ambiguousKeyPossible = RocksDBKeySerializationUtils.isAmbiguousKeyPossible(getKeySerializer(), namespaceSerializer); final byte[] nameSpaceBytes; try { RocksDBKeySerializationUtils.writeNameSpace( @@ -356,7 +356,7 @@ public Stream getKeys(String state, N namespace) { RocksIteratorWrapper iterator = getRocksIterator(db, columnInfo.f0); iterator.seekToFirst(); - final RocksStateKeysIterator iteratorWrapper = new RocksStateKeysIterator<>(iterator, state, keySerializer, keyGroupPrefixBytes, + final RocksStateKeysIterator iteratorWrapper = new RocksStateKeysIterator<>(iterator, state, getKeySerializer(), keyGroupPrefixBytes, ambiguousKeyPossible, nameSpaceBytes); Stream targetStream = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iteratorWrapper, Spliterator.ORDERED), false); @@ -550,7 +550,7 @@ void initializeSnapshotStrategy( new RocksFullSnapshotStrategy<>( db, rocksDBResourceGuard, - keySerializer, + getKeySerializer(), kvStateInformation, keyGroupRange, keyGroupPrefixBytes, @@ -577,7 +577,7 @@ void initializeSnapshotStrategy( this.checkpointSnapshotStrategy = new RocksIncrementalSnapshotStrategy<>( db, rocksDBResourceGuard, - keySerializer, + getKeySerializer(), kvStateInformation, keyGroupRange, keyGroupPrefixBytes, @@ -669,6 +669,8 @@ private static final class RocksDBFullRestoreOperation { /** The compression decorator that was used for writing the state, as determined by the meta data. */ private StreamCompressionDecorator keygroupStreamCompressionDecorator; + private boolean isKeySerializerCompatibilityChecked; + /** * Creates a restore operation object for the given state backend instance. * @@ -734,11 +736,16 @@ private void restoreKVStateMetaData() throws IOException, StateMigrationExceptio serializationProxy.read(currentStateHandleInView); - // check for key serializer compatibility; this also reconfigures the - // key serializer to be compatible, if it is required and is possible - if (!serializationProxy.getKeySerializerConfigSnapshot() - .resolveSchemaCompatibility(rocksDBKeyedStateBackend.keySerializer).isCompatibleAsIs()) { - throw new StateMigrationException("The new key serializer must be compatible."); + if (!isKeySerializerCompatibilityChecked) { + // check for key serializer compatibility; this also reconfigures the + // key serializer to be compatible, if it is required and is possible + TypeSerializerSchemaCompatibility keySerializerSchemaCompat = + rocksDBKeyedStateBackend.checkKeySerializerSchemaCompatibility(serializationProxy.getKeySerializerConfigSnapshot()); + if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) { + throw new StateMigrationException("The new key serializer must be compatible."); + } + + isKeySerializerCompatibilityChecked = true; } this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ? @@ -835,6 +842,7 @@ private static class RocksDBIncrementalRestoreOperation { private final SortedMap> restoredSstFiles; private UUID restoredBackendUID; private long lastCompletedCheckpointId; + private boolean isKeySerializerCompatibilityChecked; private RocksDBIncrementalRestoreOperation(RocksDBKeyedStateBackend stateBackend) { @@ -1279,11 +1287,16 @@ private List readMetaData( DataInputView in = new DataInputViewStreamWrapper(inputStream); serializationProxy.read(in); - // check for key serializer compatibility; this also reconfigures the - // key serializer to be compatible, if it is required and is possible - if (!serializationProxy.getKeySerializerConfigSnapshot() - .resolveSchemaCompatibility(stateBackend.keySerializer).isCompatibleAsIs()) { - throw new StateMigrationException("The new key serializer must be compatible."); + if (!isKeySerializerCompatibilityChecked) { + // check for key serializer compatibility; this also reconfigures the + // key serializer to be compatible, if it is required and is possible + TypeSerializerSchemaCompatibility keySerializerSchemaCompat = + stateBackend.checkKeySerializerSchemaCompatibility(serializationProxy.getKeySerializerConfigSnapshot()); + if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) { + throw new StateMigrationException("The new key serializer must be compatible."); + } + + isKeySerializerCompatibilityChecked = true; } return serializationProxy.getStateMetaInfoSnapshots(); @@ -1360,7 +1373,7 @@ private RegisteredKeyValueStateBackendMetaInfo u restoredKvStateMetaInfo.updateSnapshotTransformer(snapshotTransformer); TypeSerializerSchemaCompatibility s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer); - if (!s.isCompatibleAsIs()) { + if (s.isCompatibleAfterMigration() || s.isIncompatible()) { throw new StateMigrationException("The new namespace serializer must be compatible."); } From 29174f89d4921625eb058fa4bfcf8fe98c6d6e77 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Tue, 18 Dec 2018 23:06:38 +0800 Subject: [PATCH 041/147] [hotfix] Rename keySerializerConfigSnapshot to keySerializerSnapshot in KeyedBackendSerializationProxy Remove "config" from the field names and getter name to reflect the new abstraction TypeSerializerSnapshot. --- .../state/KeyedBackendSerializationProxy.java | 16 ++++++++-------- .../runtime/state/StateSerializerProvider.java | 2 +- .../state/heap/HeapKeyedStateBackend.java | 2 +- .../runtime/state/SerializationProxiesTest.java | 2 +- .../state/RocksDBKeyedStateBackend.java | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java index 585490dc75f45d..6de7c03e2cb54a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java @@ -64,7 +64,7 @@ public class KeyedBackendSerializationProxy extends VersionedIOReadableWritab // TODO the keySerializer field should be removed, once all serializers have the restoreSerializer() method implemented private TypeSerializer keySerializer; - private TypeSerializerSnapshot keySerializerConfigSnapshot; + private TypeSerializerSnapshot keySerializerSnapshot; private List stateMetaInfoSnapshots; @@ -82,7 +82,7 @@ public KeyedBackendSerializationProxy( this.usingKeyGroupCompression = compression; this.keySerializer = Preconditions.checkNotNull(keySerializer); - this.keySerializerConfigSnapshot = Preconditions.checkNotNull(keySerializer.snapshotConfiguration()); + this.keySerializerSnapshot = Preconditions.checkNotNull(keySerializer.snapshotConfiguration()); Preconditions.checkNotNull(stateMetaInfoSnapshots); Preconditions.checkArgument(stateMetaInfoSnapshots.size() <= Short.MAX_VALUE); @@ -93,8 +93,8 @@ public List getStateMetaInfoSnapshots() { return stateMetaInfoSnapshots; } - public TypeSerializerSnapshot getKeySerializerConfigSnapshot() { - return keySerializerConfigSnapshot; + public TypeSerializerSnapshot getKeySerializerSnapshot() { + return keySerializerSnapshot; } public boolean isUsingKeyGroupCompression() { @@ -118,7 +118,7 @@ public void write(DataOutputView out) throws IOException { // write the compression format used to write each key-group out.writeBoolean(usingKeyGroupCompression); - TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(out, keySerializerConfigSnapshot, keySerializer); + TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(out, keySerializerSnapshot, keySerializer); // write individual registered keyed state metainfos out.writeShort(stateMetaInfoSnapshots.size()); @@ -142,14 +142,14 @@ public void read(DataInputView in) throws IOException { // only starting from version 3, we have the key serializer and its config snapshot written if (readVersion >= 6) { - this.keySerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( + this.keySerializerSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( in, userCodeClassLoader, null); } else if (readVersion >= 3) { Tuple2, TypeSerializerSnapshot> keySerializerAndConfig = TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader).get(0); - this.keySerializerConfigSnapshot = (TypeSerializerSnapshot) keySerializerAndConfig.f1; + this.keySerializerSnapshot = (TypeSerializerSnapshot) keySerializerAndConfig.f1; } else { - this.keySerializerConfigSnapshot = new BackwardsCompatibleSerializerSnapshot<>( + this.keySerializerSnapshot = new BackwardsCompatibleSerializerSnapshot<>( TypeSerializerSerializationUtil.tryReadSerializer(in, userCodeClassLoader, true)); } this.keySerializer = null; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java index bad37ff87078c4..167e6ab5f308da 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateSerializerProvider.java @@ -162,7 +162,7 @@ public final TypeSerializer currentSchemaSerializer() { // if we are not yet registered with a new serializer, // we can just use the restore serializer to read / write the state. return previousSchemaSerializer(); - }; + } /** * Gets the serializer that recognizes the previous serialization schema of the state. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java index c95741429d85e0..2499898220ce06 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java @@ -395,7 +395,7 @@ private void restorePartitionedState(Collection state) throws // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible TypeSerializerSchemaCompatibility keySerializerSchemaCompat = - checkKeySerializerSchemaCompatibility(serializationProxy.getKeySerializerConfigSnapshot()); + checkKeySerializerSchemaCompatibility(serializationProxy.getKeySerializerSnapshot()); if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) { throw new StateMigrationException("The new key serializer must be compatible."); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java index 55aacb230574c1..7e497be3555085 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java @@ -75,7 +75,7 @@ public void testKeyedBackendSerializationProxyRoundtrip() throws Exception { } Assert.assertTrue(serializationProxy.isUsingKeyGroupCompression()); - Assert.assertTrue(serializationProxy.getKeySerializerConfigSnapshot() instanceof IntSerializer.IntSerializerSnapshot); + Assert.assertTrue(serializationProxy.getKeySerializerSnapshot() instanceof IntSerializer.IntSerializerSnapshot); assertEqualStateMetaInfoSnapshotsLists(stateMetaInfoList, serializationProxy.getStateMetaInfoSnapshots()); } diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 9335762d63b713..5a47b306596d6b 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -740,7 +740,7 @@ private void restoreKVStateMetaData() throws IOException, StateMigrationExceptio // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible TypeSerializerSchemaCompatibility keySerializerSchemaCompat = - rocksDBKeyedStateBackend.checkKeySerializerSchemaCompatibility(serializationProxy.getKeySerializerConfigSnapshot()); + rocksDBKeyedStateBackend.checkKeySerializerSchemaCompatibility(serializationProxy.getKeySerializerSnapshot()); if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) { throw new StateMigrationException("The new key serializer must be compatible."); } @@ -1291,7 +1291,7 @@ private List readMetaData( // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible TypeSerializerSchemaCompatibility keySerializerSchemaCompat = - stateBackend.checkKeySerializerSchemaCompatibility(serializationProxy.getKeySerializerConfigSnapshot()); + stateBackend.checkKeySerializerSchemaCompatibility(serializationProxy.getKeySerializerSnapshot()); if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) { throw new StateMigrationException("The new key serializer must be compatible."); } From 583b3a2ea78d91f4f805d8a6443edbc72640ca74 Mon Sep 17 00:00:00 2001 From: blueszheng Date: Thu, 3 Jan 2019 19:48:58 +0800 Subject: [PATCH 042/147] [hotfix] [doc] Fix an error in state_backends This closes #7401. --- docs/ops/state/state_backends.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ops/state/state_backends.md b/docs/ops/state/state_backends.md index e4fe5bda063045..70663f696b2159 100644 --- a/docs/ops/state/state_backends.md +++ b/docs/ops/state/state_backends.md @@ -152,7 +152,7 @@ env.setStateBackend(new FsStateBackend("hdfs://namenode:40010/flink/checkpoints" A default state backend can be configured in the `flink-conf.yaml`, using the configuration key `state.backend`. Possible values for the config entry are *jobmanager* (MemoryStateBackend), *filesystem* (FsStateBackend), *rocksdb* (RocksDBStateBackend), or the fully qualified class -name of the class that implements the state backend factory [FsStateBackendFactory](https://github.com/apache/flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/state/filesystem/FsStateBackendFactory.java), +name of the class that implements the state backend factory [StateBackendFactory](https://github.com/apache/flink/blob/master/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateBackendFactory.java), such as `org.apache.flink.contrib.streaming.state.RocksDBStateBackendFactory` for RocksDBStateBackend. The `state.checkpoints.dir` option defines the directory to which all backends write checkpoint data and meta data files. From 3d5b6b166202c423b1529316bb9a9a755628ae1a Mon Sep 17 00:00:00 2001 From: lamber-ken <2217232293@qq.com> Date: Tue, 25 Dec 2018 23:24:08 +0800 Subject: [PATCH 043/147] [hotfix] [doc] Fix incorrect http port in Elasticsearch 6.x Scala doc This closes #7361. --- docs/dev/connectors/elasticsearch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/connectors/elasticsearch.md b/docs/dev/connectors/elasticsearch.md index de24149bc758a3..d0a159b254e034 100644 --- a/docs/dev/connectors/elasticsearch.md +++ b/docs/dev/connectors/elasticsearch.md @@ -340,8 +340,8 @@ import java.util.List val input: DataStream[String] = ... val httpHosts = new java.util.ArrayList[HttpHost] -httpHosts.add(new HttpHost("127.0.0.1", 9300, "http")) -httpHosts.add(new HttpHost("10.2.3.1", 9300, "http")) +httpHosts.add(new HttpHost("127.0.0.1", 9200, "http")) +httpHosts.add(new HttpHost("10.2.3.1", 9200, "http")) val esSinkBuilder = new ElasticsearchSink.Builer[String]( httpHosts, From 35d16f9a14a175de5dafbf37bb45fa3a1d42a16e Mon Sep 17 00:00:00 2001 From: "zhuzhu.zz" Date: Fri, 14 Dec 2018 18:07:32 +0800 Subject: [PATCH 044/147] [FLINK-11165] Refine the task deploying log for easier finding of task locations --- .../java/org/apache/flink/runtime/executiongraph/Execution.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java index e3b501e52e8376..a48352c03c52f4 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java @@ -597,7 +597,7 @@ public void deploy() throws JobException { if (LOG.isInfoEnabled()) { LOG.info(String.format("Deploying %s (attempt #%d) to %s", vertex.getTaskNameWithSubtaskIndex(), - attemptNumber, getAssignedResourceLocation().getHostname())); + attemptNumber, getAssignedResourceLocation())); } final TaskDeploymentDescriptor deployment = vertex.createDeploymentDescriptor( From fe489bd31b2ce0f2d4c9254fd24e8aaf00111884 Mon Sep 17 00:00:00 2001 From: leesf <490081539@qq.com> Date: Sun, 23 Dec 2018 10:49:55 +0800 Subject: [PATCH 045/147] [FLINK-11189] Deprecated documentation for readSequenceFile function --- docs/dev/batch/index.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/dev/batch/index.md b/docs/dev/batch/index.md index 0a498dfe2a1f62..521f4624b1a961 100644 --- a/docs/dev/batch/index.md +++ b/docs/dev/batch/index.md @@ -826,9 +826,6 @@ File-based: - `readFileOfPrimitives(path, delimiter, Class)` / `PrimitiveInputFormat` - Parses files of new-line (or another char sequence) delimited primitive data types such as `String` or `Integer` using the given delimiter. -- `readSequenceFile(Key, Value, path)` / `SequenceFileInputFormat` - Creates a JobConf and reads file from the specified path with - type SequenceFileInputFormat, Key class and Value class and returns them as Tuple2. - Collection-based: @@ -879,7 +876,7 @@ DataSet> csvInput = env.readCsvFile("hdfs:///the/CSV/file") // read a file from the specified path of type SequenceFileInputFormat DataSet> tuples = - env.readSequenceFile(IntWritable.class, Text.class, "hdfs://nnHost:nnPort/path/to/file"); + env.createInput(HadoopInputs.readSequenceFile(IntWritable.class, Text.class, "hdfs://nnHost:nnPort/path/to/file")); // creates a set from some given elements DataSet value = env.fromElements("Foo", "bar", "foobar", "fubar"); From 0b28bfd0d9f368154bfcd1564388935eff05b055 Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Mon, 7 Jan 2019 17:27:27 +0100 Subject: [PATCH 046/147] [FLINK-11273][state] Fix shared InputView object between event processing and queryable state thread This was introduced with FLINK-9702. --- .../contrib/streaming/state/RocksDBMapState.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java index 13cbdedb1d91e4..4c174d43ef8438 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java @@ -242,7 +242,6 @@ public void clear() { } @Override - @SuppressWarnings("unchecked") public byte[] getSerializedValue( final byte[] serializedKeyAndNamespace, final TypeSerializer safeKeySerializer, @@ -274,13 +273,14 @@ public byte[] getSerializedValue( final TypeSerializer dupUserKeySerializer = serializer.getKeySerializer(); final TypeSerializer dupUserValueSerializer = serializer.getValueSerializer(); + final DataInputDeserializer inputView = new DataInputDeserializer(); final Iterator> iterator = new RocksDBMapIterator>( backend.db, keyPrefixBytes, dupUserKeySerializer, dupUserValueSerializer, - dataInputView + inputView ) { @Override @@ -368,7 +368,6 @@ private class RocksDBMapEntry implements Map.Entry { private final TypeSerializer valueSerializer; private final DataInputDeserializer dataInputView; - private final DataOutputSerializer dataOutputView; RocksDBMapEntry( @Nonnull final RocksDB db, @@ -377,8 +376,7 @@ private class RocksDBMapEntry implements Map.Entry { @Nonnull final byte[] rawValueBytes, @Nonnull final TypeSerializer keySerializer, @Nonnull final TypeSerializer valueSerializer, - @Nonnull DataInputDeserializer dataInputView, - @Nonnull DataOutputSerializer dataOutputView) { + @Nonnull DataInputDeserializer dataInputView) { this.db = db; this.userKeyOffset = userKeyOffset; @@ -389,7 +387,6 @@ private class RocksDBMapEntry implements Map.Entry { this.rawValueBytes = rawValueBytes; this.deleted = false; this.dataInputView = dataInputView; - this.dataOutputView = dataOutputView; } public void remove() { @@ -585,8 +582,7 @@ private void loadCache() { iterator.value(), keySerializer, valueSerializer, - dataInputView, - dataOutputView); + dataInputView); cacheEntries.add(entry); From 79f317026bcf8f5829ed32f592c91aa2a99a4051 Mon Sep 17 00:00:00 2001 From: "acqua.csq" Date: Thu, 27 Dec 2018 14:33:10 +0800 Subject: [PATCH 047/147] [FLINK-11161][scala-shell] Fix unable to import java packages in scala-shell. This closes #7370 --- .../apache/flink/api/scala/FlinkILoop.scala | 1 - .../flink/api/scala/ScalaShellITCase.scala | 89 +++++++++++++++++-- .../start-script/start-scala-shell.sh | 2 +- 3 files changed, 84 insertions(+), 8 deletions(-) diff --git a/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkILoop.scala b/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkILoop.scala index c124d8ea8a3c70..27db2a325e00ae 100644 --- a/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkILoop.scala +++ b/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkILoop.scala @@ -149,7 +149,6 @@ class FlinkILoop( "org.apache.flink.api.scala.utils._", "org.apache.flink.streaming.api.scala._", "org.apache.flink.streaming.api.windowing.time._", - "org.apache.flink.table.api._", "org.apache.flink.table.api.scala._", "org.apache.flink.types.Row" ) diff --git a/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala b/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala index fc90d8d143c60b..8fd94e5d692db9 100644 --- a/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala +++ b/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala @@ -183,9 +183,9 @@ class ScalaShellITCase extends TestLogger { |:q """.stripMargin val output = processInShell(input) - Assert.assertFalse(output.contains("failed")) - Assert.assertFalse(output.contains("error")) - Assert.assertFalse(output.contains("Exception")) + Assert.assertFalse(output.toLowerCase.contains("failed")) + Assert.assertFalse(output.toLowerCase.contains("error")) + Assert.assertFalse(output.toLowerCase.contains("exception")) Assert.assertTrue(output.contains("1,Hi")) Assert.assertTrue(output.contains("3,Hello world")) } @@ -218,9 +218,9 @@ class ScalaShellITCase extends TestLogger { Assert.assertTrue(output.contains("6,1")) Assert.assertTrue(output.contains("1,2")) Assert.assertTrue(output.contains("2,1")) - Assert.assertFalse(output.contains("failed")) - Assert.assertFalse(output.contains("error")) - Assert.assertFalse(output.contains("Exception")) + Assert.assertFalse(output.toLowerCase.contains("failed")) + Assert.assertFalse(output.toLowerCase.contains("error")) + Assert.assertFalse(output.toLowerCase.contains("exception")) } /** @@ -362,6 +362,83 @@ class ScalaShellITCase extends TestLogger { Assert.assertFalse(output.contains("ERROR")) Assert.assertFalse(output.contains("Exception")) } + + @Test + def testImportJavaCollection(): Unit = { + val input = """ + import java.util.List + val jul: List[Int] = new java.util.ArrayList[Int]() + jul.add(2) + jul.add(4) + jul.add(6) + jul.add(8) + jul.add(10) + val str = "the java list size is: " + jul.size + """.stripMargin + + val output = processInShell(input) + + Assert.assertTrue(output.contains("the java list size is: 5")) + Assert.assertFalse(output.toLowerCase.contains("failed")) + Assert.assertFalse(output.toLowerCase.contains("error")) + Assert.assertFalse(output.toLowerCase.contains("exception")) + + } + + @Test + def testImplicitConversionBetweenJavaAndScala(): Unit = { + val input = + """ + import collection.JavaConversions._ + import scala.collection.mutable.ArrayBuffer + val jul:java.util.List[Int] = ArrayBuffer(1,2,3,4,5) + val buf: Seq[Int] = jul + var sum = 0 + buf.foreach(num => sum += num) + val str = "sum is: " + sum + val scala2jul = List(1,2,3) + scala2jul.add(7) + """.stripMargin + + val output = processInShell(input) + + Assert.assertTrue(output.contains("sum is: 15")) + Assert.assertFalse(output.toLowerCase.contains("failed")) + Assert.assertFalse(output.toLowerCase.contains("error")) + Assert.assertTrue(output.contains("java.lang.UnsupportedOperationException")) + } + + @Test + def testImportPackageConflict(): Unit = { + val input = + """ + import org.apache.flink.table.api._ + import java.util.List + val jul: List[Int] = new java.util.ArrayList[Int]() + jul.add(2) + jul.add(4) + jul.add(6) + jul.add(8) + jul.add(10) + val str = "the java list size is: " + jul.size + """.stripMargin + + val output = processInShell(input) + Assert.assertTrue(output.contains("error: object util is not a member of package org.apache." + + "flink.table.api.java")) + } + + @Test + def testGetMultiExecutionEnvironment(): Unit = { + val input = + """ + |val newEnv = ExecutionEnvironment.getExecutionEnvironment + """.stripMargin + val output = processInShell(input) + Assert.assertTrue(output.contains("java.lang.UnsupportedOperationException: Execution " + + "Environment is already defined for this shell.")) + } + } object ScalaShellITCase { diff --git a/flink-scala-shell/start-script/start-scala-shell.sh b/flink-scala-shell/start-script/start-scala-shell.sh index e3571145f13ed3..489d50b25d51c1 100644 --- a/flink-scala-shell/start-script/start-scala-shell.sh +++ b/flink-scala-shell/start-script/start-scala-shell.sh @@ -54,7 +54,7 @@ FLINK_CLASSPATH=`constructFlinkClassPath` # Append flink-table jar into class path opt=`dirname "$0"` -opt=`cd ../"$opt"/opt; pwd` +opt=`cd "$opt"/../opt; pwd` FLINK_TABLE_LIB_PATH=$opt/`ls $opt|grep flink-table_*` FLINK_CLASSPATH=$FLINK_CLASSPATH:$FLINK_TABLE_LIB_PATH From 90f3b424c74b3673f90093acc8384acf1ef69f74 Mon Sep 17 00:00:00 2001 From: XuQianJin-Stars Date: Tue, 8 Jan 2019 12:45:26 +0800 Subject: [PATCH 048/147] [FLINK-11279] [table] Fix week interval parsing in ExpressionParser This closes #7426. --- .../apache/flink/table/expressions/ExpressionParser.scala | 2 +- .../apache/flink/table/expressions/ScalarFunctionsTest.scala | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala index d5d64b48d69d70..b45ce801018ea1 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionParser.scala @@ -303,7 +303,7 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers { case expr ~ _ ~ (MONTHS.key | MONTH.key) => toMonthInterval(expr, 1) - case expr ~ _ ~ (WEEKS.key | WEEKS.key) => toMilliInterval(expr, 7 * MILLIS_PER_DAY) + case expr ~ _ ~ (WEEKS.key | WEEK.key) => toMilliInterval(expr, 7 * MILLIS_PER_DAY) case expr ~ _ ~ (DAYS.key | DAY.key) => toMilliInterval(expr, MILLIS_PER_DAY) diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala index 31c9b6c3d0167a..23bd2ae3379983 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala @@ -2764,6 +2764,11 @@ class ScalarFunctionsTest extends ScalarTypesTestBase { "'2016-03-31'.toDate - 1.month", "timestampadd(MONTH, -1, date '2016-03-31')", "2016-02-29") + + testAllApis("2016-03-31".toDate - 1.week, + "'2016-03-31'.toDate - 1.week", + "timestampadd(WEEK, -1, date '2016-03-31')", + "2016-03-24") } // ---------------------------------------------------------------------------------------------- From 392647d09cc43ea94a53aa56d8c46bfe7d16d53e Mon Sep 17 00:00:00 2001 From: Dawid Wysakowicz Date: Tue, 18 Dec 2018 15:37:42 +0100 Subject: [PATCH 049/147] [FLINK-11191] [table] Check for ambiguous columns in MATCH_RECOGNIZE Added a validation that checks if no ambiguous columns are defined in MATCH_RECOGNIZE clause. Without the check there is a cryptic message thrown from code generation stack. This closes #7328. --- .../datastream/DataStreamMatchRule.scala | 53 +++++++++++++++++-- .../match/MatchRecognizeValidationTest.scala | 24 +++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala index bc0f56e38bac12..046a49aefdfe66 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala @@ -18,12 +18,15 @@ package org.apache.flink.table.plan.rules.datastream +import java.util.{List => JList} + import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelTraitSet} import org.apache.calcite.rel.RelNode +import org.apache.calcite.rel.`type`.RelDataType import org.apache.calcite.rel.convert.ConverterRule -import org.apache.calcite.rex.{RexCall, RexNode} +import org.apache.calcite.rex.{RexCall, RexInputRef, RexNode} import org.apache.calcite.sql.SqlAggFunction -import org.apache.flink.table.api.TableException +import org.apache.flink.table.api.{TableException, ValidationException} import org.apache.flink.table.plan.logical.MatchRecognize import org.apache.flink.table.plan.nodes.FlinkConventions import org.apache.flink.table.plan.nodes.datastream.DataStreamMatch @@ -33,6 +36,7 @@ import org.apache.flink.table.plan.util.RexDefaultVisitor import org.apache.flink.table.util.MatchUtil import scala.collection.JavaConverters._ +import scala.collection.mutable class DataStreamMatchRule extends ConverterRule( @@ -46,6 +50,8 @@ class DataStreamMatchRule validateAggregations(logicalMatch.getMeasures.values().asScala) validateAggregations(logicalMatch.getPatternDefinitions.values().asScala) + // This check might be obsolete once CALCITE-2747 is resolved + validateAmbiguousColumns(logicalMatch) true } @@ -91,7 +97,48 @@ class DataStreamMatchRule expr.foreach(_.accept(validator)) } - class AggregationsValidator extends RexDefaultVisitor[Object] { + private def validateAmbiguousColumns(logicalMatch: FlinkLogicalMatch): Unit = { + if (logicalMatch.isAllRows) { + throw new TableException("All rows per match mode is not supported yet.") + } else { + val refNameFinder = new RefNameFinder(logicalMatch.getInput.getRowType) + validateAmbiguousColumnsOnRowPerMatch( + logicalMatch.getPartitionKeys, + logicalMatch.getMeasures.keySet().asScala, + logicalMatch.getRowType, + refNameFinder) + } + } + + private def validateAmbiguousColumnsOnRowPerMatch( + partitionKeys: JList[RexNode], + measuresNames: mutable.Set[String], + expectedSchema: RelDataType, + refNameFinder: RefNameFinder) + : Unit = { + val actualSize = partitionKeys.size() + measuresNames.size + val expectedSize = expectedSchema.getFieldCount + if (actualSize != expectedSize) { + //try to find ambiguous column + + val ambiguousColumns = partitionKeys.asScala.map(_.accept(refNameFinder)) + .filter(measuresNames.contains).mkString("{", ", ", "}") + + throw new ValidationException(s"Columns ambiguously defined: $ambiguousColumns") + } + } + + private class RefNameFinder(inputSchema: RelDataType) extends RexDefaultVisitor[String] { + + override def visitInputRef(inputRef: RexInputRef): String = { + inputSchema.getFieldList.get(inputRef.getIndex).getName + } + + override def visitNode(rexNode: RexNode): String = + throw new TableException(s"PARTITION BY clause accepts only input reference. Found $rexNode") + } + + private class AggregationsValidator extends RexDefaultVisitor[Object] { override def visitCall(call: RexCall): AnyRef = { call.getOperator match { diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchRecognizeValidationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchRecognizeValidationTest.scala index b77a60e1af5195..4e6562da281ff0 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchRecognizeValidationTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/match/MatchRecognizeValidationTest.scala @@ -174,6 +174,30 @@ class MatchRecognizeValidationTest extends TableTestBase { streamUtils.tableEnv.sqlQuery(sqlQuery).toAppendStream[Row] } + @Test + def testValidatingAmbiguousColumns(): Unit = { + thrown.expectMessage("Columns ambiguously defined: {symbol, price}") + thrown.expect(classOf[ValidationException]) + + val sqlQuery = + s""" + |SELECT * + |FROM Ticker + |MATCH_RECOGNIZE ( + | PARTITION BY symbol, price + | ORDER BY proctime + | MEASURES + | A.symbol AS symbol, + | A.price AS price + | PATTERN (A) + | DEFINE + | A AS symbol = 'a' + |) AS T + |""".stripMargin + + streamUtils.tableEnv.sqlQuery(sqlQuery).toAppendStream[Row] + } + // *************************************************************************************** // * Those validations are temporary. We should remove those tests once we support those * // * features. * From d77151c3907895abc586e0fc651a49f148a302be Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Tue, 8 Jan 2019 11:23:20 +0100 Subject: [PATCH 050/147] Revert "[FLINK-10848][YARN] properly remove YARN ContainerRequest upon container allocation success" This reverts commit e26d90fc86b266978b4bac84fe02ca34b62983fe. --- .../test/java/org/apache/flink/yarn/YarnTestBase.java | 2 -- .../org/apache/flink/yarn/YarnFlinkResourceManager.java | 2 -- .../java/org/apache/flink/yarn/YarnResourceManager.java | 3 +-- .../apache/flink/yarn/YarnFlinkResourceManagerTest.java | 9 --------- .../org/apache/flink/yarn/YarnResourceManagerTest.java | 4 ---- 5 files changed, 1 insertion(+), 19 deletions(-) diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java index f1e6a3a767cd92..3763f6592afeee 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java @@ -165,8 +165,6 @@ public abstract class YarnTestBase extends TestLogger { YARN_CONFIGURATION.setInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 3600); YARN_CONFIGURATION.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false); YARN_CONFIGURATION.setInt(YarnConfiguration.NM_VCORES, 666); // memory is overwritten in the MiniYARNCluster. - YARN_CONFIGURATION.set("yarn.scheduler.capacity.resource-calculator", - "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator"); // so we have to change the number of cores for testing. YARN_CONFIGURATION.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 20000); // 20 seconds expiry (to ensure we properly heartbeat with YARN). } diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java index 3327505e32df56..8e686bbbe34c10 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java @@ -438,8 +438,6 @@ private void containersAllocated(List containers) { numPendingContainerRequests = Math.max(0, numPendingContainerRequests - 1); LOG.info("Received new container: {} - Remaining pending container requests: {}", container.getId(), numPendingContainerRequests); - resourceManagerClient.removeContainerRequest(new AMRMClient.ContainerRequest( - container.getResource(), null, null, container.getPriority())); // decide whether to return the container, or whether to start a TaskManager if (numRegistered + containersInLaunch.size() < numRequired) { diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java index 6669f16fa405dd..6ff5cd66487112 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java @@ -361,8 +361,7 @@ public void onContainersAllocated(List containers) { "Received new container: {} - Remaining pending container requests: {}", container.getId(), numPendingContainerRequests); - resourceManagerClient.removeContainerRequest(new AMRMClient.ContainerRequest( - container.getResource(), null, null, container.getPriority())); + if (numPendingContainerRequests > 0) { numPendingContainerRequests--; diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java index d665df6bc7ce07..10b2ce97d6fe48 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java @@ -43,8 +43,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.NMClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; @@ -71,11 +69,8 @@ import scala.concurrent.duration.FiniteDuration; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -130,8 +125,6 @@ public void testYarnFlinkResourceManagerJobManagerLostLeadership() throws Except 1), i)); when(mockContainer.getNodeId()).thenReturn(NodeId.newInstance("container", 1234)); - when(mockContainer.getResource()).thenReturn(Resource.newInstance(200, 1)); - when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED); containerList.add(mockContainer); } @@ -240,8 +233,6 @@ public Object answer(InvocationOnMock invocation) throws Throwable { int numberOfRegisteredResources = (Integer) Await.result(numberOfRegisteredResourcesFuture, deadline.timeLeft()); - verify(resourceManagerClient, times(numInitialTaskManagers)).removeContainerRequest( - any(AMRMClient.ContainerRequest.class)); assertEquals(numInitialTaskManagers, numberOfRegisteredResources); } finally { if (resourceManager != null) { diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java index ee325dad0f7b1d..d41d42d7a05f97 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java @@ -401,8 +401,6 @@ public void testStopWorker() throws Exception { resourceManager.onContainersAllocated(ImmutableList.of(testingContainer)); verify(mockResourceManagerClient).addContainerRequest(any(AMRMClient.ContainerRequest.class)); - verify(mockResourceManagerClient).removeContainerRequest( - any(AMRMClient.ContainerRequest.class)); verify(mockNMClient).startContainer(eq(testingContainer), any(ContainerLaunchContext.class)); // Remote task executor registers with YarnResourceManager. @@ -498,8 +496,6 @@ public void testOnContainerCompleted() throws Exception { resourceManager.onContainersAllocated(ImmutableList.of(testingContainer)); verify(mockResourceManagerClient).addContainerRequest(any(AMRMClient.ContainerRequest.class)); - verify(mockResourceManagerClient).removeContainerRequest( - any(AMRMClient.ContainerRequest.class)); verify(mockNMClient).startContainer(eq(testingContainer), any(ContainerLaunchContext.class)); // Callback from YARN when container is Completed, pending request can not be fulfilled by pending From 7a1147b9b8d5a92ea38e25358a645d35dc16f9b5 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Tue, 8 Jan 2019 11:17:50 +0100 Subject: [PATCH 051/147] [FLINK-11280] [rocksdb] Lazily create RocksDBSerializedCompositeKeyBuilder only after restore Prior to this commit, the composite key builder was created in the constructor of the RocksDBKeyedStateBackend. The creation of the builder requires providing a key serializer. This is problematic, because the key serializer may be reconfigured during the restore phase, therefore invalidating the key serializer used by the composite key builder. This commit resolves this by lazily creating the composite key builder only after the restore phase, which would be the point-in-time when we are certain the key serializer will no longer be changed and is final. --- .../state/RocksDBKeyedStateBackend.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 5a47b306596d6b..e95b7e86cd35db 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -242,8 +242,14 @@ IS createState( /** The native metrics monitor. */ private RocksDBNativeMetricMonitor nativeMetricMonitor; - /** Helper to build the byte arrays of composite keys to address data in RocksDB. Shared across all states.*/ - private final RocksDBSerializedCompositeKeyBuilder sharedRocksKeyBuilder; + /** + * Helper to build the byte arrays of composite keys to address data in RocksDB. Shared across all states. + * + *

We create the builder after the restore phase in the {@link #restore(Object)} method. The timing of + * the creation is important, because only after the restore we are certain that the key serializer + * is final after potential reconfigurations during the restore. + */ + private RocksDBSerializedCompositeKeyBuilder sharedRocksKeyBuilder; public RocksDBKeyedStateBackend( String operatorIdentifier, @@ -297,7 +303,6 @@ public RocksDBKeyedStateBackend( this.kvStateInformation = new LinkedHashMap<>(); this.writeOptions = new WriteOptions().setDisableWAL(true); - this.sharedRocksKeyBuilder = new RocksDBSerializedCompositeKeyBuilder<>(keySerializer, keyGroupPrefixBytes, 32); this.metricOptions = metricOptions; this.metricGroup = metricGroup; @@ -535,6 +540,14 @@ public void restore(Collection restoreState) throws Exception } } + // it is important that we only create the key builder after the restore, and not before; + // restore operations may reconfigure the key serializer, so accessing the key serializer + // only now we can be certain that the key serializer used in the builder is final. + this.sharedRocksKeyBuilder = new RocksDBSerializedCompositeKeyBuilder<>( + getKeySerializer(), + keyGroupPrefixBytes, + 32); + initializeSnapshotStrategy(incrementalRestoreOperation); } catch (Exception ex) { dispose(); From ef08a86b53fc936b4aa2a05d7673b4a373eb8fa2 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Tue, 8 Jan 2019 11:25:57 +0100 Subject: [PATCH 052/147] [hotfix] [rocksdb] Only log snapshot restore message when state is actually restored --- .../contrib/streaming/state/RocksDBKeyedStateBackend.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index e95b7e86cd35db..7ddf9939d632a7 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -517,10 +517,6 @@ public void restore(Collection restoreState) throws Exception LOG.info("Initializing RocksDB keyed state backend."); - if (LOG.isDebugEnabled()) { - LOG.debug("Restoring snapshot from state handles: {}, will use {} thread(s) to download files from DFS.", restoreState, restoringThreadNum); - } - // clear all meta data kvStateInformation.clear(); @@ -529,6 +525,10 @@ public void restore(Collection restoreState) throws Exception if (restoreState == null || restoreState.isEmpty()) { createDB(); } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Restoring snapshot from state handles: {}, will use {} thread(s) to download files from DFS.", restoreState, restoringThreadNum); + } + KeyedStateHandle firstStateHandle = restoreState.iterator().next(); if (firstStateHandle instanceof IncrementalKeyedStateHandle || firstStateHandle instanceof IncrementalLocalKeyedStateHandle) { From f39fd2929f2b231d54cb8435ccf2b84174168ebc Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Tue, 8 Jan 2019 11:27:39 +0100 Subject: [PATCH 053/147] [hotfix] [rocksdb] Remove unused method in RocksDBSerializedCompositeKeyBuilder --- .../state/RocksDBSerializedCompositeKeyBuilder.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBSerializedCompositeKeyBuilder.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBSerializedCompositeKeyBuilder.java index 8e83e292820953..41aa685e45f025 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBSerializedCompositeKeyBuilder.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBSerializedCompositeKeyBuilder.java @@ -198,9 +198,4 @@ boolean isAmbiguousCompositeKeyPossible(TypeSerializer namespaceSerializer) { return keySerializerTypeVariableSized & RocksDBKeySerializationUtils.isSerializerTypeVariableSized(namespaceSerializer); } - - @VisibleForTesting - boolean isKeySerializerTypeVariableSized() { - return keySerializerTypeVariableSized; - } } From 31685a36d1344e34c37b678c0335d560ccf6f4f4 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Tue, 8 Jan 2019 13:42:34 +0100 Subject: [PATCH 054/147] [FLINK-11280] [state backends, tests] Do not set current key before restore in TtlStateTestBase tests --- .../flink/runtime/state/ttl/StateBackendTestContext.java | 3 +-- .../org/apache/flink/runtime/state/ttl/TtlStateTestBase.java | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/StateBackendTestContext.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/StateBackendTestContext.java index eaec234e425e88..c1de3cba49d7f3 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/StateBackendTestContext.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/StateBackendTestContext.java @@ -76,9 +76,8 @@ void createAndRestoreKeyedStateBackend() { keyedStateBackend = stateBackend.createKeyedStateBackend( env, new JobID(), "test", StringSerializer.INSTANCE, 10, new KeyGroupRange(0, 9), env.getTaskKvStateRegistry(), timeProvider); - keyedStateBackend.setCurrentKey("defaultKey"); } catch (Exception e) { - throw new RuntimeException("unexpected"); + throw new RuntimeException("unexpected", e); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/TtlStateTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/TtlStateTestBase.java index f9f108a3a12513..9b6882a35f5799 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/TtlStateTestBase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/TtlStateTestBase.java @@ -113,6 +113,7 @@ private void initTest(StateTtlConfig ttlConfig) throws Exception { this.ttlConfig = ttlConfig; sbetc.createAndRestoreKeyedStateBackend(); sbetc.restoreSnapshot(null); + sbetc.setCurrentKey("defaultKey"); createState(); ctx().initTestValues(); } @@ -129,6 +130,7 @@ private void takeAndRestoreSnapshot() throws Exception { KeyedStateHandle snapshot = sbetc.takeSnapshot(); sbetc.createAndRestoreKeyedStateBackend(); sbetc.restoreSnapshot(snapshot); + sbetc.setCurrentKey("defaultKey"); createState(); } @@ -397,6 +399,7 @@ public void testRestoreTtlAndRegisterNonTtlStateCompatFailure() throws Exception sbetc.createAndRestoreKeyedStateBackend(); sbetc.restoreSnapshot(snapshot); + sbetc.setCurrentKey("defaultKey"); sbetc.createState(ctx().createStateDescriptor(), ""); } From beee5b168875fc54bc09780942c4fc724ffccbf0 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Wed, 5 Dec 2018 15:51:12 +0800 Subject: [PATCH 055/147] [FLINK-11073] [core] Introduce CompositeTypeSerializerSnapshot The CompositeTypeSerializerSnapshot encapsulates logic for handling writing, reading, and deriving final compatibility results for composite serializers that have multiple nested serializers as well as some static outer configuration (e.g. type class in the GenericArraySerializer). his base class has its own versioning for the format in which it writes the outer snapshot and the nested serializer snapshots. The version of the serialization format of this based class is defined by getCurrentVersion(). This is independent of the version in which subclasses writes their outer snapshot, defined by getCurrentOuterSnapshotVersion(). This means that the outer snapshot's version can be maintained only taking into account changes in how the outer snapshot is written. Any changes in the base format does not require upticks in the outer snapshot's version --- .../CompositeSerializerSnapshot.java | 10 + .../CompositeTypeSerializerSnapshot.java | 346 +++++++++++ .../CompositeTypeSerializerSnapshotTest.java | 544 ++++++++++++++++++ 3 files changed, 900 insertions(+) create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshot.java create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshotTest.java diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeSerializerSnapshot.java index 93f5a70325733c..5b3f7751087f98 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeSerializerSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeSerializerSnapshot.java @@ -93,10 +93,20 @@ public TypeSerializer getRestoreSerializer(int pos) { return snapshot.restoreSerializer(); } + /** + * Returns the snapshots of the nested serializers. + * + * @return the snapshots of the nested serializers. + */ + public TypeSerializerSnapshot[] getNestedSerializerSnapshots() { + return nestedSnapshots; + } + /** * Resolves the compatibility of the nested serializer snapshots with the nested * serializers of the new outer serializer. */ + @Deprecated public TypeSerializerSchemaCompatibility resolveCompatibilityWithNested( TypeSerializerSchemaCompatibility outerCompatibility, TypeSerializer... newNestedSerializers) { diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshot.java new file mode 100644 index 00000000000000..71c0836b35304d --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshot.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.typeutils; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.typeutils.base.GenericArraySerializer; +import org.apache.flink.api.common.typeutils.base.ListSerializer; +import org.apache.flink.api.common.typeutils.base.MapSerializer; +import org.apache.flink.api.java.typeutils.runtime.EitherSerializer; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.util.Preconditions; + +import java.io.IOException; + +/** + * A {@link CompositeTypeSerializerSnapshot} is a convenient serializer snapshot class that can be used by + * simple serializers which 1) delegates its serialization to multiple nested serializers, and 2) may contain + * some extra static information that needs to be persisted as part of its snapshot. + * + *

Examples for this would be the {@link ListSerializer}, {@link MapSerializer}, {@link EitherSerializer}, etc., + * in which case the serializer, called the "outer" serializer in this context, has only some nested serializers that + * needs to be persisted as its snapshot, and nothing else that needs to be persisted as the "outer" snapshot. + * An example which has non-empty outer snapshots would be the {@link GenericArraySerializer}, which beyond the + * nested component serializer, also contains a class of the component type that needs to be persisted. + * + *

Serializers that do have some outer snapshot needs to make sure to implement the methods + * {@link #writeOuterSnapshot(DataOutputView)}, {@link #readOuterSnapshot(int, DataInputView, ClassLoader)}, and + * {@link #isOuterSnapshotCompatible(TypeSerializer)} when using this class as the base for its serializer snapshot + * class. By default, the base implementations of these methods are empty, i.e. this class assumes that + * subclasses do not have any outer snapshot that needs to be persisted. + * + *

Snapshot Versioning

+ * + *

This base class has its own versioning for the format in which it writes the outer snapshot and the + * nested serializer snapshots. The version of the serialization format of this based class is defined + * by {@link #getCurrentVersion()}. This is independent of the version in which subclasses writes their outer snapshot, + * defined by {@link #getCurrentOuterSnapshotVersion()}. + * This means that the outer snapshot's version can be maintained only taking into account changes in how the + * outer snapshot is written. Any changes in the base format does not require upticks in the outer snapshot's version. + * + *

Serialization Format + * + *

The current version of the serialization format of a {@link CompositeTypeSerializerSnapshot} is as follows: + * + * 

{@code
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * | CompositeTypeSerializerSnapshot | CompositeTypeSerializerSnapshot |          Outer snapshot         |
+ * |           version               |          MAGIC_NUMBER           |              version            |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |                                               Outer snapshot                                        |
+ * |                                   #writeOuterSnapshot(DataOutputView out)                           |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |      Delegate MAGIC_NUMBER      |         Delegate version        |     Num. nested serializers     |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |                                     Nested serializer snapshots                                     |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * }
+ * + * @param The data type that the originating serializer of this snapshot serializes. + * @param The type of the originating serializer. + */ +@PublicEvolving +public abstract class CompositeTypeSerializerSnapshot implements TypeSerializerSnapshot { + + /** Magic number for integrity checks during deserialization. */ + private static final int MAGIC_NUMBER = 911108; + + /** + * Current version of the base serialization format. + * + *

NOTE: We start from version 3. This version is represented by the {@link #getCurrentVersion()} method. + * Previously, this method was used to represent the outer snapshot's version (now, represented + * by the {@link #getCurrentOuterSnapshotVersion()} method). + * + *

To bridge this transition, we set the starting version of the base format to be at least + * larger than the highest version of previously defined values in implementing subclasses, + * which was {@link #HIGHEST_LEGACY_READ_VERSION}. This allows us to identify legacy deserialization paths, + * which did not contain versioning for the base format, simply by checking if the read + * version of the snapshot is smaller than or equal to {@link #HIGHEST_LEGACY_READ_VERSION}. + */ + private static final int VERSION = 3; + + private static final int HIGHEST_LEGACY_READ_VERSION = 2; + + private CompositeSerializerSnapshot compositeSerializerSnapshot; + + private final Class correspondingSerializerClass; + + /** + * Constructor to be used for read instantiation. + * + * @param correspondingSerializerClass the expected class of the new serializer. + */ + public CompositeTypeSerializerSnapshot(Class correspondingSerializerClass) { + this.correspondingSerializerClass = Preconditions.checkNotNull(correspondingSerializerClass); + } + + /** + * Constructor to be used for writing the snapshot. + * + * @param serializerInstance an instance of the originating serializer of this snapshot. + */ + @SuppressWarnings("unchecked") + public CompositeTypeSerializerSnapshot(S serializerInstance) { + Preconditions.checkNotNull(serializerInstance); + this.compositeSerializerSnapshot = new CompositeSerializerSnapshot(getNestedSerializers(serializerInstance)); + this.correspondingSerializerClass = (Class) serializerInstance.getClass(); + } + + @Override + public final int getCurrentVersion() { + return VERSION; + } + + @Override + public final void writeSnapshot(DataOutputView out) throws IOException { + internalWriteOuterSnapshot(out); + compositeSerializerSnapshot.writeCompositeSnapshot(out); + } + + @Override + public final void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { + if (readVersion > HIGHEST_LEGACY_READ_VERSION) { + internalReadOuterSnapshot(in, userCodeClassLoader); + } else { + legacyInternalReadOuterSnapshot(readVersion, in, userCodeClassLoader); + } + this.compositeSerializerSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, userCodeClassLoader); + } + + @Override + public final TypeSerializerSchemaCompatibility resolveSchemaCompatibility(TypeSerializer newSerializer) { + if (newSerializer.getClass() != correspondingSerializerClass) { + return TypeSerializerSchemaCompatibility.incompatible(); + } + + S castedNewSerializer = correspondingSerializerClass.cast(newSerializer); + + // check that outer configuration is compatible; if not, short circuit result + if (!isOuterSnapshotCompatible(castedNewSerializer)) { + return TypeSerializerSchemaCompatibility.incompatible(); + } + + // since outer configuration is compatible, the final compatibility result depends only on the nested serializers + return constructFinalSchemaCompatibilityResult( + getNestedSerializers(castedNewSerializer), + compositeSerializerSnapshot.getNestedSerializerSnapshots()); + } + + @Override + public final TypeSerializer restoreSerializer() { + @SuppressWarnings("unchecked") + TypeSerializer serializer = (TypeSerializer) + createOuterSerializerWithNestedSerializers(compositeSerializerSnapshot.getRestoreSerializers()); + + return serializer; + } + + // ------------------------------------------------------------------------------------------ + // Outer serializer access methods + // ------------------------------------------------------------------------------------------ + + /** + * Returns the version of the current outer snapshot's written binary format. + * + * @return the version of the current outer snapshot's written binary format. + */ + protected abstract int getCurrentOuterSnapshotVersion(); + + /** + * Gets the nested serializers from the outer serializer. + * + * @param outerSerializer the outer serializer. + * + * @return the nested serializers. + */ + protected abstract TypeSerializer[] getNestedSerializers(S outerSerializer); + + /** + * Creates an instance of the outer serializer with a given array of its nested serializers. + * + * @param nestedSerializers array of nested serializers to create the outer serializer with. + * + * @return an instance of the outer serializer. + */ + protected abstract S createOuterSerializerWithNestedSerializers(TypeSerializer[] nestedSerializers); + + // ------------------------------------------------------------------------------------------ + // Outer snapshot methods; need to be overridden if outer snapshot is not empty, + // or in other words, the outer serializer has extra configuration beyond its nested serializers. + // ------------------------------------------------------------------------------------------ + + /** + * Writes the outer snapshot, i.e. any information beyond the nested serializers of the outer serializer. + * + *

The base implementation of this methods writes nothing, i.e. it assumes that the outer serializer + * only has nested serializers and no extra information. Otherwise, if the outer serializer contains + * some extra information that needs to be persisted as part of the serializer snapshot, this + * must be overridden. Note that this method and the corresponding methods + * {@link #readOuterSnapshot(int, DataInputView, ClassLoader)}, {@link #isOuterSnapshotCompatible(TypeSerializer)} + * needs to be implemented. + * + * @param out the {@link DataOutputView} to write the outer snapshot to. + */ + protected void writeOuterSnapshot(DataOutputView out) throws IOException {} + + /** + * Reads the outer snapshot, i.e. any information beyond the nested serializers of the outer serializer. + * + *

The base implementation of this methods reads nothing, i.e. it assumes that the outer serializer + * only has nested serializers and no extra information. Otherwise, if the outer serializer contains + * some extra information that has been persisted as part of the serializer snapshot, this + * must be overridden. Note that this method and the corresponding methods + * {@link #writeOuterSnapshot(DataOutputView)}, {@link #isOuterSnapshotCompatible(TypeSerializer)} + * needs to be implemented. + * + * @param readOuterSnapshotVersion the read version of the outer snapshot. + * @param in the {@link DataInputView} to read the outer snapshot from. + * @param userCodeClassLoader the user code class loader. + */ + protected void readOuterSnapshot(int readOuterSnapshotVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {} + + /** + * Checks whether the outer snapshot is compatible with a given new serializer. + * + *

The base implementation of this method just returns {@code true}, i.e. it assumes that the outer serializer + * only has nested serializers and no extra information, and therefore the result of the check must always + * be true. Otherwise, if the outer serializer contains + * some extra information that has been persisted as part of the serializer snapshot, this + * must be overridden. Note that this method and the corresponding methods + * {@link #writeOuterSnapshot(DataOutputView)}, {@link #readOuterSnapshot(int, DataInputView, ClassLoader)} + * needs to be implemented. + * + * @param newSerializer the new serializer, which contains the new outer information to check against. + * + * @return a flag indicating whether or not the new serializer's outer information is compatible with the one + * written in this snapshot. + */ + protected boolean isOuterSnapshotCompatible(S newSerializer) { + return true; + } + + // ------------------------------------------------------------------------------------------ + // Utilities + // ------------------------------------------------------------------------------------------ + + private void internalWriteOuterSnapshot(DataOutputView out) throws IOException { + out.writeInt(MAGIC_NUMBER); + out.writeInt(getCurrentOuterSnapshotVersion()); + + writeOuterSnapshot(out); + } + + private void internalReadOuterSnapshot(DataInputView in, ClassLoader userCodeClassLoader) throws IOException { + final int magicNumber = in.readInt(); + if (magicNumber != MAGIC_NUMBER) { + throw new IOException(String.format("Corrupt data, magic number mismatch. Expected %8x, found %8x", + MAGIC_NUMBER, magicNumber)); + } + + final int outerSnapshotVersion = in.readInt(); + readOuterSnapshot(outerSnapshotVersion, in, userCodeClassLoader); + } + + private void legacyInternalReadOuterSnapshot( + int legacyReadVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { + + // legacy versions did not contain the pre-fixed magic numbers; just read the outer snapshot + readOuterSnapshot(legacyReadVersion, in, userCodeClassLoader); + } + + private TypeSerializerSchemaCompatibility constructFinalSchemaCompatibilityResult( + TypeSerializer[] newNestedSerializers, + TypeSerializerSnapshot[] nestedSerializerSnapshots) { + + Preconditions.checkArgument(newNestedSerializers.length == nestedSerializerSnapshots.length, + "Different number of new serializers and existing serializer snapshots."); + + TypeSerializer[] reconfiguredNestedSerializers = new TypeSerializer[newNestedSerializers.length]; + + // check nested serializers for compatibility + boolean nestedSerializerRequiresMigration = false; + boolean hasReconfiguredNestedSerializers = false; + for (int i = 0; i < nestedSerializerSnapshots.length; i++) { + TypeSerializerSchemaCompatibility compatibility = + resolveCompatibility(newNestedSerializers[i], nestedSerializerSnapshots[i]); + + // if any one of the new nested serializers is incompatible, we can just short circuit the result + if (compatibility.isIncompatible()) { + return TypeSerializerSchemaCompatibility.incompatible(); + } + + if (compatibility.isCompatibleAfterMigration()) { + nestedSerializerRequiresMigration = true; + } else if (compatibility.isCompatibleWithReconfiguredSerializer()) { + hasReconfiguredNestedSerializers = true; + reconfiguredNestedSerializers[i] = compatibility.getReconfiguredSerializer(); + } else if (compatibility.isCompatibleAsIs()) { + reconfiguredNestedSerializers[i] = newNestedSerializers[i]; + } else { + throw new IllegalStateException("Undefined compatibility type."); + } + } + + if (nestedSerializerRequiresMigration) { + return TypeSerializerSchemaCompatibility.compatibleAfterMigration(); + } + + if (hasReconfiguredNestedSerializers) { + @SuppressWarnings("unchecked") + TypeSerializer reconfiguredCompositeSerializer = createOuterSerializerWithNestedSerializers(reconfiguredNestedSerializers); + return TypeSerializerSchemaCompatibility.compatibleWithReconfiguredSerializer(reconfiguredCompositeSerializer); + } + + // ends up here if everything is compatible as is + return TypeSerializerSchemaCompatibility.compatibleAsIs(); + } + + @SuppressWarnings("unchecked") + private static TypeSerializerSchemaCompatibility resolveCompatibility( + TypeSerializer serializer, + TypeSerializerSnapshot snapshot) { + + TypeSerializer typedSerializer = (TypeSerializer) serializer; + TypeSerializerSnapshot typedSnapshot = (TypeSerializerSnapshot) snapshot; + + return typedSnapshot.resolveSchemaCompatibility(typedSerializer); + } +} diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshotTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshotTest.java new file mode 100644 index 00000000000000..0f77e3d7376325 --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshotTest.java @@ -0,0 +1,544 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.typeutils; + +import org.apache.flink.api.common.typeutils.base.StringSerializer; +import org.apache.flink.core.memory.DataInputDeserializer; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputSerializer; +import org.apache.flink.core.memory.DataOutputView; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; + +/** + * Test suite for the {@link CompositeTypeSerializerSnapshot}. + */ +public class CompositeTypeSerializerSnapshotTest { + + // ------------------------------------------------------------------------------------------------ + // Scope: tests CompositeTypeSerializerSnapshot#resolveSchemaCompatibility + // ------------------------------------------------------------------------------------------------ + + @Test + public void testIncompatiblePrecedence() throws IOException { + final String OUTER_CONFIG = "outer-config"; + final TypeSerializer[] testNestedSerializers = { + new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), + new NestedSerializer(TargetCompatibility.COMPATIBLE_AFTER_MIGRATION), + new NestedSerializer(TargetCompatibility.INCOMPATIBLE), + new NestedSerializer(TargetCompatibility.COMPATIBLE_WITH_RECONFIGURED_SERIALIZER) + }; + + TypeSerializerSchemaCompatibility compatibility = + snapshotCompositeSerializerAndGetSchemaCompatibilityAfterRestore( + testNestedSerializers, + OUTER_CONFIG, + OUTER_CONFIG); + + Assert.assertTrue(compatibility.isIncompatible()); + } + + @Test + public void testCompatibleAfterMigrationPrecedence() throws IOException { + final String OUTER_CONFIG = "outer-config"; + TypeSerializer[] testNestedSerializers = { + new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), + new NestedSerializer(TargetCompatibility.COMPATIBLE_AFTER_MIGRATION), + new NestedSerializer(TargetCompatibility.COMPATIBLE_WITH_RECONFIGURED_SERIALIZER), + new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), + }; + + TypeSerializerSchemaCompatibility compatibility = + snapshotCompositeSerializerAndGetSchemaCompatibilityAfterRestore( + testNestedSerializers, + OUTER_CONFIG, + OUTER_CONFIG); + + Assert.assertTrue(compatibility.isCompatibleAfterMigration()); + } + + @Test + public void testCompatibleWithReconfiguredSerializerPrecedence() throws IOException { + final String OUTER_CONFIG = "outer-config"; + TypeSerializer[] testNestedSerializers = { + new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), + new NestedSerializer(TargetCompatibility.COMPATIBLE_WITH_RECONFIGURED_SERIALIZER), + new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), + }; + + TypeSerializerSchemaCompatibility compatibility = + snapshotCompositeSerializerAndGetSchemaCompatibilityAfterRestore( + testNestedSerializers, + OUTER_CONFIG, + OUTER_CONFIG); + + Assert.assertTrue(compatibility.isCompatibleWithReconfiguredSerializer()); + + TestCompositeTypeSerializer reconfiguredSerializer = + (TestCompositeTypeSerializer) compatibility.getReconfiguredSerializer(); + TypeSerializer[] reconfiguredNestedSerializers = reconfiguredSerializer.getNestedSerializers(); + // nested serializer at index 1 should strictly be a ReconfiguredNestedSerializer + Assert.assertTrue(reconfiguredNestedSerializers[0].getClass() == NestedSerializer.class); + Assert.assertTrue(reconfiguredNestedSerializers[1].getClass() == ReconfiguredNestedSerializer.class); + Assert.assertTrue(reconfiguredNestedSerializers[2].getClass() == NestedSerializer.class); + } + + @Test + public void testCompatibleAsIsPrecedence() throws IOException { + final String OUTER_CONFIG = "outer-config"; + TypeSerializer[] testNestedSerializers = { + new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), + new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), + }; + + TypeSerializerSchemaCompatibility compatibility = + snapshotCompositeSerializerAndGetSchemaCompatibilityAfterRestore( + testNestedSerializers, + OUTER_CONFIG, + OUTER_CONFIG); + + Assert.assertTrue(compatibility.isCompatibleAsIs()); + } + + @Test + public void testOuterSnapshotCompatibilityPrecedence() throws IOException { + final String INIT_OUTER_CONFIG = "outer-config"; + final String INCOMPAT_OUTER_CONFIG = "incompat-outer-config"; + TypeSerializer[] testNestedSerializers = { + new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), + }; + + TypeSerializerSchemaCompatibility compatibility = + snapshotCompositeSerializerAndGetSchemaCompatibilityAfterRestore( + testNestedSerializers, + INIT_OUTER_CONFIG, + INCOMPAT_OUTER_CONFIG); + + // even though nested serializers are compatible, incompatibility of the outer + // snapshot should have higher precedence in the final result + Assert.assertTrue(compatibility.isIncompatible()); + } + + private TypeSerializerSchemaCompatibility snapshotCompositeSerializerAndGetSchemaCompatibilityAfterRestore( + TypeSerializer[] testNestedSerializers, + String initialOuterConfiguration, + String newOuterConfiguration) throws IOException { + TestCompositeTypeSerializer testSerializer = + new TestCompositeTypeSerializer(initialOuterConfiguration, testNestedSerializers); + + TypeSerializerSnapshot testSerializerSnapshot = testSerializer.snapshotConfiguration(); + + DataOutputSerializer out = new DataOutputSerializer(128); + TypeSerializerSnapshot.writeVersionedSnapshot(out, testSerializerSnapshot); + + DataInputDeserializer in = new DataInputDeserializer(out.getCopyOfBuffer()); + testSerializerSnapshot = TypeSerializerSnapshot.readVersionedSnapshot( + in, Thread.currentThread().getContextClassLoader()); + + TestCompositeTypeSerializer newTestSerializer = + new TestCompositeTypeSerializer(newOuterConfiguration, testNestedSerializers); + return testSerializerSnapshot.resolveSchemaCompatibility(newTestSerializer); + } + + // ------------------------------------------------------------------------------------------------ + // Scope: tests CompositeTypeSerializerSnapshot#restoreSerializer + // ------------------------------------------------------------------------------------------------ + + @Test + public void testRestoreCompositeTypeSerializer() throws IOException { + // the target compatibilities of the nested serializers doesn't matter, + // because we're only testing the restore serializer + TypeSerializer[] testNestedSerializers = { + new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), + new NestedSerializer(TargetCompatibility.INCOMPATIBLE), + new NestedSerializer(TargetCompatibility.COMPATIBLE_AFTER_MIGRATION) + }; + + TestCompositeTypeSerializer testSerializer = new TestCompositeTypeSerializer("outer-config", testNestedSerializers); + + TypeSerializerSnapshot testSerializerSnapshot = testSerializer.snapshotConfiguration(); + + DataOutputSerializer out = new DataOutputSerializer(128); + TypeSerializerSnapshot.writeVersionedSnapshot(out, testSerializerSnapshot); + + DataInputDeserializer in = new DataInputDeserializer(out.getCopyOfBuffer()); + testSerializerSnapshot = TypeSerializerSnapshot.readVersionedSnapshot( + in, Thread.currentThread().getContextClassLoader()); + + // now, restore the composite type serializer; + // the restored nested serializer should be a RestoredNestedSerializer + testSerializer = (TestCompositeTypeSerializer) testSerializerSnapshot.restoreSerializer(); + Assert.assertTrue(testSerializer.getNestedSerializers()[0].getClass() == RestoredNestedSerializer.class); + Assert.assertTrue(testSerializer.getNestedSerializers()[1].getClass() == RestoredNestedSerializer.class); + Assert.assertTrue(testSerializer.getNestedSerializers()[2].getClass() == RestoredNestedSerializer.class); + } + + // ------------------------------------------------------------------------------------------------ + // Test utilities + // ------------------------------------------------------------------------------------------------ + + /** + * A simple composite serializer used for testing. + * It can be configured with an array of nested serializers, as well as outer configuration (represented as String). + */ + public static class TestCompositeTypeSerializer extends TypeSerializer { + + private static final long serialVersionUID = -545688468997398105L; + + private static final StringSerializer delegateSerializer = StringSerializer.INSTANCE; + + private final String outerConfiguration; + + private final TypeSerializer[] nestedSerializers; + + TestCompositeTypeSerializer( + String outerConfiguration, + TypeSerializer[] nestedSerializers) { + this.outerConfiguration = outerConfiguration; + this.nestedSerializers = nestedSerializers; + } + + public String getOuterConfiguration() { + return outerConfiguration; + } + + TypeSerializer[] getNestedSerializers() { + return nestedSerializers; + } + + @Override + public TypeSerializerSnapshot snapshotConfiguration() { + return new TestCompositeTypeSerializerSnapshot(this); + } + + // -------------------------------------------------------------------------------- + // Serialization delegation + // -------------------------------------------------------------------------------- + + @Override + public String deserialize(String reuse, DataInputView source) throws IOException { + return delegateSerializer.deserialize(reuse, source); + } + + @Override + public String deserialize(DataInputView source) throws IOException { + return delegateSerializer.deserialize(source); + } + + @Override + public void serialize(String record, DataOutputView target) throws IOException { + delegateSerializer.serialize(record, target); + } + + @Override + public void copy(DataInputView source, DataOutputView target) throws IOException { + delegateSerializer.copy(source, target); + } + + @Override + public String copy(String from) { + return delegateSerializer.copy(from); + } + + @Override + public String copy(String from, String reuse) { + return delegateSerializer.copy(from, reuse); + } + + @Override + public String createInstance() { + return delegateSerializer.createInstance(); + } + + @Override + public TypeSerializer duplicate() { + return this; + } + + @Override + public boolean isImmutableType() { + return false; + } + + @Override + public int getLength() { + return 0; + } + + @Override + public boolean equals(Object obj) { + if (canEqual(obj)) { + return Arrays.equals(nestedSerializers, ((TestCompositeTypeSerializer) obj).getNestedSerializers()); + } + return false; + } + + @Override + public int hashCode() { + return Arrays.hashCode(nestedSerializers); + } + + @Override + public boolean canEqual(Object obj) { + return obj instanceof TestCompositeTypeSerializer; + } + } + + /** + * Snapshot class for the {@link TestCompositeTypeSerializer}. + */ + public static class TestCompositeTypeSerializerSnapshot extends CompositeTypeSerializerSnapshot { + + private String outerConfiguration; + + public TestCompositeTypeSerializerSnapshot() { + super(TestCompositeTypeSerializer.class); + } + + TestCompositeTypeSerializerSnapshot(TestCompositeTypeSerializer serializer) { + super(serializer); + this.outerConfiguration = serializer.getOuterConfiguration(); + } + + @Override + protected TestCompositeTypeSerializer createOuterSerializerWithNestedSerializers(TypeSerializer[] nestedSerializers) { + return new TestCompositeTypeSerializer(outerConfiguration, nestedSerializers); + } + + @Override + protected TypeSerializer[] getNestedSerializers(TestCompositeTypeSerializer outerSerializer) { + return outerSerializer.getNestedSerializers(); + } + + @Override + protected void writeOuterSnapshot(DataOutputView out) throws IOException { + out.writeUTF(outerConfiguration); + } + + @Override + public void readOuterSnapshot(int readOuterSnapshotVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { + Assert.assertEquals(getCurrentOuterSnapshotVersion(), readOuterSnapshotVersion); + this.outerConfiguration = in.readUTF(); + } + + @Override + protected boolean isOuterSnapshotCompatible(TestCompositeTypeSerializer newSerializer) { + return outerConfiguration.equals(newSerializer.getOuterConfiguration()); + } + + @Override + public int getCurrentOuterSnapshotVersion() { + return 1; + } + } + + public enum TargetCompatibility { + COMPATIBLE_AS_IS, + COMPATIBLE_AFTER_MIGRATION, + COMPATIBLE_WITH_RECONFIGURED_SERIALIZER, + INCOMPATIBLE + } + + /** + * Used as nested serializers in the test composite serializer. + * A nested serializer can be configured with a {@link TargetCompatibility}, + * which indicates what the result of the schema compatibility check should be + * when a new instance of it is being checked for compatibility. + */ + public static class NestedSerializer extends TypeSerializer { + + private static final long serialVersionUID = -6175000932620623446L; + + private static final StringSerializer delegateSerializer = StringSerializer.INSTANCE; + + private final TargetCompatibility targetCompatibility; + + NestedSerializer(TargetCompatibility targetCompatibility) { + this.targetCompatibility = targetCompatibility; + } + + @Override + public TypeSerializerSnapshot snapshotConfiguration() { + return new NestedSerializerSnapshot(targetCompatibility); + } + + // -------------------------------------------------------------------------------- + // Serialization delegation + // -------------------------------------------------------------------------------- + + @Override + public String deserialize(String reuse, DataInputView source) throws IOException { + return delegateSerializer.deserialize(reuse, source); + } + + @Override + public String deserialize(DataInputView source) throws IOException { + return delegateSerializer.deserialize(source); + } + + @Override + public void serialize(String record, DataOutputView target) throws IOException { + delegateSerializer.serialize(record, target); + } + + @Override + public void copy(DataInputView source, DataOutputView target) throws IOException { + delegateSerializer.copy(source, target); + } + + @Override + public String copy(String from) { + return delegateSerializer.copy(from); + } + + @Override + public String copy(String from, String reuse) { + return delegateSerializer.copy(from, reuse); + } + + @Override + public String createInstance() { + return delegateSerializer.createInstance(); + } + + @Override + public TypeSerializer duplicate() { + return this; + } + + @Override + public boolean isImmutableType() { + return false; + } + + @Override + public int getLength() { + return 0; + } + + @Override + public boolean equals(Object obj) { + if (canEqual(obj)) { + return targetCompatibility == ((NestedSerializer) obj).targetCompatibility; + } + return false; + } + + @Override + public int hashCode() { + return targetCompatibility.hashCode(); + } + + @Override + public boolean canEqual(Object obj) { + return obj instanceof NestedSerializer; + } + } + + /** + * Snapshot of the {@link NestedSerializer}. + */ + public static class NestedSerializerSnapshot implements TypeSerializerSnapshot { + + private TargetCompatibility targetCompatibility; + + public NestedSerializerSnapshot() {} + + public NestedSerializerSnapshot(TargetCompatibility targetCompatibility) { + this.targetCompatibility = targetCompatibility; + } + + @Override + public void writeSnapshot(DataOutputView out) throws IOException { + out.writeInt(targetCompatibility.ordinal()); + } + + @Override + public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { + this.targetCompatibility = TargetCompatibility.values()[in.readInt()]; + } + + @Override + public TypeSerializerSchemaCompatibility resolveSchemaCompatibility(TypeSerializer newSerializer) { + // checks the exact class instead of using instanceof; + // this ensures that we get a new serializer, and not a ReconfiguredNestedSerializer or RestoredNestedSerializer + if (newSerializer.getClass() == NestedSerializer.class) { + switch (targetCompatibility) { + case COMPATIBLE_AS_IS: + return TypeSerializerSchemaCompatibility.compatibleAsIs(); + case COMPATIBLE_AFTER_MIGRATION: + return TypeSerializerSchemaCompatibility.compatibleAfterMigration(); + case COMPATIBLE_WITH_RECONFIGURED_SERIALIZER: + return TypeSerializerSchemaCompatibility.compatibleWithReconfiguredSerializer( + new ReconfiguredNestedSerializer(targetCompatibility)); + case INCOMPATIBLE: + return TypeSerializerSchemaCompatibility.incompatible(); + default: + throw new IllegalStateException("Unexpected target compatibility."); + } + } + + throw new IllegalArgumentException("Expected the new serializer to be of class " + NestedSerializer.class); + } + + @Override + public TypeSerializer restoreSerializer() { + return new RestoredNestedSerializer(targetCompatibility); + } + + @Override + public int getCurrentVersion() { + return 1; + } + } + + /** + * A variant of the {@link NestedSerializer} used only when creating a reconfigured instance + * of the serializer. This is used in tests as a tag to identify that the correct serializer + * instances are being used. + */ + static class ReconfiguredNestedSerializer extends NestedSerializer { + + private static final long serialVersionUID = -1396401178636869659L; + + public ReconfiguredNestedSerializer(TargetCompatibility targetCompatibility) { + super(targetCompatibility); + } + + } + + /** + * A variant of the {@link NestedSerializer} used only when creating a restored instance + * of the serializer. This is used in tests as a tag to identify that the correct serializer + * instances are being used. + */ + static class RestoredNestedSerializer extends NestedSerializer { + + private static final long serialVersionUID = -1396401178636869659L; + + public RestoredNestedSerializer(TargetCompatibility targetCompatibility) { + super(targetCompatibility); + } + + } +} From 817485473005dfd054467eab4c2a275d833af723 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Wed, 5 Dec 2018 15:55:23 +0800 Subject: [PATCH 056/147] [FLINK-11079] [core] Let ListSerializerSnapshot extend CompositeTypeSerializerSnapshot --- .../CollectionSerializerConfigSnapshot.java | 9 ++- .../common/typeutils/base/ListSerializer.java | 2 +- .../base/ListSerializerSnapshot.java | 56 +++++-------------- 3 files changed, 21 insertions(+), 46 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java index 377dd4cc1985ac..762a4410c85d0d 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java @@ -49,10 +49,13 @@ public CollectionSerializerConfigSnapshot(TypeSerializer elementSerializer) { @Override public TypeSerializerSchemaCompatibility resolveSchemaCompatibility(TypeSerializer newSerializer) { if (newSerializer instanceof ListSerializer) { - ListSerializerSnapshot listSerializerSnapshot = - new ListSerializerSnapshot<>(((ListSerializer) newSerializer).getElementSerializer()); + ListSerializer newListSerializer = (ListSerializer) newSerializer; + ListSerializerSnapshot listSerializerSnapshot = new ListSerializerSnapshot<>(newListSerializer); - return listSerializerSnapshot.resolveSchemaCompatibility((ListSerializer) newSerializer); + @SuppressWarnings("unchecked") + TypeSerializerSchemaCompatibility result = (TypeSerializerSchemaCompatibility) + listSerializerSnapshot.resolveSchemaCompatibility(newListSerializer); + return result; } else { return super.resolveSchemaCompatibility(newSerializer); } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java index 08b3333465302a..51780319036734 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java @@ -175,6 +175,6 @@ public int hashCode() { @Override public TypeSerializerSnapshot> snapshotConfiguration() { - return new ListSerializerSnapshot<>(elementSerializer); + return new ListSerializerSnapshot<>(this); } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerSnapshot.java index 5f89d94973f815..f90e22a651a711 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerSnapshot.java @@ -18,73 +18,45 @@ package org.apache.flink.api.common.typeutils.base; -import org.apache.flink.api.common.typeutils.CompositeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; -import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataOutputView; -import org.apache.flink.util.Preconditions; - -import java.io.IOException; import java.util.List; -import static org.apache.flink.util.Preconditions.checkState; - /** * Snapshot class for the {@link ListSerializer}. */ -public class ListSerializerSnapshot implements TypeSerializerSnapshot> { +public class ListSerializerSnapshot extends CompositeTypeSerializerSnapshot, ListSerializer> { private static final int CURRENT_VERSION = 1; - private CompositeSerializerSnapshot nestedElementSerializerSnapshot; - /** * Constructor for read instantiation. */ - public ListSerializerSnapshot() {} + public ListSerializerSnapshot() { + super(ListSerializer.class); + } /** * Constructor to create the snapshot for writing. */ - public ListSerializerSnapshot(TypeSerializer elementSerializer) { - this.nestedElementSerializerSnapshot = new CompositeSerializerSnapshot(Preconditions.checkNotNull(elementSerializer)); + public ListSerializerSnapshot(ListSerializer listSerializer) { + super(listSerializer); } @Override - public int getCurrentVersion() { + public int getCurrentOuterSnapshotVersion() { return CURRENT_VERSION; } @Override - public TypeSerializer> restoreSerializer() { - return new ListSerializer<>(nestedElementSerializerSnapshot.getRestoreSerializer(0)); - } - - @Override - public TypeSerializerSchemaCompatibility> resolveSchemaCompatibility(TypeSerializer> newSerializer) { - checkState(nestedElementSerializerSnapshot != null); - - if (newSerializer instanceof ListSerializer) { - ListSerializer serializer = (ListSerializer) newSerializer; - - return nestedElementSerializerSnapshot.resolveCompatibilityWithNested( - TypeSerializerSchemaCompatibility.compatibleAsIs(), - serializer.getElementSerializer()); - } - else { - return TypeSerializerSchemaCompatibility.incompatible(); - } - } - - @Override - public void writeSnapshot(DataOutputView out) throws IOException { - nestedElementSerializerSnapshot.writeCompositeSnapshot(out); + protected ListSerializer createOuterSerializerWithNestedSerializers(TypeSerializer[] nestedSerializers) { + @SuppressWarnings("unchecked") + TypeSerializer elementSerializer = (TypeSerializer) nestedSerializers[0]; + return new ListSerializer<>(elementSerializer); } @Override - public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { - this.nestedElementSerializerSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, userCodeClassLoader); + protected TypeSerializer[] getNestedSerializers(ListSerializer outerSerializer) { + return new TypeSerializer[] { outerSerializer.getElementSerializer() }; } } From 9822cc8a73cc4a54cb401d3ee31ce2e8894a6d6d Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Wed, 5 Dec 2018 16:11:11 +0800 Subject: [PATCH 057/147] [FLINK-11073] [core] Let MapViewSerializerSnapshot be a CompositeTypeSerializerSnapshot --- .../dataview/MapViewSerializerSnapshot.java | 55 +++++-------------- .../table/dataview/MapViewSerializer.scala | 2 +- 2 files changed, 15 insertions(+), 42 deletions(-) diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerSnapshot.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerSnapshot.java index f59fc0a3654236..132f42f33205e9 100644 --- a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerSnapshot.java +++ b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerSnapshot.java @@ -18,78 +18,51 @@ package org.apache.flink.table.dataview; -import org.apache.flink.api.common.typeutils.CompositeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.table.api.dataview.MapView; -import org.apache.flink.util.Preconditions; -import java.io.IOException; import java.util.Map; -import static org.apache.flink.util.Preconditions.checkState; - /** * A {@link TypeSerializerSnapshot} for the {@link MapViewSerializer}. * * @param the key type of the map entries. * @param the value type of the map entries. */ -public class MapViewSerializerSnapshot implements TypeSerializerSnapshot> { +public class MapViewSerializerSnapshot extends CompositeTypeSerializerSnapshot, MapViewSerializer> { private static final int CURRENT_VERSION = 1; - private CompositeSerializerSnapshot nestedMapSerializerSnapshot; - /** * Constructor for read instantiation. */ - public MapViewSerializerSnapshot() {} + public MapViewSerializerSnapshot() { + super(MapViewSerializer.class); + } /** * Constructor to create the snapshot for writing. */ - public MapViewSerializerSnapshot(TypeSerializer> mapSerializer) { - this.nestedMapSerializerSnapshot = new CompositeSerializerSnapshot(Preconditions.checkNotNull(mapSerializer)); + public MapViewSerializerSnapshot(MapViewSerializer mapViewSerializer) { + super(mapViewSerializer); } @Override - public int getCurrentVersion() { + public int getCurrentOuterSnapshotVersion() { return CURRENT_VERSION; } @Override - public TypeSerializer> restoreSerializer() { - return new MapViewSerializer<>(nestedMapSerializerSnapshot.getRestoreSerializer(0)); - } - - @Override - public TypeSerializerSchemaCompatibility> resolveSchemaCompatibility( - TypeSerializer> newSerializer) { - checkState(nestedMapSerializerSnapshot != null); - - if (newSerializer instanceof MapViewSerializer) { - MapViewSerializer serializer = (MapViewSerializer) newSerializer; - - return nestedMapSerializerSnapshot.resolveCompatibilityWithNested( - TypeSerializerSchemaCompatibility.compatibleAsIs(), - serializer.getMapSerializer()); - } - else { - return TypeSerializerSchemaCompatibility.incompatible(); - } - } - - @Override - public void writeSnapshot(DataOutputView out) throws IOException { - nestedMapSerializerSnapshot.writeCompositeSnapshot(out); + protected MapViewSerializer createOuterSerializerWithNestedSerializers(TypeSerializer[] nestedSerializers) { + @SuppressWarnings("unchecked") + TypeSerializer> mapSerializer = (TypeSerializer>) nestedSerializers[0]; + return new MapViewSerializer<>(mapSerializer); } @Override - public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { - this.nestedMapSerializerSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, userCodeClassLoader); + protected TypeSerializer[] getNestedSerializers(MapViewSerializer outerSerializer) { + return new TypeSerializer[] { outerSerializer.getMapSerializer() }; } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala index 89cdf70174983d..e0067c57b10aea 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala @@ -78,7 +78,7 @@ class MapViewSerializer[K, V](val mapSerializer: TypeSerializer[java.util.Map[K, mapSerializer.equals(obj.asInstanceOf[MapViewSerializer[_, _]].mapSerializer) override def snapshotConfiguration(): MapViewSerializerSnapshot[K, V] = - new MapViewSerializerSnapshot[K, V](mapSerializer) + new MapViewSerializerSnapshot[K, V](this) // copy and modified from MapSerializer.ensureCompatibility override def ensureCompatibility(configSnapshot: TypeSerializerConfigSnapshot[_]) From 383cc9e1875c2b19bb362b3af025547cc2b1dfbf Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Wed, 5 Dec 2018 16:14:20 +0800 Subject: [PATCH 058/147] [FLINK-11073] [core] Let ArrayListSerializerSnapshot be a CompositeTypeSerializerSnapshot --- .../runtime/state/ArrayListSerializer.java | 2 +- .../state/ArrayListSerializerSnapshot.java | 54 +++++-------------- 2 files changed, 15 insertions(+), 41 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java index 6fa9f02d41946e..d442d0d3ec73cb 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java @@ -147,7 +147,7 @@ public int hashCode() { @Override public TypeSerializerSnapshot> snapshotConfiguration() { - return new ArrayListSerializerSnapshot<>(elementSerializer); + return new ArrayListSerializerSnapshot<>(this); } /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializerSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializerSnapshot.java index 7fc8c51c8cf6f9..dde8d1a1e60ea9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializerSnapshot.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializerSnapshot.java @@ -18,72 +18,46 @@ package org.apache.flink.runtime.state; -import org.apache.flink.api.common.typeutils.CompositeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; -import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataOutputView; -import java.io.IOException; import java.util.ArrayList; -import static org.apache.flink.util.Preconditions.checkState; - /** * Snapshot class for the {@link ArrayListSerializer}. */ -public class ArrayListSerializerSnapshot implements TypeSerializerSnapshot> { +public class ArrayListSerializerSnapshot extends CompositeTypeSerializerSnapshot, ArrayListSerializer> { private static final int CURRENT_VERSION = 1; - private CompositeSerializerSnapshot nestedElementSerializerSnapshot; - /** * Constructor for read instantiation. */ - public ArrayListSerializerSnapshot() {} + public ArrayListSerializerSnapshot() { + super(ArrayListSerializer.class); + } /** * Constructor for creating the snapshot for writing. */ - public ArrayListSerializerSnapshot(TypeSerializer elementSerializer) { - this.nestedElementSerializerSnapshot = new CompositeSerializerSnapshot(elementSerializer); + public ArrayListSerializerSnapshot(ArrayListSerializer arrayListSerializer) { + super(arrayListSerializer); } @Override - public int getCurrentVersion() { + public int getCurrentOuterSnapshotVersion() { return CURRENT_VERSION; } @Override - public TypeSerializer> restoreSerializer() { - return new ArrayListSerializer<>(nestedElementSerializerSnapshot.getRestoreSerializer(0)); - } - - @Override - public TypeSerializerSchemaCompatibility> resolveSchemaCompatibility(TypeSerializer> newSerializer) { - checkState(nestedElementSerializerSnapshot != null); - - if (newSerializer instanceof ArrayListSerializer) { - ArrayListSerializer serializer = (ArrayListSerializer) newSerializer; - - return nestedElementSerializerSnapshot.resolveCompatibilityWithNested( - TypeSerializerSchemaCompatibility.compatibleAsIs(), - serializer.getElementSerializer()); - } - else { - return TypeSerializerSchemaCompatibility.incompatible(); - } - } - - @Override - public void writeSnapshot(DataOutputView out) throws IOException { - nestedElementSerializerSnapshot.writeCompositeSnapshot(out); + protected ArrayListSerializer createOuterSerializerWithNestedSerializers(TypeSerializer[] nestedSerializers) { + @SuppressWarnings("unchecked") + TypeSerializer elementSerializer = (TypeSerializer) nestedSerializers[0]; + return new ArrayListSerializer<>(elementSerializer); } @Override - public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { - this.nestedElementSerializerSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, userCodeClassLoader); + protected TypeSerializer[] getNestedSerializers(ArrayListSerializer outerSerializer) { + return new TypeSerializer[] { outerSerializer.getElementSerializer() }; } } From eda73763dfad7c7367f03556134637cf8edc3160 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Wed, 5 Dec 2018 16:22:01 +0800 Subject: [PATCH 059/147] [FLINK-11073] [core] Let LockableTypeSerializerSnapshot be a CompositeTypeSerializerSnapshot --- .../flink/cep/nfa/sharedbuffer/Lockable.java | 2 +- .../LockableTypeSerializerSnapshot.java | 56 +++++-------------- 2 files changed, 15 insertions(+), 43 deletions(-) diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java index 7afbc508230c71..ae1452bf672936 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java @@ -187,7 +187,7 @@ public boolean canEqual(Object obj) { @Override public TypeSerializerSnapshot> snapshotConfiguration() { - return new LockableTypeSerializerSnapshot<>(elementSerializer); + return new LockableTypeSerializerSnapshot<>(this); } /** diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableTypeSerializerSnapshot.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableTypeSerializerSnapshot.java index 44a4670cc1da8d..13867aca0c3970 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableTypeSerializerSnapshot.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableTypeSerializerSnapshot.java @@ -19,74 +19,46 @@ package org.apache.flink.cep.nfa.sharedbuffer; import org.apache.flink.annotation.Internal; -import org.apache.flink.api.common.typeutils.CompositeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataOutputView; -import org.apache.flink.util.Preconditions; - -import java.io.IOException; - -import static org.apache.flink.util.Preconditions.checkState; /** * A {@link TypeSerializerSnapshot} for the {@link Lockable.LockableTypeSerializer}. */ @Internal -public class LockableTypeSerializerSnapshot implements TypeSerializerSnapshot> { +public class LockableTypeSerializerSnapshot extends CompositeTypeSerializerSnapshot, Lockable.LockableTypeSerializer> { private static final int CURRENT_VERSION = 1; - private CompositeSerializerSnapshot nestedElementSerializerSnapshot; - /** * Constructor for read instantiation. */ - public LockableTypeSerializerSnapshot() {} + public LockableTypeSerializerSnapshot() { + super(Lockable.LockableTypeSerializer.class); + } /** * Constructor to create the snapshot for writing. */ - public LockableTypeSerializerSnapshot(TypeSerializer elementSerializer) { - this.nestedElementSerializerSnapshot = new CompositeSerializerSnapshot(Preconditions.checkNotNull(elementSerializer)); + public LockableTypeSerializerSnapshot(Lockable.LockableTypeSerializer lockableTypeSerializer) { + super(lockableTypeSerializer); } @Override - public int getCurrentVersion() { + public int getCurrentOuterSnapshotVersion() { return CURRENT_VERSION; } @Override - public TypeSerializer> restoreSerializer() { - return new Lockable.LockableTypeSerializer<>(nestedElementSerializerSnapshot.getRestoreSerializer(0)); + protected Lockable.LockableTypeSerializer createOuterSerializerWithNestedSerializers(TypeSerializer[] nestedSerializers) { + @SuppressWarnings("unchecked") + TypeSerializer elementSerializer = (TypeSerializer) nestedSerializers[0]; + return new Lockable.LockableTypeSerializer<>(elementSerializer); } @Override - public TypeSerializerSchemaCompatibility> resolveSchemaCompatibility(TypeSerializer> newSerializer) { - checkState(nestedElementSerializerSnapshot != null); - - if (newSerializer instanceof Lockable.LockableTypeSerializer) { - Lockable.LockableTypeSerializer serializer = (Lockable.LockableTypeSerializer) newSerializer; - - return nestedElementSerializerSnapshot.resolveCompatibilityWithNested( - TypeSerializerSchemaCompatibility.compatibleAsIs(), - serializer.getElementSerializer()); - } - else { - return TypeSerializerSchemaCompatibility.incompatible(); - } + protected TypeSerializer[] getNestedSerializers(Lockable.LockableTypeSerializer outerSerializer) { + return new TypeSerializer[] { outerSerializer.getElementSerializer() }; } - - @Override - public void writeSnapshot(DataOutputView out) throws IOException { - nestedElementSerializerSnapshot.writeCompositeSnapshot(out); - } - - @Override - public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { - this.nestedElementSerializerSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, userCodeClassLoader); - } - } From faf093d33284b3a0e9e7b7ba7267b91503b69891 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Wed, 5 Dec 2018 16:26:13 +0800 Subject: [PATCH 060/147] [FLINK-11073] [core] Let ListViewSerializerSnapshot be a CompositeTypeSerializerSnapshot --- .../dataview/ListViewSerializerSnapshot.java | 54 +++++-------------- .../table/dataview/ListViewSerializer.scala | 2 +- 2 files changed, 15 insertions(+), 41 deletions(-) diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerSnapshot.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerSnapshot.java index cca84d28d17dbc..90468acf8c370d 100644 --- a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerSnapshot.java +++ b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerSnapshot.java @@ -18,76 +18,50 @@ package org.apache.flink.table.dataview; -import org.apache.flink.api.common.typeutils.CompositeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.table.api.dataview.ListView; -import org.apache.flink.util.Preconditions; -import java.io.IOException; import java.util.List; -import static org.apache.flink.util.Preconditions.checkState; - /** * A {@link TypeSerializerSnapshot} for the {@link ListViewSerializer}. * * @param the type of the list elements. */ -public final class ListViewSerializerSnapshot implements TypeSerializerSnapshot> { +public final class ListViewSerializerSnapshot extends CompositeTypeSerializerSnapshot, ListViewSerializer> { private static final int CURRENT_VERSION = 1; - private CompositeSerializerSnapshot nestedListSerializerSnapshot; - /** * Constructor for read instantiation. */ - public ListViewSerializerSnapshot() {} + public ListViewSerializerSnapshot() { + super(ListViewSerializer.class); + } /** * Constructor to create the snapshot for writing. */ - public ListViewSerializerSnapshot(TypeSerializer> listSerializer) { - this.nestedListSerializerSnapshot = new CompositeSerializerSnapshot(Preconditions.checkNotNull(listSerializer)); + public ListViewSerializerSnapshot(ListViewSerializer listViewSerializer) { + super(listViewSerializer); } @Override - public int getCurrentVersion() { + public int getCurrentOuterSnapshotVersion() { return CURRENT_VERSION; } @Override - public TypeSerializer> restoreSerializer() { - return new ListViewSerializer<>(nestedListSerializerSnapshot.getRestoreSerializer(0)); - } - - @Override - public TypeSerializerSchemaCompatibility> resolveSchemaCompatibility(TypeSerializer> newSerializer) { - checkState(nestedListSerializerSnapshot != null); - - if (newSerializer instanceof ListViewSerializer) { - ListViewSerializer serializer = (ListViewSerializer) newSerializer; - - return nestedListSerializerSnapshot.resolveCompatibilityWithNested( - TypeSerializerSchemaCompatibility.compatibleAsIs(), - serializer.getListSerializer()); - } - else { - return TypeSerializerSchemaCompatibility.incompatible(); - } - } - - @Override - public void writeSnapshot(DataOutputView out) throws IOException { - nestedListSerializerSnapshot.writeCompositeSnapshot(out); + protected ListViewSerializer createOuterSerializerWithNestedSerializers(TypeSerializer[] nestedSerializers) { + @SuppressWarnings("unchecked") + TypeSerializer> listSerializer = (TypeSerializer>) nestedSerializers[0]; + return new ListViewSerializer<>(listSerializer); } @Override - public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { - this.nestedListSerializerSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, userCodeClassLoader); + protected TypeSerializer[] getNestedSerializers(ListViewSerializer outerSerializer) { + return new TypeSerializer[] { outerSerializer.getListSerializer() }; } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala index 246af6c0dab334..2d48c3daf16d92 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala @@ -77,7 +77,7 @@ class ListViewSerializer[T](val listSerializer: TypeSerializer[java.util.List[T] listSerializer.equals(obj.asInstanceOf[ListViewSerializer[_]].listSerializer) override def snapshotConfiguration(): ListViewSerializerSnapshot[T] = - new ListViewSerializerSnapshot[T](listSerializer) + new ListViewSerializerSnapshot[T](this) override def ensureCompatibility( configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[ListView[T]] = { From 5e6664dfea9668c178047ebfc782278a176afaaf Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Wed, 5 Dec 2018 16:29:52 +0800 Subject: [PATCH 061/147] [FLINK-11073] [core] Let MapSerializerSnapshot be a CompositeTypeSerializerSnapshot --- .../common/typeutils/base/MapSerializer.java | 2 +- .../base/MapSerializerConfigSnapshot.java | 3 +- .../typeutils/base/MapSerializerSnapshot.java | 60 +++++-------------- 3 files changed, 18 insertions(+), 47 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java index dd3b81bade73ad..bedaf693608cfa 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java @@ -202,6 +202,6 @@ public int hashCode() { @Override public TypeSerializerSnapshot> snapshotConfiguration() { - return new MapSerializerSnapshot<>(keySerializer, valueSerializer); + return new MapSerializerSnapshot<>(this); } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java index 000924f0245465..2b78b527f7e0fa 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java @@ -50,8 +50,7 @@ public TypeSerializerSchemaCompatibility> resolveSchemaCompatibility(T // redirect the compatibility check to the new MapSerializerConfigSnapshot MapSerializer mapSerializer = (MapSerializer) newSerializer; - MapSerializerSnapshot mapSerializerSnapshot = - new MapSerializerSnapshot<>(mapSerializer.getKeySerializer(), mapSerializer.getValueSerializer()); + MapSerializerSnapshot mapSerializerSnapshot = new MapSerializerSnapshot<>(mapSerializer); return mapSerializerSnapshot.resolveSchemaCompatibility(newSerializer); } else { diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerSnapshot.java index be2e4b0cbe6a0c..a6db0ef74e6474 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerSnapshot.java @@ -18,78 +18,50 @@ package org.apache.flink.api.common.typeutils.base; -import org.apache.flink.api.common.typeutils.CompositeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; -import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataOutputView; -import org.apache.flink.util.Preconditions; -import java.io.IOException; import java.util.Map; -import static org.apache.flink.util.Preconditions.checkState; - /** * Snapshot class for the {@link MapSerializer}. */ -public class MapSerializerSnapshot implements TypeSerializerSnapshot> { +public class MapSerializerSnapshot extends CompositeTypeSerializerSnapshot, MapSerializer> { private static final int CURRENT_VERSION = 1; - private CompositeSerializerSnapshot nestedKeyValueSerializerSnapshot; - /** * Constructor for read instantiation. */ - public MapSerializerSnapshot() {} + public MapSerializerSnapshot() { + super(MapSerializer.class); + } /** * Constructor to create the snapshot for writing. */ - public MapSerializerSnapshot(TypeSerializer keySerializer, TypeSerializer valueSerializer) { - Preconditions.checkNotNull(keySerializer); - Preconditions.checkNotNull(valueSerializer); - this.nestedKeyValueSerializerSnapshot = new CompositeSerializerSnapshot(keySerializer, valueSerializer); + public MapSerializerSnapshot(MapSerializer mapSerializer) { + super(mapSerializer); } @Override - public int getCurrentVersion() { + public int getCurrentOuterSnapshotVersion() { return CURRENT_VERSION; } @Override - public TypeSerializer> restoreSerializer() { - return new MapSerializer<>( - nestedKeyValueSerializerSnapshot.getRestoreSerializer(0), - nestedKeyValueSerializerSnapshot.getRestoreSerializer(1)); - } + protected MapSerializer createOuterSerializerWithNestedSerializers(TypeSerializer[] nestedSerializers) { + @SuppressWarnings("unchecked") + TypeSerializer keySerializer = (TypeSerializer) nestedSerializers[0]; - @Override - public TypeSerializerSchemaCompatibility> resolveSchemaCompatibility(TypeSerializer> newSerializer) { - checkState(nestedKeyValueSerializerSnapshot != null); + @SuppressWarnings("unchecked") + TypeSerializer valueSerializer = (TypeSerializer) nestedSerializers[1]; - if (newSerializer instanceof MapSerializer) { - MapSerializer serializer = (MapSerializer) newSerializer; - - return nestedKeyValueSerializerSnapshot.resolveCompatibilityWithNested( - TypeSerializerSchemaCompatibility.compatibleAsIs(), - serializer.getKeySerializer(), - serializer.getValueSerializer()); - } - else { - return TypeSerializerSchemaCompatibility.incompatible(); - } - } - - @Override - public void writeSnapshot(DataOutputView out) throws IOException { - nestedKeyValueSerializerSnapshot.writeCompositeSnapshot(out); + return new MapSerializer<>(keySerializer, valueSerializer); } @Override - public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { - this.nestedKeyValueSerializerSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, userCodeClassLoader); + protected TypeSerializer[] getNestedSerializers(MapSerializer outerSerializer) { + return new TypeSerializer[] { outerSerializer.getKeySerializer(), outerSerializer.getValueSerializer() }; } } From 68fab1277ca31ad65ddd594b0b6b7d4d9c0e4383 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Wed, 5 Dec 2018 16:34:23 +0800 Subject: [PATCH 062/147] [FLINK-11073] [core] Let ScalaEitherSerializerSnapshot be a CompositeTypeSerializerSnapshot --- .../ScalaEitherSerializerSnapshot.java | 62 +++++-------------- .../scala/typeutils/EitherSerializer.scala | 2 +- 2 files changed, 17 insertions(+), 47 deletions(-) diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerSnapshot.java b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerSnapshot.java index b67e47b4effa2f..26cfef5a1a8400 100644 --- a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerSnapshot.java +++ b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerSnapshot.java @@ -18,81 +18,51 @@ package org.apache.flink.api.scala.typeutils; -import org.apache.flink.api.common.typeutils.CompositeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; -import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataOutputView; -import org.apache.flink.util.Preconditions; - -import java.io.IOException; import scala.util.Either; -import static org.apache.flink.util.Preconditions.checkState; - /** * Configuration snapshot for serializers of Scala's {@link Either} type, * containing configuration snapshots of the Left and Right serializers. */ -public class ScalaEitherSerializerSnapshot implements TypeSerializerSnapshot> { +public class ScalaEitherSerializerSnapshot extends CompositeTypeSerializerSnapshot, EitherSerializer> { private static final int CURRENT_VERSION = 1; - private CompositeSerializerSnapshot nestedLeftRightSerializerSnapshot; - /** * Constructor for read instantiation. */ - public ScalaEitherSerializerSnapshot() {} + public ScalaEitherSerializerSnapshot() { + super(EitherSerializer.class); + } /** * Constructor to create the snapshot for writing. */ - public ScalaEitherSerializerSnapshot(TypeSerializer leftSerializer, TypeSerializer rightSerializer) { - Preconditions.checkNotNull(leftSerializer); - Preconditions.checkNotNull(rightSerializer); - this.nestedLeftRightSerializerSnapshot = new CompositeSerializerSnapshot(leftSerializer, rightSerializer); + public ScalaEitherSerializerSnapshot(EitherSerializer eitherSerializer) { + super(eitherSerializer); } @Override - public int getCurrentVersion() { + public int getCurrentOuterSnapshotVersion() { return CURRENT_VERSION; } @Override - public TypeSerializer> restoreSerializer() { - return new EitherSerializer<>( - nestedLeftRightSerializerSnapshot.getRestoreSerializer(0), - nestedLeftRightSerializerSnapshot.getRestoreSerializer(1)); - } - - @Override - public TypeSerializerSchemaCompatibility> resolveSchemaCompatibility( - TypeSerializer> newSerializer) { - checkState(nestedLeftRightSerializerSnapshot != null); + protected EitherSerializer createOuterSerializerWithNestedSerializers(TypeSerializer[] nestedSerializers) { + @SuppressWarnings("unchecked") + TypeSerializer leftSerializer = (TypeSerializer) nestedSerializers[0]; - if (newSerializer instanceof EitherSerializer) { - EitherSerializer serializer = (EitherSerializer) newSerializer; + @SuppressWarnings("unchecked") + TypeSerializer rightSerializer = (TypeSerializer) nestedSerializers[1]; - return nestedLeftRightSerializerSnapshot.resolveCompatibilityWithNested( - TypeSerializerSchemaCompatibility.compatibleAsIs(), - serializer.getLeftSerializer(), - serializer.getRightSerializer()); - } - else { - return TypeSerializerSchemaCompatibility.incompatible(); - } - } - - @Override - public void writeSnapshot(DataOutputView out) throws IOException { - nestedLeftRightSerializerSnapshot.writeCompositeSnapshot(out); + return new EitherSerializer<>(leftSerializer, rightSerializer); } @Override - public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { - this.nestedLeftRightSerializerSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, userCodeClassLoader); + protected TypeSerializer[] getNestedSerializers(EitherSerializer outerSerializer) { + return new TypeSerializer[] { outerSerializer.getLeftSerializer(), outerSerializer.getRightSerializer() }; } } diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala index 68432a6f1a5029..0427bb35f2b02f 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala @@ -125,7 +125,7 @@ class EitherSerializer[A, B]( // -------------------------------------------------------------------------------------------- override def snapshotConfiguration(): ScalaEitherSerializerSnapshot[A, B] = { - new ScalaEitherSerializerSnapshot[A, B](leftSerializer, rightSerializer) + new ScalaEitherSerializerSnapshot[A, B](this) } override def ensureCompatibility( From e791b1a29b675af4f290be9b68ae7f03b2de43c5 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Thu, 6 Dec 2018 19:07:16 +0800 Subject: [PATCH 063/147] [FLINK-11073] [core] Replace GenericArraySerializerConfigSnapshot with new GenericArraySerializerSnapshot --- .../base/GenericArraySerializer.java | 4 +- .../GenericArraySerializerConfigSnapshot.java | 20 +++-- .../base/GenericArraySerializerSnapshot.java | 81 +++++++++++++++++++ ...teTypeSerializerSnapshotMigrationTest.java | 4 +- 4 files changed, 94 insertions(+), 15 deletions(-) create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerSnapshot.java diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java index 55ba8ab477f5a6..a4949fb6fa773c 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java @@ -206,7 +206,7 @@ public String toString() { // -------------------------------------------------------------------------------------------- @Override - public GenericArraySerializerConfigSnapshot snapshotConfiguration() { - return new GenericArraySerializerConfigSnapshot<>(this); + public GenericArraySerializerSnapshot snapshotConfiguration() { + return new GenericArraySerializerSnapshot<>(this); } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java index b0aa24128d214b..8cbe76c07b1304 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java @@ -38,8 +38,12 @@ * Point-in-time configuration of a {@link GenericArraySerializer}. * * @param The component type. + * + * @deprecated this is deprecated and no longer used by the {@link GenericArraySerializer}. + * It has been replaced by {@link GenericArraySerializerSnapshot}. */ @Internal +@Deprecated public final class GenericArraySerializerConfigSnapshot implements TypeSerializerSnapshot { private static final int CURRENT_VERSION = 2; @@ -118,18 +122,12 @@ public TypeSerializer restoreSerializer() { @Override public TypeSerializerSchemaCompatibility resolveSchemaCompatibility(TypeSerializer newSerializer) { - checkState(componentClass != null && nestedSnapshot != null); - if (newSerializer instanceof GenericArraySerializer) { - GenericArraySerializer serializer = (GenericArraySerializer) newSerializer; - TypeSerializerSchemaCompatibility compat = serializer.getComponentClass() == componentClass ? - TypeSerializerSchemaCompatibility.compatibleAsIs() : - TypeSerializerSchemaCompatibility.incompatible(); - - return nestedSnapshot.resolveCompatibilityWithNested( - compat, serializer.getComponentSerializer()); - } - else { + // delegate to the new snapshot class + GenericArraySerializer castedNewSerializer = (GenericArraySerializer) newSerializer; + GenericArraySerializerSnapshot newSnapshot = new GenericArraySerializerSnapshot<>(castedNewSerializer); + return newSnapshot.resolveSchemaCompatibility(castedNewSerializer); + } else { return TypeSerializerSchemaCompatibility.incompatible(); } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerSnapshot.java new file mode 100644 index 00000000000000..3f54dee67cb24e --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerSnapshot.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.typeutils.base; + +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.util.InstantiationUtil; + +import java.io.IOException; + +/** + * Point-in-time configuration of a {@link GenericArraySerializer}. + * + * @param The component type. + */ +public final class GenericArraySerializerSnapshot extends CompositeTypeSerializerSnapshot { + + private static final int CURRENT_VERSION = 1; + + private Class componentClass; + + /** + * Constructor to be used for read instantiation. + */ + public GenericArraySerializerSnapshot() { + super(GenericArraySerializer.class); + } + + /** + * Constructor to be used for writing the snapshot. + */ + public GenericArraySerializerSnapshot(GenericArraySerializer genericArraySerializer) { + super(genericArraySerializer); + this.componentClass = genericArraySerializer.getComponentClass(); + } + + @Override + protected int getCurrentOuterSnapshotVersion() { + return CURRENT_VERSION; + } + + @Override + protected void writeOuterSnapshot(DataOutputView out) throws IOException { + out.writeUTF(componentClass.getName()); + } + + @Override + protected void readOuterSnapshot(int readOuterSnapshotVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException { + this.componentClass = InstantiationUtil.resolveClassByName(in, userCodeClassLoader); + } + + @Override + protected GenericArraySerializer createOuterSerializerWithNestedSerializers(TypeSerializer[] nestedSerializers) { + @SuppressWarnings("unchecked") + TypeSerializer componentSerializer = (TypeSerializer) nestedSerializers[0]; + return new GenericArraySerializer<>(componentClass, componentSerializer); + } + + @Override + protected TypeSerializer[] getNestedSerializers(GenericArraySerializer outerSerializer) { + return new TypeSerializer[] { outerSerializer.getComponentSerializer() }; + } +} diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshotMigrationTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshotMigrationTest.java index c7b002a21ec0f4..c6b49a4839ebc6 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshotMigrationTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshotMigrationTest.java @@ -19,7 +19,7 @@ package org.apache.flink.api.common.typeutils; import org.apache.flink.api.common.typeutils.base.GenericArraySerializer; -import org.apache.flink.api.common.typeutils.base.GenericArraySerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.base.GenericArraySerializerSnapshot; import org.apache.flink.api.common.typeutils.base.IntSerializer; import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.api.java.typeutils.runtime.EitherSerializer; @@ -55,7 +55,7 @@ public static Collection testSpecifications() { // GenericArray - final TestSpecification array = TestSpecification.builder("1.6-generic-array", GenericArraySerializer.class, GenericArraySerializerConfigSnapshot.class) + final TestSpecification array = TestSpecification.builder("1.6-generic-array", GenericArraySerializer.class, GenericArraySerializerSnapshot.class) .withSerializerProvider(() -> new GenericArraySerializer<>(String.class, StringSerializer.INSTANCE)) .withSnapshotDataLocation("flink-1.6-array-type-serializer-snapshot") .withTestData("flink-1.6-array-type-serializer-data", 10); From 301be5552290788864676b131e09741f769ef471 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Mon, 7 Jan 2019 12:50:27 +0100 Subject: [PATCH 064/147] [FLINK-11073] [core] Replace EitherSerializerSnapshot with new JavaEitherSerializerSnapshot --- .../typeutils/runtime/EitherSerializer.java | 4 +- .../runtime/EitherSerializerSnapshot.java | 12 ++-- .../runtime/JavaEitherSerializerSnapshot.java | 61 +++++++++++++++++++ ...teTypeSerializerSnapshotMigrationTest.java | 4 +- 4 files changed, 72 insertions(+), 9 deletions(-) create mode 100644 flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/JavaEitherSerializerSnapshot.java diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java index 3d4e8e92762a98..01286400694a07 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java @@ -205,7 +205,7 @@ public int hashCode() { // ------------------------------------------------------------------------ @Override - public EitherSerializerSnapshot snapshotConfiguration() { - return new EitherSerializerSnapshot<>(leftSerializer, rightSerializer); + public JavaEitherSerializerSnapshot snapshotConfiguration() { + return new JavaEitherSerializerSnapshot<>(this); } } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerSnapshot.java index 016fd0430f7120..3b7a8e785ef450 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerSnapshot.java @@ -35,8 +35,12 @@ /** * Configuration snapshot for the {@link EitherSerializer}. + * + * @deprecated this snapshot class is no longer used by any serializers. + * Instead, {@link JavaEitherSerializerSnapshot} is used. */ @Internal +@Deprecated public final class EitherSerializerSnapshot implements TypeSerializerSnapshot> { private static final int CURRENT_VERSION = 2; @@ -110,12 +114,10 @@ public TypeSerializerSchemaCompatibility> resolveSchemaCompatibilit checkState(nestedSnapshot != null); if (newSerializer instanceof EitherSerializer) { + // delegate compatibility check to the new snapshot class EitherSerializer serializer = (EitherSerializer) newSerializer; - - return nestedSnapshot.resolveCompatibilityWithNested( - TypeSerializerSchemaCompatibility.compatibleAsIs(), - serializer.getLeftSerializer(), - serializer.getRightSerializer()); + JavaEitherSerializerSnapshot newSnapshot = new JavaEitherSerializerSnapshot<>(serializer); + return newSnapshot.resolveSchemaCompatibility(serializer); } else { return TypeSerializerSchemaCompatibility.incompatible(); diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/JavaEitherSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/JavaEitherSerializerSnapshot.java new file mode 100644 index 00000000000000..503634599f8879 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/JavaEitherSerializerSnapshot.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.java.typeutils.runtime; + +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.types.Either; + +/** + * Snapshot class for the {@link EitherSerializer}. + */ +public class JavaEitherSerializerSnapshot extends CompositeTypeSerializerSnapshot, EitherSerializer> { + + private static final int CURRENT_VERSION = 1; + + /** + * Constructor for read instantiation. + */ + @SuppressWarnings("unused") + public JavaEitherSerializerSnapshot() { + super(EitherSerializer.class); + } + + /** + * Constructor to create the snapshot for writing. + */ + public JavaEitherSerializerSnapshot(EitherSerializer eitherSerializer) { + super(eitherSerializer); + } + + @Override + protected int getCurrentOuterSnapshotVersion() { + return CURRENT_VERSION; + } + + @Override + protected EitherSerializer createOuterSerializerWithNestedSerializers(TypeSerializer[] nestedSerializers) { + return new EitherSerializer<>(nestedSerializers[0], nestedSerializers[1]); + } + + @Override + protected TypeSerializer[] getNestedSerializers(EitherSerializer outerSerializer) { + return new TypeSerializer[]{ outerSerializer.getLeftSerializer(), outerSerializer.getRightSerializer() }; + } +} diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshotMigrationTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshotMigrationTest.java index c6b49a4839ebc6..62135d7e550863 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshotMigrationTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshotMigrationTest.java @@ -23,7 +23,7 @@ import org.apache.flink.api.common.typeutils.base.IntSerializer; import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.api.java.typeutils.runtime.EitherSerializer; -import org.apache.flink.api.java.typeutils.runtime.EitherSerializerSnapshot; +import org.apache.flink.api.java.typeutils.runtime.JavaEitherSerializerSnapshot; import org.apache.flink.types.Either; import org.junit.runner.RunWith; @@ -48,7 +48,7 @@ public static Collection testSpecifications() { // Either - final TestSpecification> either = TestSpecification.>builder("1.6-either", EitherSerializer.class, EitherSerializerSnapshot.class) + final TestSpecification> either = TestSpecification.>builder("1.6-either", EitherSerializer.class, JavaEitherSerializerSnapshot.class) .withSerializerProvider(() -> new EitherSerializer<>(StringSerializer.INSTANCE, IntSerializer.INSTANCE)) .withSnapshotDataLocation("flink-1.6-either-type-serializer-snapshot") .withTestData("flink-1.6-either-type-serializer-data", 10); From 25dc4832324bb0be0731747c5856441a6b984a95 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Thu, 6 Dec 2018 19:14:17 +0800 Subject: [PATCH 065/147] [hotfix] [core] Assert current serializer snapshot is expected snapshot class in TypeSerializerSnapshotMigrationTestBase This strengthens the test base to make use of the provided snapshot class in the test specifications, to assert that serializers are really using the expected classes for their current serializer snapshots. --- .../typeutils/TypeSerializerSnapshotMigrationTestBase.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java index 186f50482afc42..ea18309ac98933 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotMigrationTestBase.java @@ -103,6 +103,8 @@ public void movingForward() throws IOException { TypeSerializer restoredSerializer = previousSnapshot.restoreSerializer(); TypeSerializerSnapshot nextSnapshot = restoredSerializer.snapshotConfiguration(); + assertThat(nextSnapshot, instanceOf(testSpecification.snapshotClass)); + TypeSerializerSnapshot nextSnapshotDeserialized = writeAndThenReadTheSnapshot(restoredSerializer, nextSnapshot); assertThat(nextSnapshotDeserialized, allOf( @@ -247,6 +249,10 @@ private Path getSnapshotDataLocation() { return resourcePath(this.snapshotDataLocation); } + public Class> getSnapshotClass() { + return snapshotClass; + } + @Override public String toString() { return String.format("%s , %s, %s", name, serializerType.getSimpleName(), snapshotClass.getSimpleName()); From 5bcaf2e0fdf9c05f257eeb67d67bf1320faeea8f Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Mon, 7 Jan 2019 13:28:19 +0100 Subject: [PATCH 066/147] [FLINK-11073] [core] Rename CompositeSerializerSnapshot to NestedSerializersSnapshotDelegate After introducing CompositeTypeSerializerSnapshot, the CompositeSerializerSnapshot has been reworked to only deal with concerns of delegating reading and writing of the nested serializers' snapshots. It no longer deals with resolving the final compatibility result for the outer composite serializer. Therefore, it is renamed properly as NestedSerializersSnapshotDelegate, and also annotated as an internal class, since we want users to use the more powerful CompositeTypeSerializerSnapshot instead. This closes #7422. --- .../CompositeTypeSerializerSnapshot.java | 12 +++--- ...=> NestedSerializersSnapshotDelegate.java} | 37 ++++++++++--------- .../GenericArraySerializerConfigSnapshot.java | 14 +++---- .../runtime/EitherSerializerSnapshot.java | 16 ++++---- 4 files changed, 41 insertions(+), 38 deletions(-) rename flink-core/src/main/java/org/apache/flink/api/common/typeutils/{CompositeSerializerSnapshot.java => NestedSerializersSnapshotDelegate.java} (81%) diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshot.java index 71c0836b35304d..c73e24c0897e6f 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerSnapshot.java @@ -99,7 +99,7 @@ public abstract class CompositeTypeSerializerSnapshot correspondingSerializerClass; @@ -120,7 +120,7 @@ public CompositeTypeSerializerSnapshot(Class correspondingSerializerClass) { @SuppressWarnings("unchecked") public CompositeTypeSerializerSnapshot(S serializerInstance) { Preconditions.checkNotNull(serializerInstance); - this.compositeSerializerSnapshot = new CompositeSerializerSnapshot(getNestedSerializers(serializerInstance)); + this.nestedSerializersSnapshotDelegate = new NestedSerializersSnapshotDelegate(getNestedSerializers(serializerInstance)); this.correspondingSerializerClass = (Class) serializerInstance.getClass(); } @@ -132,7 +132,7 @@ public final int getCurrentVersion() { @Override public final void writeSnapshot(DataOutputView out) throws IOException { internalWriteOuterSnapshot(out); - compositeSerializerSnapshot.writeCompositeSnapshot(out); + nestedSerializersSnapshotDelegate.writeNestedSerializerSnapshots(out); } @Override @@ -142,7 +142,7 @@ public final void readSnapshot(int readVersion, DataInputView in, ClassLoader us } else { legacyInternalReadOuterSnapshot(readVersion, in, userCodeClassLoader); } - this.compositeSerializerSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, userCodeClassLoader); + this.nestedSerializersSnapshotDelegate = NestedSerializersSnapshotDelegate.readNestedSerializerSnapshots(in, userCodeClassLoader); } @Override @@ -161,14 +161,14 @@ public final TypeSerializerSchemaCompatibility resolveSchemaCompatibility(Typ // since outer configuration is compatible, the final compatibility result depends only on the nested serializers return constructFinalSchemaCompatibilityResult( getNestedSerializers(castedNewSerializer), - compositeSerializerSnapshot.getNestedSerializerSnapshots()); + nestedSerializersSnapshotDelegate.getNestedSerializerSnapshots()); } @Override public final TypeSerializer restoreSerializer() { @SuppressWarnings("unchecked") TypeSerializer serializer = (TypeSerializer) - createOuterSerializerWithNestedSerializers(compositeSerializerSnapshot.getRestoreSerializers()); + createOuterSerializerWithNestedSerializers(nestedSerializersSnapshotDelegate.getRestoredNestedSerializers()); return serializer; } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/NestedSerializersSnapshotDelegate.java similarity index 81% rename from flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeSerializerSnapshot.java rename to flink-core/src/main/java/org/apache/flink/api/common/typeutils/NestedSerializersSnapshotDelegate.java index 5b3f7751087f98..a4dcdd2cda969e 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeSerializerSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/NestedSerializersSnapshotDelegate.java @@ -18,7 +18,7 @@ package org.apache.flink.api.common.typeutils; -import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.annotation.Internal; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -30,21 +30,21 @@ import static org.apache.flink.util.Preconditions.checkArgument; /** - * A CompositeSerializerSnapshot represents the snapshots of multiple serializers that are used + * A NestedSerializersSnapshotDelegate represents the snapshots of multiple serializers that are used * by an outer serializer. Examples would be tuples, where the outer serializer is the tuple - * format serializer, an the CompositeSerializerSnapshot holds the serializers for the + * format serializer, and the NestedSerializersSnapshotDelegate holds the serializers for the * different tuple fields. * - *

The CompositeSerializerSnapshot does not implement the {@link TypeSerializerSnapshot} interface. + *

The NestedSerializersSnapshotDelegate does not implement the {@link TypeSerializerSnapshot} interface. * It is not meant to be inherited from, but to be composed with a serializer snapshot implementation. * - *

The CompositeSerializerSnapshot has its own versioning internally, it does not couple its + *

The NestedSerializersSnapshotDelegate has its own versioning internally, it does not couple its * versioning to the versioning of the TypeSerializerSnapshot that builds on top of this class. - * That way, the CompositeSerializerSnapshot and enclosing TypeSerializerSnapshot the can evolve + * That way, the NestedSerializersSnapshotDelegate and enclosing TypeSerializerSnapshot the can evolve * their formats independently. */ -@PublicEvolving -public class CompositeSerializerSnapshot { +@Internal +public class NestedSerializersSnapshotDelegate { /** Magic number for integrity checks during deserialization. */ private static final int MAGIC_NUMBER = 1333245; @@ -58,14 +58,14 @@ public class CompositeSerializerSnapshot { /** * Constructor to create a snapshot for writing. */ - public CompositeSerializerSnapshot(TypeSerializer... serializers) { + public NestedSerializersSnapshotDelegate(TypeSerializer... serializers) { this.nestedSnapshots = TypeSerializerUtils.snapshotBackwardsCompatible(serializers); } /** * Constructor to create a snapshot during deserialization. */ - private CompositeSerializerSnapshot(TypeSerializerSnapshot[] snapshots) { + private NestedSerializersSnapshotDelegate(TypeSerializerSnapshot[] snapshots) { this.nestedSnapshots = snapshots; } @@ -77,14 +77,14 @@ private CompositeSerializerSnapshot(TypeSerializerSnapshot[] snapshots) { * Produces a restore serializer from each contained serializer configuration snapshot. * The serializers are returned in the same order as the snapshots are stored. */ - public TypeSerializer[] getRestoreSerializers() { + public TypeSerializer[] getRestoredNestedSerializers() { return snapshotsToRestoreSerializers(nestedSnapshots); } /** * Creates the restore serializer from the pos-th config snapshot. */ - public TypeSerializer getRestoreSerializer(int pos) { + public TypeSerializer getRestoredNestedSerializer(int pos) { checkArgument(pos < nestedSnapshots.length); @SuppressWarnings("unchecked") @@ -105,6 +105,9 @@ public TypeSerializerSnapshot[] getNestedSerializerSnapshots() { /** * Resolves the compatibility of the nested serializer snapshots with the nested * serializers of the new outer serializer. + * + * @deprecated this no method will be removed in the future. Resolving compatibility for nested + * serializers is now handled by {@link CompositeTypeSerializerSnapshot}. */ @Deprecated public TypeSerializerSchemaCompatibility resolveCompatibilityWithNested( @@ -145,7 +148,7 @@ public TypeSerializerSchemaCompatibility resolveCompatibilityWithNested( /** * Writes the composite snapshot of all the contained serializers. */ - public final void writeCompositeSnapshot(DataOutputView out) throws IOException { + public final void writeNestedSerializerSnapshots(DataOutputView out) throws IOException { out.writeInt(MAGIC_NUMBER); out.writeInt(VERSION); @@ -158,7 +161,7 @@ public final void writeCompositeSnapshot(DataOutputView out) throws IOException /** * Reads the composite snapshot of all the contained serializers. */ - public static CompositeSerializerSnapshot readCompositeSnapshot(DataInputView in, ClassLoader cl) throws IOException { + public static NestedSerializersSnapshotDelegate readNestedSerializerSnapshots(DataInputView in, ClassLoader cl) throws IOException { final int magicNumber = in.readInt(); if (magicNumber != MAGIC_NUMBER) { throw new IOException(String.format("Corrupt data, magic number mismatch. Expected %8x, found %8x", @@ -177,14 +180,14 @@ public static CompositeSerializerSnapshot readCompositeSnapshot(DataInputView in nestedSnapshots[i] = TypeSerializerSnapshot.readVersionedSnapshot(in, cl); } - return new CompositeSerializerSnapshot(nestedSnapshots); + return new NestedSerializersSnapshotDelegate(nestedSnapshots); } /** * Reads the composite snapshot of all the contained serializers in a way that is compatible * with Version 1 of the deprecated {@link CompositeTypeSerializerConfigSnapshot}. */ - public static CompositeSerializerSnapshot legacyReadProductSnapshots(DataInputView in, ClassLoader cl) throws IOException { + public static NestedSerializersSnapshotDelegate legacyReadNestedSerializerSnapshots(DataInputView in, ClassLoader cl) throws IOException { @SuppressWarnings("deprecation") final List, TypeSerializerSnapshot>> serializersAndSnapshots = TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, cl); @@ -193,7 +196,7 @@ public static CompositeSerializerSnapshot legacyReadProductSnapshots(DataInputVi .map(t -> t.f1) .toArray(TypeSerializerSnapshot[]::new); - return new CompositeSerializerSnapshot(nestedSnapshots); + return new NestedSerializersSnapshotDelegate(nestedSnapshots); } // ------------------------------------------------------------------------ diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java index 8cbe76c07b1304..cfc2e9825d7efa 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java @@ -19,7 +19,7 @@ package org.apache.flink.api.common.typeutils.base; import org.apache.flink.annotation.Internal; -import org.apache.flink.api.common.typeutils.CompositeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.NestedSerializersSnapshotDelegate; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; @@ -54,7 +54,7 @@ public final class GenericArraySerializerConfigSnapshot implements TypeSerial /** Snapshot handling for the component serializer snapshot. */ @Nullable - private CompositeSerializerSnapshot nestedSnapshot; + private NestedSerializersSnapshotDelegate nestedSnapshot; /** * Constructor for read instantiation. @@ -67,7 +67,7 @@ public GenericArraySerializerConfigSnapshot() {} */ public GenericArraySerializerConfigSnapshot(GenericArraySerializer serializer) { this.componentClass = serializer.getComponentClass(); - this.nestedSnapshot = new CompositeSerializerSnapshot(serializer.getComponentSerializer()); + this.nestedSnapshot = new NestedSerializersSnapshotDelegate(serializer.getComponentSerializer()); } // ------------------------------------------------------------------------ @@ -81,7 +81,7 @@ public int getCurrentVersion() { public void writeSnapshot(DataOutputView out) throws IOException { checkState(componentClass != null && nestedSnapshot != null); out.writeUTF(componentClass.getName()); - nestedSnapshot.writeCompositeSnapshot(out); + nestedSnapshot.writeNestedSerializerSnapshots(out); } @Override @@ -99,7 +99,7 @@ public void readSnapshot(int readVersion, DataInputView in, ClassLoader classLoa } private void readV1(DataInputView in, ClassLoader classLoader) throws IOException { - nestedSnapshot = CompositeSerializerSnapshot.legacyReadProductSnapshots(in, classLoader); + nestedSnapshot = NestedSerializersSnapshotDelegate.legacyReadNestedSerializerSnapshots(in, classLoader); try (DataInputViewStream inViewWrapper = new DataInputViewStream(in)) { componentClass = InstantiationUtil.deserializeObject(inViewWrapper, classLoader); @@ -111,13 +111,13 @@ private void readV1(DataInputView in, ClassLoader classLoader) throws IOExceptio private void readV2(DataInputView in, ClassLoader classLoader) throws IOException { componentClass = InstantiationUtil.resolveClassByName(in, classLoader); - nestedSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, classLoader); + nestedSnapshot = NestedSerializersSnapshotDelegate.readNestedSerializerSnapshots(in, classLoader); } @Override public TypeSerializer restoreSerializer() { checkState(componentClass != null && nestedSnapshot != null); - return new GenericArraySerializer<>(componentClass, nestedSnapshot.getRestoreSerializer(0)); + return new GenericArraySerializer<>(componentClass, nestedSnapshot.getRestoredNestedSerializer(0)); } @Override diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerSnapshot.java index 3b7a8e785ef450..1779ec8e744aa5 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerSnapshot.java @@ -19,7 +19,7 @@ package org.apache.flink.api.java.typeutils.runtime; import org.apache.flink.annotation.Internal; -import org.apache.flink.api.common.typeutils.CompositeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.NestedSerializersSnapshotDelegate; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; @@ -47,7 +47,7 @@ public final class EitherSerializerSnapshot implements TypeSerializerSnaps /** Snapshot handling for the component serializer snapshot. */ @Nullable - private CompositeSerializerSnapshot nestedSnapshot; + private NestedSerializersSnapshotDelegate nestedSnapshot; /** * Constructor for read instantiation. @@ -62,7 +62,7 @@ public EitherSerializerSnapshot( TypeSerializer leftSerializer, TypeSerializer rightSerializer) { - this.nestedSnapshot = new CompositeSerializerSnapshot(leftSerializer, rightSerializer); + this.nestedSnapshot = new NestedSerializersSnapshotDelegate(leftSerializer, rightSerializer); } // ------------------------------------------------------------------------ @@ -75,7 +75,7 @@ public int getCurrentVersion() { @Override public void writeSnapshot(DataOutputView out) throws IOException { checkState(nestedSnapshot != null); - nestedSnapshot.writeCompositeSnapshot(out); + nestedSnapshot.writeNestedSerializerSnapshots(out); } @Override @@ -93,19 +93,19 @@ public void readSnapshot(int readVersion, DataInputView in, ClassLoader classLoa } private void readV1(DataInputView in, ClassLoader classLoader) throws IOException { - nestedSnapshot = CompositeSerializerSnapshot.legacyReadProductSnapshots(in, classLoader); + nestedSnapshot = NestedSerializersSnapshotDelegate.legacyReadNestedSerializerSnapshots(in, classLoader); } private void readV2(DataInputView in, ClassLoader classLoader) throws IOException { - nestedSnapshot = CompositeSerializerSnapshot.readCompositeSnapshot(in, classLoader); + nestedSnapshot = NestedSerializersSnapshotDelegate.readNestedSerializerSnapshots(in, classLoader); } @Override public TypeSerializer> restoreSerializer() { checkState(nestedSnapshot != null); return new EitherSerializer<>( - nestedSnapshot.getRestoreSerializer(0), - nestedSnapshot.getRestoreSerializer(1)); + nestedSnapshot.getRestoredNestedSerializer(0), + nestedSnapshot.getRestoredNestedSerializer(1)); } @Override From 1072bbfe500999eb3e3710271f15ff53a534475d Mon Sep 17 00:00:00 2001 From: Tzu-Li Chen Date: Tue, 16 Oct 2018 01:29:56 +0800 Subject: [PATCH 067/147] [FLINK-10555] [test] Port AkkaSslITCase to new code base [FLINK-10555] tests are all of BlobServer initialization, also remove already covered tests This closes #6849. --- .../flink/runtime/blob/BlobServerSSLTest.java | 110 ++++++++++++ .../flink/runtime/akka/AkkaSslITCase.scala | 156 ------------------ 2 files changed, 110 insertions(+), 156 deletions(-) create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobServerSSLTest.java delete mode 100644 flink-runtime/src/test/scala/org/apache/flink/runtime/akka/AkkaSslITCase.scala diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobServerSSLTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobServerSSLTest.java new file mode 100644 index 00000000000000..49957c22a26444 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobServerSSLTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.blob; + +import org.apache.flink.configuration.AkkaOptions; +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.JobManagerOptions; +import org.apache.flink.configuration.SecurityOptions; +import org.apache.flink.configuration.TaskManagerOptions; +import org.apache.flink.util.TestLogger; + +import org.junit.Test; + +import java.io.IOException; + +import static org.apache.flink.util.ExceptionUtils.findThrowable; +import static org.apache.flink.util.ExceptionUtils.findThrowableWithMessage; +import static org.junit.Assert.fail; + +/** + * Testing a {@link BlobServer} would fail with improper SSL config. + */ +public class BlobServerSSLTest extends TestLogger { + + @Test + public void testFailedToInitWithTwoProtocolsSet() { + final Configuration config = new Configuration(); + + config.setString(JobManagerOptions.ADDRESS, "127.0.0.1"); + config.setString(TaskManagerOptions.HOST, "127.0.0.1"); + config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); + config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1); + + config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true); + config.setString(SecurityOptions.SSL_KEYSTORE, + getClass().getResource("/local127.keystore").getPath()); + config.setString(SecurityOptions.SSL_KEYSTORE_PASSWORD, "password"); + config.setString(SecurityOptions.SSL_KEY_PASSWORD, "password"); + config.setString(SecurityOptions.SSL_TRUSTSTORE, + getClass().getResource("/local127.truststore").getPath()); + + config.setString(SecurityOptions.SSL_TRUSTSTORE_PASSWORD, "password"); + config.setString(SecurityOptions.SSL_ALGORITHMS, "TLSv1,TLSv1.1"); + + try (final BlobServer blobServer = new BlobServer(config, new VoidBlobStore())) { + fail(); + } catch (Exception e) { + findThrowable(e, IOException.class); + findThrowableWithMessage(e, "Unable to open BLOB Server in specified port range: 0"); + } + } + + @Test + public void testFailedToInitWithInvalidSslKeystoreConfigured() { + final Configuration config = new Configuration(); + + config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); + config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1); + config.setString(AkkaOptions.ASK_TIMEOUT, "2 s"); + + config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true); + config.setString(SecurityOptions.SSL_KEYSTORE, "invalid.keystore"); + config.setString(SecurityOptions.SSL_KEYSTORE_PASSWORD, "password"); + config.setString(SecurityOptions.SSL_KEY_PASSWORD, "password"); + config.setString(SecurityOptions.SSL_TRUSTSTORE, "invalid.keystore"); + config.setString(SecurityOptions.SSL_TRUSTSTORE_PASSWORD, "password"); + + try (final BlobServer blobServer = new BlobServer(config, new VoidBlobStore())) { + fail(); + } catch (Exception e) { + findThrowable(e, IOException.class); + findThrowableWithMessage(e, "Failed to initialize SSL for the blob server"); + } + } + + @Test + public void testFailedToInitWithMissingMandatorySslConfiguration() { + final Configuration config = new Configuration(); + + config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); + config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1); + config.setString(AkkaOptions.ASK_TIMEOUT, "2 s"); + + config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true); + + try (final BlobServer blobServer = new BlobServer(config, new VoidBlobStore())) { + fail(); + } catch (Exception e) { + findThrowable(e, IOException.class); + findThrowableWithMessage(e, "Failed to initialize SSL for the blob server"); + } + } +} diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/akka/AkkaSslITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/akka/AkkaSslITCase.scala deleted file mode 100644 index a6b602892b3189..00000000000000 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/akka/AkkaSslITCase.scala +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.akka - -import akka.actor.ActorSystem -import akka.testkit.{ImplicitSender, TestKit} -import org.apache.flink.configuration._ -import org.apache.flink.runtime.testingUtils.{ScalaTestingUtils, TestingCluster, TestingUtils} -import org.junit.runner.RunWith -import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} -import org.scalatest.junit.JUnitRunner - -/** - * Testing the flink cluster using SSL transport for akka remoting - */ -@RunWith(classOf[JUnitRunner]) -class AkkaSslITCase(_system: ActorSystem) - extends TestKit(_system) - with ImplicitSender - with WordSpecLike - with Matchers - with BeforeAndAfterAll - with ScalaTestingUtils { - - def this() = this(ActorSystem("TestingActorSystem", TestingUtils.testConfig)) - - override def afterAll(): Unit = { - TestKit.shutdownActorSystem(system) - } - - "The flink Cluster" must { - - "start with akka ssl enabled" in { - - val config = new Configuration() - config.setString(JobManagerOptions.ADDRESS, "127.0.0.1") - config.setString(TaskManagerOptions.HOST, "127.0.0.1") - config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1) - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1) - - config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true) - config.setString(SecurityOptions.SSL_KEYSTORE, - getClass.getResource("/local127.keystore").getPath) - config.setString(SecurityOptions.SSL_KEYSTORE_PASSWORD, "password") - config.setString(SecurityOptions.SSL_KEY_PASSWORD, "password") - config.setString(SecurityOptions.SSL_TRUSTSTORE, - getClass.getResource("/local127.truststore").getPath) - - config.setString(SecurityOptions.SSL_TRUSTSTORE_PASSWORD, "password") - - val cluster = new TestingCluster(config, false) - - cluster.start(true) - - assert(cluster.running) - } - - "Failed to start ssl enabled akka with two protocols set" in { - - an[Exception] should be thrownBy { - - val config = new Configuration() - config.setString(JobManagerOptions.ADDRESS, "127.0.0.1") - config.setString(TaskManagerOptions.HOST, "127.0.0.1") - config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1) - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1) - - config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true) - config.setString(SecurityOptions.SSL_KEYSTORE, - getClass.getResource("/local127.keystore").getPath) - config.setString(SecurityOptions.SSL_KEYSTORE_PASSWORD, "password") - config.setString(SecurityOptions.SSL_KEY_PASSWORD, "password") - config.setString(SecurityOptions.SSL_TRUSTSTORE, - getClass.getResource("/local127.truststore").getPath) - - config.setString(SecurityOptions.SSL_TRUSTSTORE_PASSWORD, "password") - config.setString(SecurityOptions.SSL_ALGORITHMS, "TLSv1,TLSv1.1") - - val cluster = new TestingCluster(config, false) - - cluster.start(true) - } - } - - "start with akka ssl disabled" in { - - val config = new Configuration() - config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1) - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1) - config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, false) - - val cluster = new TestingCluster(config, false) - - cluster.start(true) - - assert(cluster.running) - } - - "fail to start with invalid ssl keystore configured" in { - - an[Exception] should be thrownBy { - - val config = new Configuration() - config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1) - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1) - config.setString(AkkaOptions.ASK_TIMEOUT, "2 s") - - config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true) - config.setString(SecurityOptions.SSL_KEYSTORE, "invalid.keystore") - config.setString(SecurityOptions.SSL_KEYSTORE_PASSWORD, "password") - config.setString(SecurityOptions.SSL_KEY_PASSWORD, "password") - config.setString(SecurityOptions.SSL_TRUSTSTORE, "invalid.keystore") - config.setString(SecurityOptions.SSL_TRUSTSTORE_PASSWORD, "password") - - val cluster = new TestingCluster(config, false) - - cluster.start(true) - } - } - - "fail to start with missing mandatory ssl configuration" in { - - an[Exception] should be thrownBy { - - val config = new Configuration() - config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1) - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1) - config.setString(AkkaOptions.ASK_TIMEOUT, "2 s") - - config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true) - - val cluster = new TestingCluster(config, false) - - cluster.start(true) - } - } - - } - -} From fc98a1f96cce951355c08a6be4193bbdef02b10c Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Tue, 8 Jan 2019 19:04:18 +0100 Subject: [PATCH 068/147] [FLINK-10555] Clean-up BlobServerSSLTest --- .../flink/runtime/blob/BlobServerSSLTest.java | 23 +++---------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobServerSSLTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobServerSSLTest.java index 49957c22a26444..6f849304643a79 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobServerSSLTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/blob/BlobServerSSLTest.java @@ -18,12 +18,8 @@ package org.apache.flink.runtime.blob; -import org.apache.flink.configuration.AkkaOptions; -import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.JobManagerOptions; import org.apache.flink.configuration.SecurityOptions; -import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.util.TestLogger; import org.junit.Test; @@ -43,11 +39,6 @@ public class BlobServerSSLTest extends TestLogger { public void testFailedToInitWithTwoProtocolsSet() { final Configuration config = new Configuration(); - config.setString(JobManagerOptions.ADDRESS, "127.0.0.1"); - config.setString(TaskManagerOptions.HOST, "127.0.0.1"); - config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1); - config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true); config.setString(SecurityOptions.SSL_KEYSTORE, getClass().getResource("/local127.keystore").getPath()); @@ -59,7 +50,7 @@ public void testFailedToInitWithTwoProtocolsSet() { config.setString(SecurityOptions.SSL_TRUSTSTORE_PASSWORD, "password"); config.setString(SecurityOptions.SSL_ALGORITHMS, "TLSv1,TLSv1.1"); - try (final BlobServer blobServer = new BlobServer(config, new VoidBlobStore())) { + try (final BlobServer ignored = new BlobServer(config, new VoidBlobStore())) { fail(); } catch (Exception e) { findThrowable(e, IOException.class); @@ -71,10 +62,6 @@ public void testFailedToInitWithTwoProtocolsSet() { public void testFailedToInitWithInvalidSslKeystoreConfigured() { final Configuration config = new Configuration(); - config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1); - config.setString(AkkaOptions.ASK_TIMEOUT, "2 s"); - config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true); config.setString(SecurityOptions.SSL_KEYSTORE, "invalid.keystore"); config.setString(SecurityOptions.SSL_KEYSTORE_PASSWORD, "password"); @@ -82,7 +69,7 @@ public void testFailedToInitWithInvalidSslKeystoreConfigured() { config.setString(SecurityOptions.SSL_TRUSTSTORE, "invalid.keystore"); config.setString(SecurityOptions.SSL_TRUSTSTORE_PASSWORD, "password"); - try (final BlobServer blobServer = new BlobServer(config, new VoidBlobStore())) { + try (final BlobServer ignored = new BlobServer(config, new VoidBlobStore())) { fail(); } catch (Exception e) { findThrowable(e, IOException.class); @@ -94,13 +81,9 @@ public void testFailedToInitWithInvalidSslKeystoreConfigured() { public void testFailedToInitWithMissingMandatorySslConfiguration() { final Configuration config = new Configuration(); - config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1); - config.setString(AkkaOptions.ASK_TIMEOUT, "2 s"); - config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true); - try (final BlobServer blobServer = new BlobServer(config, new VoidBlobStore())) { + try (final BlobServer ignored = new BlobServer(config, new VoidBlobStore())) { fail(); } catch (Exception e) { findThrowable(e, IOException.class); From f9fbbd76a6de058178af18062b8f4116a1b76e1a Mon Sep 17 00:00:00 2001 From: ZILI CHEN Date: Wed, 9 Jan 2019 02:28:43 +0800 Subject: [PATCH 069/147] [FLINK-10700] [cluster managerment] Remove LegacyStandaloneClusterDescriptor (#6946) --- .../LegacyStandaloneClusterDescriptor.java | 76 ------------------- 1 file changed, 76 deletions(-) delete mode 100644 flink-clients/src/main/java/org/apache/flink/client/deployment/LegacyStandaloneClusterDescriptor.java diff --git a/flink-clients/src/main/java/org/apache/flink/client/deployment/LegacyStandaloneClusterDescriptor.java b/flink-clients/src/main/java/org/apache/flink/client/deployment/LegacyStandaloneClusterDescriptor.java deleted file mode 100644 index b44897054d6af4..00000000000000 --- a/flink-clients/src/main/java/org/apache/flink/client/deployment/LegacyStandaloneClusterDescriptor.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.client.deployment; - -import org.apache.flink.client.program.StandaloneClusterClient; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.JobManagerOptions; -import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.util.FlinkException; - -/** - * A deployment descriptor for an existing cluster. - */ -public class LegacyStandaloneClusterDescriptor implements ClusterDescriptor { - - private final Configuration config; - - public LegacyStandaloneClusterDescriptor(Configuration config) { - this.config = config; - } - - @Override - public String getClusterDescription() { - String host = config.getString(JobManagerOptions.ADDRESS, ""); - int port = config.getInteger(JobManagerOptions.PORT, -1); - return "Standalone cluster at " + host + ":" + port; - } - - @Override - public StandaloneClusterClient retrieve(StandaloneClusterId standaloneClusterId) throws ClusterRetrieveException { - try { - return new StandaloneClusterClient(config); - } catch (Exception e) { - throw new ClusterRetrieveException("Couldn't retrieve standalone cluster", e); - } - } - - @Override - public StandaloneClusterClient deploySessionCluster(ClusterSpecification clusterSpecification) { - throw new UnsupportedOperationException("Can't deploy a standalone cluster."); - } - - @Override - public StandaloneClusterClient deployJobCluster( - ClusterSpecification clusterSpecification, - JobGraph jobGraph, - boolean detached) { - throw new UnsupportedOperationException("Can't deploy a standalone per-job cluster."); - } - - @Override - public void killCluster(StandaloneClusterId clusterId) throws FlinkException { - throw new UnsupportedOperationException("Cannot terminate standalone clusters."); - } - - @Override - public void close() throws Exception { - // nothing to do - } -} From 9ab04daf142ed6c339aa7f8eb2fde5b59a2fa647 Mon Sep 17 00:00:00 2001 From: Bo WANG <42056379+eaglewatcherwb@users.noreply.github.com> Date: Wed, 9 Jan 2019 16:05:13 +0800 Subject: [PATCH 070/147] [FLINK-11232][rest] Fix subtask start-time field name --- .../runtime/rest/messages/JobVertexDetailsInfo.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobVertexDetailsInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobVertexDetailsInfo.java index df007f45c07590..3ad1e411d29c8d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobVertexDetailsInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobVertexDetailsInfo.java @@ -110,7 +110,8 @@ public static final class VertexTaskDetail { public static final String FIELD_NAME_STATUS = "status"; public static final String FIELD_NAME_ATTEMPT = "attempt"; public static final String FIELD_NAME_HOST = "host"; - public static final String FIELD_NAME_START_TIME = "start_time"; + public static final String FIELD_NAME_START_TIME = "start-time"; + public static final String FIELD_NAME_COMPATIBLE_START_TIME = "start_time"; public static final String FIELD_NAME_END_TIME = "end-time"; public static final String FIELD_NAME_DURATION = "duration"; public static final String FIELD_NAME_METRICS = "metrics"; @@ -130,6 +131,9 @@ public static final class VertexTaskDetail { @JsonProperty(FIELD_NAME_START_TIME) private final long startTime; + @JsonProperty(FIELD_NAME_COMPATIBLE_START_TIME) + private final long startTimeCompatible; + @JsonProperty(FIELD_NAME_END_TIME) private final long endTime; @@ -154,6 +158,7 @@ public VertexTaskDetail( this.attempt = attempt; this.host = checkNotNull(host); this.startTime = startTime; + this.startTimeCompatible = startTime; this.endTime = endTime; this.duration = duration; this.metrics = checkNotNull(metrics); @@ -175,6 +180,7 @@ public boolean equals(Object o) { attempt == that.attempt && Objects.equals(host, that.host) && startTime == that.startTime && + startTimeCompatible == that.startTimeCompatible && endTime == that.endTime && duration == that.duration && Objects.equals(metrics, that.metrics); @@ -182,7 +188,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(subtask, status, attempt, host, startTime, endTime, duration, metrics); + return Objects.hash(subtask, status, attempt, host, startTime, startTimeCompatible, endTime, duration, metrics); } } } From 9742ef7a10d89d27dce1a2fa3d5e91e4adb296ed Mon Sep 17 00:00:00 2001 From: Tony Wei Date: Wed, 9 Jan 2019 16:12:08 +0800 Subject: [PATCH 071/147] [FLINK-11251][metrics] Exclude variable values from logical scope of generic groups --- .../metrics/groups/AbstractMetricGroup.java | 2 +- .../groups/GenericValueMetricGroup.java | 2 +- .../metrics/groups/MetricGroupTest.java | 33 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroup.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroup.java index fb321303222c72..42e1cbbc0b71a3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroup.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroup.java @@ -172,7 +172,7 @@ String getLogicalScope(CharacterFilter filter, char delimiter, int reporterIndex } } - private String createLogicalScope(CharacterFilter filter, char delimiter) { + protected String createLogicalScope(CharacterFilter filter, char delimiter) { final String groupName = getGroupName(filter); return parent == null ? groupName diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/GenericValueMetricGroup.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/GenericValueMetricGroup.java index ef8e6e8bee8a2c..41e9a91d386787 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/GenericValueMetricGroup.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/GenericValueMetricGroup.java @@ -51,7 +51,7 @@ protected void putVariables(Map variables) { } @Override - public String getLogicalScope(CharacterFilter filter, char delimiter) { + protected String createLogicalScope(CharacterFilter filter, char delimiter) { return parent.getLogicalScope(filter, delimiter); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupTest.java index 71ae7f18ecb7e0..41d3fc9cb2bef2 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupTest.java @@ -19,6 +19,8 @@ package org.apache.flink.runtime.metrics.groups; import org.apache.flink.api.common.JobID; +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.CharacterFilter; import org.apache.flink.metrics.Gauge; import org.apache.flink.metrics.Metric; @@ -31,6 +33,7 @@ import org.apache.flink.runtime.metrics.dump.QueryScopeInfo; import org.apache.flink.runtime.metrics.scope.ScopeFormat; import org.apache.flink.runtime.metrics.util.DummyCharacterFilter; +import org.apache.flink.runtime.metrics.util.TestReporter; import org.apache.flink.util.AbstractID; import org.apache.flink.util.TestLogger; @@ -38,10 +41,13 @@ import org.junit.Before; import org.junit.Test; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -216,6 +222,33 @@ public void testNameCollisionAfterKeyValueGroup() { assertFalse("Value is present in logical scope.", logicalScope.contains(value)); } + /** + * Verifies that calling {@link AbstractMetricGroup#getLogicalScope(CharacterFilter, char, int)} on {@link GenericValueMetricGroup} + * should ignore value as well. + */ + @Test + public void testLogicalScopeShouldIgnoreValueGroupName() throws Exception { + Configuration config = new Configuration(); + config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter.class.getName()); + + MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(config)); + try { + GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root"); + + String key = "key"; + String value = "value"; + + MetricGroup group = root.addGroup(key, value); + + String logicalScope = ((AbstractMetricGroup) group) + .getLogicalScope(new DummyCharacterFilter(), registry.getDelimiter(), 0); + assertThat("Key is missing from logical scope.", logicalScope, containsString(key)); + assertThat("Value is present in logical scope.", logicalScope, not(containsString(value))); + } finally { + registry.shutdown().get(); + } + } + @Test public void closedGroupDoesNotRegisterMetrics() { GenericMetricGroup group = new GenericMetricGroup( From 5e4648715f97338bb8ee0c8cc790bdbecc5b4422 Mon Sep 17 00:00:00 2001 From: Dawid Wysakowicz Date: Mon, 7 Jan 2019 15:23:40 +0100 Subject: [PATCH 072/147] [hotfix][cep] Change contract of cep TimerContext#timestamp to never return null --- .../flink/cep/context/TimerContext.java | 6 ++-- .../PatternTimeoutFlatSelectAdapter.java | 3 +- .../adaptors/PatternTimeoutSelectAdapter.java | 3 +- .../flink/cep/operator/CepOperator.java | 11 +++--- .../CepProcessFunctionContextTest.java | 35 ++++++++++--------- 5 files changed, 28 insertions(+), 30 deletions(-) diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/context/TimerContext.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/context/TimerContext.java index 23367cdfc6f0f3..4f0eab2a348392 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/context/TimerContext.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/context/TimerContext.java @@ -32,10 +32,10 @@ public interface TimerContext { /** * Timestamp of the element currently being processed. * - *

This might be {@code null}, for example if the time characteristic of your program - * is set to {@link org.apache.flink.streaming.api.TimeCharacteristic#ProcessingTime}. + *

In case of {@link org.apache.flink.streaming.api.TimeCharacteristic#ProcessingTime} this will be set to the + * time when event entered the cep operator. */ - Long timestamp(); + long timestamp(); /** Returns the current processing time. */ long currentProcessingTime(); diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/adaptors/PatternTimeoutFlatSelectAdapter.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/adaptors/PatternTimeoutFlatSelectAdapter.java index ab9c97dde6665e..13d04bdfbff2d4 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/adaptors/PatternTimeoutFlatSelectAdapter.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/adaptors/PatternTimeoutFlatSelectAdapter.java @@ -78,8 +78,7 @@ public void processTimedOutMatch( Map> match, Context ctx) throws Exception { sideCollector.setCtx(ctx); - long timestamp = ctx.timestamp() != null ? ctx.timestamp() : ctx.currentProcessingTime(); - flatTimeoutFunction.timeout(match, timestamp, sideCollector); + flatTimeoutFunction.timeout(match, ctx.timestamp(), sideCollector); } /** diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/adaptors/PatternTimeoutSelectAdapter.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/adaptors/PatternTimeoutSelectAdapter.java index 29b0cf768fa30c..81999a8dd77cd4 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/adaptors/PatternTimeoutSelectAdapter.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/adaptors/PatternTimeoutSelectAdapter.java @@ -71,8 +71,7 @@ public void processTimedOutMatch( final Map> match, final Context ctx) throws Exception { - final long resultTimestamp = ctx.timestamp() != null ? ctx.timestamp() : ctx.currentProcessingTime(); - final T timedOutPatternResult = timeoutFunction.timeout(match, resultTimestamp); + final T timedOutPatternResult = timeoutFunction.timeout(match, ctx.timestamp()); ctx.output(timedOutPartialMatchesTag, timedOutPatternResult); } diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/CepOperator.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/CepOperator.java index 6df1fa020eed0a..262d89dd7338a7 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/CepOperator.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/CepOperator.java @@ -459,8 +459,9 @@ private void processTimedOutSequences(Collection>, L private void setTimestamp(long timestamp) { if (!isProcessingTime) { collector.setAbsoluteTimestamp(timestamp); - context.setTimestamp(timestamp); } + + context.setTimestamp(timestamp); } /** @@ -493,12 +494,8 @@ void setTimestamp(long timestamp) { } @Override - public Long timestamp() { - if (isProcessingTime) { - return null; - } else { - return timestamp; - } + public long timestamp() { + return timestamp; } @Override diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CepProcessFunctionContextTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CepProcessFunctionContextTest.java index 09477487880da3..baefe844c45f40 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CepProcessFunctionContextTest.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CepProcessFunctionContextTest.java @@ -45,8 +45,8 @@ */ public class CepProcessFunctionContextTest extends TestLogger { - private static final boolean PROCESSING_TIME = false; - private static final boolean EVENT_TIME = true; + private static final boolean PROCESSING_TIME = true; + private static final boolean EVENT_TIME = false; @Test public void testTimestampPassingInEventTime() throws Exception { @@ -56,7 +56,7 @@ public void testTimestampPassingInEventTime() throws Exception { createCepOperator( extractTimestampAndNames(1), new NFAForwardingFactory(), - PROCESSING_TIME))) { + EVENT_TIME))) { harness.open(); // events out of order to test if internal sorting does not mess up the timestamps @@ -81,15 +81,18 @@ public void testTimestampPassingInProcessingTime() throws Exception { createCepOperator( extractTimestampAndNames(1), new NFAForwardingFactory(), - EVENT_TIME))) { + PROCESSING_TIME))) { harness.open(); + harness.setProcessingTime(1); harness.processElement(event().withName("A").withTimestamp(5).asStreamRecord()); + harness.setProcessingTime(2); harness.processElement(event().withName("B").withTimestamp(3).asStreamRecord()); + harness.setProcessingTime(3); assertOutput(harness.getOutput()) - .nextElementEquals("(NO_TIMESTAMP):A") - .nextElementEquals("(NO_TIMESTAMP):B") + .nextElementEquals("1:A") + .nextElementEquals("2:B") .hasNoMoreElements(); } } @@ -102,7 +105,7 @@ public void testCurrentProcessingTimeInProcessingTime() throws Exception { createCepOperator( extractCurrentProcessingTimeAndNames(1), new NFAForwardingFactory(), - EVENT_TIME))) { + PROCESSING_TIME))) { harness.open(); harness.setProcessingTime(15); @@ -125,7 +128,7 @@ public void testCurrentProcessingTimeInEventTime() throws Exception { createCepOperator( extractCurrentProcessingTimeAndNames(1), new NFAForwardingFactory(), - PROCESSING_TIME))) { + EVENT_TIME))) { harness.open(); harness.setProcessingTime(10); @@ -150,7 +153,7 @@ public void testTimestampPassingForTimedOutInEventTime() throws Exception { createCepOperator( extractTimestampAndNames(2, timedOut), new NFATimingOutFactory(), - PROCESSING_TIME))) { + EVENT_TIME))) { harness.open(); // events out of order to test if internal sorting does not mess up the timestamps @@ -181,7 +184,7 @@ public void testTimestampPassingForTimedOutInProcessingTime() throws Exception { createCepOperator( extractTimestampAndNames(2, timedOut), new NFATimingOutFactory(), - EVENT_TIME))) { + PROCESSING_TIME))) { harness.open(); harness.setProcessingTime(3); @@ -192,11 +195,11 @@ public void testTimestampPassingForTimedOutInProcessingTime() throws Exception { harness.processElement(event().withName("B").withTimestamp(20).asStreamRecord()); assertOutput(harness.getOutput()) - .nextElementEquals("(NO_TIMESTAMP):A:C") + .nextElementEquals("5:A:C") .hasNoMoreElements(); assertOutput(harness.getSideOutput(timedOut)) - .nextElementEquals("(NO_TIMESTAMP):C") + .nextElementEquals("15:C") .hasNoMoreElements(); } } @@ -211,7 +214,7 @@ public void testCurrentProcessingTimeForTimedOutInEventTime() throws Exception { createCepOperator( extractCurrentProcessingTimeAndNames(2, sideOutputTag), new NFATimingOutFactory(), - PROCESSING_TIME))) { + EVENT_TIME))) { harness.open(); // events out of order to test if internal sorting does not mess up the timestamps @@ -243,7 +246,7 @@ public void testCurrentProcessingTimeForTimedOutInProcessingTime() throws Except createCepOperator( extractCurrentProcessingTimeAndNames(2, sideOutputTag), new NFATimingOutFactory(), - EVENT_TIME))) { + PROCESSING_TIME))) { harness.open(); harness.setProcessingTime(3); @@ -290,7 +293,7 @@ private CepOperator createCepOperator( */ private static PatternProcessFunction extractTimestampAndNames(int stateNumber) { return new AccessContextWithNames(stateNumber, - context -> context.timestamp() != null ? String.valueOf(context.timestamp()) : NO_TIMESTAMP); + context -> String.valueOf(context.timestamp())); } /** @@ -310,7 +313,7 @@ private static PatternProcessFunction extractTimestampAndNames( OutputTag timedOutTag) { return new AccessContextWithNamesWithTimedOut(stateNumber, timedOutTag, - context -> context.timestamp() != null ? String.valueOf(context.timestamp()) : NO_TIMESTAMP); + context -> String.valueOf(context.timestamp())); } /** From abe7ae23d5dc969b9034b206b8d492903c790b30 Mon Sep 17 00:00:00 2001 From: Dawid Wysakowicz Date: Wed, 12 Dec 2018 09:16:07 +0100 Subject: [PATCH 073/147] [hotfix][cep] Introduced nfa test harness --- .../flink/cep/nfa/AfterMatchSkipITCase.java | 110 +++++----- .../apache/flink/cep/nfa/GreedyITCase.java | 4 +- .../org/apache/flink/cep/nfa/GroupITCase.java | 8 +- .../cep/nfa/IterativeConditionsITCase.java | 4 +- .../org/apache/flink/cep/nfa/NFAITCase.java | 91 ++++----- .../flink/cep/nfa/NFAStateAccessTest.java | 41 +--- .../flink/cep/nfa/NFAStatusChangeITCase.java | 20 +- .../org/apache/flink/cep/nfa/NFATest.java | 148 ++++++-------- .../flink/cep/nfa/NotPatternITCase.java | 4 +- .../flink/cep/nfa/SameElementITCase.java | 11 +- .../flink/cep/nfa/TimesOrMoreITCase.java | 4 +- .../flink/cep/nfa/TimesRangeITCase.java | 4 +- .../flink/cep/nfa/UntilConditionITCase.java | 36 ++-- .../flink/cep/utils/NFATestHarness.java | 191 ++++++++++++++++++ .../cep/{nfa => utils}/NFATestUtilities.java | 60 +----- 15 files changed, 417 insertions(+), 319 deletions(-) create mode 100644 flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/NFATestHarness.java rename flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/{nfa => utils}/NFATestUtilities.java (57%) diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/AfterMatchSkipITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/AfterMatchSkipITCase.java index f6bf0a8f9f028c..89f3bf82842f62 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/AfterMatchSkipITCase.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/AfterMatchSkipITCase.java @@ -22,10 +22,10 @@ import org.apache.flink.cep.nfa.aftermatch.AfterMatchSkipStrategy; import org.apache.flink.cep.nfa.aftermatch.SkipPastLastStrategy; import org.apache.flink.cep.nfa.sharedbuffer.SharedBuffer; -import org.apache.flink.cep.nfa.sharedbuffer.SharedBufferAccessor; import org.apache.flink.cep.pattern.Pattern; import org.apache.flink.cep.pattern.conditions.IterativeCondition; import org.apache.flink.cep.pattern.conditions.SimpleCondition; +import org.apache.flink.cep.utils.NFATestHarness; import org.apache.flink.cep.utils.TestSharedBuffer; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.util.FlinkRuntimeException; @@ -40,9 +40,7 @@ import java.util.Collections; import java.util.List; -import static org.apache.flink.cep.nfa.NFATestUtilities.compareMaps; -import static org.apache.flink.cep.nfa.NFATestUtilities.feedNFA; -import static org.apache.flink.cep.utils.NFAUtils.compile; +import static org.apache.flink.cep.utils.NFATestUtilities.compareMaps; import static org.junit.Assert.assertThat; /** @@ -77,9 +75,9 @@ public boolean filter(Event value) throws Exception { } }).times(3); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.newArrayList( Lists.newArrayList(a1, a2, a3), @@ -141,9 +139,11 @@ public boolean filter(Event value) throws Exception { } }); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern) + .withAfterMatchSkipStrategy(skipStrategy) + .build(); - return feedNFA(streamEvents, nfa, skipStrategy); + return nfaTestHarness.feedRecords(streamEvents); } } @@ -198,9 +198,11 @@ public boolean filter(Event value) throws Exception { } }).oneOrMore(); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern) + .withAfterMatchSkipStrategy(skipStrategy) + .build(); - return feedNFA(streamEvents, nfa, skipStrategy); + return nfaTestHarness.feedRecords(streamEvents); } } @@ -231,9 +233,9 @@ public boolean filter(Event value) throws Exception { } }).times(3); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.newArrayList( Lists.newArrayList(a1, a2, a3), @@ -275,9 +277,9 @@ public boolean filter(Event value) throws Exception { } }).times(2); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.newArrayList( Lists.newArrayList(ab1, ab2, ab3, ab4), @@ -318,9 +320,9 @@ public boolean filter(Event value) throws Exception { return value.getName().contains("b"); } }).times(2); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.newArrayList( Lists.newArrayList(ab1, ab2, ab3, ab4), @@ -376,9 +378,9 @@ public boolean filter(Event value) throws Exception { return value.getName().contains("d"); } }); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Collections.singletonList( Lists.newArrayList(a1, b1, c1, d1) @@ -415,9 +417,9 @@ public boolean filter(Event value) throws Exception { } } ); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(a2, b2) @@ -459,9 +461,9 @@ public boolean filter(Event value) throws Exception { return value.getName().contains("c"); } }); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.newArrayList( Lists.newArrayList(ab1, c1), @@ -498,9 +500,9 @@ public boolean filter(Event value) throws Exception { return value.getName().contains("c"); } }); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.newArrayList( Lists.newArrayList(ab1, c1), @@ -543,9 +545,9 @@ public boolean filter(Event value) throws Exception { return value.getName().contains("b"); } }).oneOrMore().consecutive(); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.newArrayList( Lists.newArrayList(a1, b1), @@ -573,9 +575,9 @@ public boolean filter(Event value) throws Exception { } } ); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); //skip to first element of a match should throw exception if they are enabled, //this mode is used in MATCH RECOGNIZE which assumes that skipping to first element @@ -645,9 +647,11 @@ public boolean filter(Event value) throws Exception { return value.getName().contains("c"); } }); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern) + .withAfterMatchSkipStrategy(skipStrategy) + .build(); - return feedNFA(streamEvents, nfa, skipStrategy); + return nfaTestHarness.feedRecords(streamEvents); } } @@ -686,9 +690,9 @@ public boolean filter(Event value) throws Exception { return value.getName().contains("b"); } }).oneOrMore().consecutive(); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.newArrayList( Lists.newArrayList(a1, b1), @@ -728,9 +732,9 @@ public boolean filter(Event value) throws Exception { return value.getName().contains("b"); } }); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Collections.singletonList( Lists.newArrayList(a1, a2, a3, b1) @@ -768,9 +772,9 @@ public boolean filter(Event value) throws Exception { return value.getName().contains("b"); } }); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.newArrayList( Lists.newArrayList(a1, a2, a3, b1), @@ -809,9 +813,9 @@ public boolean filter(Event value) throws Exception { return value.getName().contains("b"); } }); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.newArrayList( Lists.newArrayList(a1, a2, a3, b1), @@ -851,9 +855,9 @@ public boolean filter(Event value) throws Exception { return value.getName().contains("b"); } }); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.newArrayList( Lists.newArrayList(a1, a2, a3, b1), @@ -913,9 +917,9 @@ public boolean filter(Event value) throws Exception { return value.getName().contains("d"); } }); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.newArrayList( Lists.newArrayList(a, b, c1, c2, c3, d), @@ -962,9 +966,9 @@ public boolean filter(Event value, Context ctx) throws Exception { ctx.getEventsForPattern("a").iterator().next().getPrice() == value.getPrice(); } }); - NFA nfa = compile(pattern, false); + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); - List> resultingPatterns = feedNFA(streamEvents, nfa, pattern.getAfterMatchSkipStrategy()); + List> resultingPatterns = nfaTestHarness.feedRecords(streamEvents); compareMaps(resultingPatterns, Lists.newArrayList( Lists.newArrayList(a1, c1, b2), @@ -991,22 +995,10 @@ public boolean filter(Event value) throws Exception { } }).times(2); - NFA nfa = compile(pattern, false); - SharedBuffer sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer()); - NFAState nfaState = nfa.createInitialNFAState(); - - for (StreamRecord inputEvent : inputEvents) { - try (SharedBufferAccessor sharedBufferAccessor = sharedBuffer.getAccessor()) { - nfa.advanceTime(sharedBufferAccessor, nfaState, inputEvent.getTimestamp()); - nfa.process( - sharedBufferAccessor, - nfaState, - inputEvent.getValue(), - inputEvent.getTimestamp(), - matchSkipStrategy); - } - } + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).withSharedBuffer(sharedBuffer).build(); + + nfaTestHarness.feedRecords(inputEvents); assertThat(sharedBuffer.isEmpty(), Matchers.is(true)); } diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/GreedyITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/GreedyITCase.java index 9e00130e715d0d..e33e59ae41d5a9 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/GreedyITCase.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/GreedyITCase.java @@ -31,8 +31,8 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.flink.cep.nfa.NFATestUtilities.compareMaps; -import static org.apache.flink.cep.nfa.NFATestUtilities.feedNFA; +import static org.apache.flink.cep.utils.NFATestUtilities.compareMaps; +import static org.apache.flink.cep.utils.NFATestUtilities.feedNFA; import static org.apache.flink.cep.utils.NFAUtils.compile; /** diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/GroupITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/GroupITCase.java index 93116ff44ee2c8..9387966625129b 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/GroupITCase.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/GroupITCase.java @@ -22,6 +22,7 @@ import org.apache.flink.cep.pattern.GroupPattern; import org.apache.flink.cep.pattern.Pattern; import org.apache.flink.cep.pattern.conditions.SimpleCondition; +import org.apache.flink.cep.utils.NFATestHarness; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.util.TestLogger; @@ -32,8 +33,8 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.flink.cep.nfa.NFATestUtilities.compareMaps; -import static org.apache.flink.cep.nfa.NFATestUtilities.feedNFA; +import static org.apache.flink.cep.utils.NFATestUtilities.compareMaps; +import static org.apache.flink.cep.utils.NFATestUtilities.feedNFA; import static org.apache.flink.cep.utils.NFAUtils.compile; import static org.junit.Assert.assertEquals; @@ -1077,7 +1078,8 @@ public boolean filter(Event value) throws Exception { NFAState nfaState = nfa.createInitialNFAState(); - final List> resultingPatterns = feedNFA(inputEvents, nfa, nfaState); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); + final List> resultingPatterns = nfaTestHarness.feedRecords(inputEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(c, a1, b1, d), diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/IterativeConditionsITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/IterativeConditionsITCase.java index 90123793c7eeed..ce28ddd5a54af5 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/IterativeConditionsITCase.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/IterativeConditionsITCase.java @@ -33,8 +33,8 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.flink.cep.nfa.NFATestUtilities.compareMaps; -import static org.apache.flink.cep.nfa.NFATestUtilities.feedNFA; +import static org.apache.flink.cep.utils.NFATestUtilities.compareMaps; +import static org.apache.flink.cep.utils.NFATestUtilities.feedNFA; import static org.apache.flink.cep.utils.NFAUtils.compile; /** diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java index 2391d547fcb113..62d76db9ea7fff 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java @@ -21,11 +21,13 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.cep.Event; import org.apache.flink.cep.SubEvent; +import org.apache.flink.cep.nfa.aftermatch.AfterMatchSkipStrategy; import org.apache.flink.cep.nfa.sharedbuffer.SharedBuffer; import org.apache.flink.cep.nfa.sharedbuffer.SharedBufferAccessor; import org.apache.flink.cep.pattern.Pattern; import org.apache.flink.cep.pattern.Quantifier; import org.apache.flink.cep.pattern.conditions.SimpleCondition; +import org.apache.flink.cep.utils.NFATestHarness; import org.apache.flink.cep.utils.TestSharedBuffer; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; @@ -48,8 +50,8 @@ import java.util.Map; import java.util.Set; -import static org.apache.flink.cep.nfa.NFATestUtilities.compareMaps; -import static org.apache.flink.cep.nfa.NFATestUtilities.feedNFA; +import static org.apache.flink.cep.utils.NFATestUtilities.compareMaps; +import static org.apache.flink.cep.utils.NFATestUtilities.feedNFA; import static org.apache.flink.cep.utils.NFAUtils.compile; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.anyLong; @@ -414,7 +416,11 @@ public boolean filter(Event value) throws Exception { Collection>, Long>> timeoutPatterns = nfa.advanceTime(sharedBufferAccessor, nfaState, event.getTimestamp()); Collection>> matchedPatterns = - nfa.process(sharedBufferAccessor, nfaState, event.getValue(), event.getTimestamp()); + nfa.process(sharedBufferAccessor, + nfaState, + event.getValue(), + event.getTimestamp(), + AfterMatchSkipStrategy.noSkip()); resultingPatterns.addAll(matchedPatterns); resultingTimeoutPatterns.addAll(timeoutPatterns); @@ -2342,11 +2348,13 @@ public boolean filter(Event value) throws Exception { NFAState nfaState = nfa.createInitialNFAState(); - nfa.process(sharedBufferAccessor, nfaState, startEvent, 1); - nfa.process(sharedBufferAccessor, nfaState, middleEvent1, 2); - nfa.process(sharedBufferAccessor, nfaState, middleEvent2, 3); - nfa.process(sharedBufferAccessor, nfaState, middleEvent3, 4); - nfa.process(sharedBufferAccessor, nfaState, end1, 6); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); + + nfaTestHarness.feedRecord(new StreamRecord<>(startEvent, 1)); + nfaTestHarness.feedRecord(new StreamRecord<>(middleEvent1, 2)); + nfaTestHarness.feedRecord(new StreamRecord<>(middleEvent2, 3)); + nfaTestHarness.feedRecord(new StreamRecord<>(middleEvent3, 4)); + nfaTestHarness.feedRecord(new StreamRecord<>(end1, 6)); //pruning element nfa.advanceTime(sharedBufferAccessor, nfaState, 10); @@ -2387,10 +2395,11 @@ public boolean filter(Event value) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - nfa.process(sharedBufferAccessor, nfaState, startEvent, 1); - nfa.process(sharedBufferAccessor, nfaState, middleEvent, 5); - nfa.process(sharedBufferAccessor, nfaState, end1, 6); + nfaTestHarness.feedRecord(new StreamRecord<>(startEvent, 1)); + nfaTestHarness.feedRecord(new StreamRecord<>(middleEvent, 5)); + nfaTestHarness.feedRecord(new StreamRecord<>(end1, 6)); //pruning element nfa.advanceTime(sharedBufferAccessor, nfaState, 10); @@ -2432,11 +2441,12 @@ public boolean filter(Event value) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - nfa.process(sharedBufferAccessor, nfaState, startEvent, 1); - nfa.process(sharedBufferAccessor, nfaState, middleEvent1, 3); - nfa.process(sharedBufferAccessor, nfaState, middleEvent2, 4); - nfa.process(sharedBufferAccessor, nfaState, end1, 6); + nfaTestHarness.consumeRecord(new StreamRecord<>(startEvent, 1)); + nfaTestHarness.consumeRecord(new StreamRecord<>(middleEvent1, 3)); + nfaTestHarness.consumeRecord(new StreamRecord<>(middleEvent2, 4)); + nfaTestHarness.consumeRecord(new StreamRecord<>(end1, 6)); //pruning element nfa.advanceTime(sharedBufferAccessor, nfaState, 10); @@ -2478,11 +2488,12 @@ public boolean filter(Event value) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - nfa.process(sharedBufferAccessor, nfaState, startEvent, 1); - nfa.process(sharedBufferAccessor, nfaState, middleEvent1, 3); - nfa.process(sharedBufferAccessor, nfaState, middleEvent2, 4); - nfa.process(sharedBufferAccessor, nfaState, end1, 6); + nfaTestHarness.consumeRecord(new StreamRecord<>(startEvent, 1)); + nfaTestHarness.consumeRecord(new StreamRecord<>(middleEvent1, 3)); + nfaTestHarness.consumeRecord(new StreamRecord<>(middleEvent2, 4)); + nfaTestHarness.consumeRecord(new StreamRecord<>(end1, 6)); //pruning element nfa.advanceTime(sharedBufferAccessor, nfaState, 10); @@ -2734,25 +2745,12 @@ public boolean filter(Event s) throws Exception { } }).times(3).consecutive(); - NFA nfa = compile(pattern, false); - - List>> resultingPatterns = new ArrayList<>(); - - NFAState nfaState = nfa.createInitialNFAState(); - - for (StreamRecord inputEvent : inputEvents) { - Collection>> patterns = nfa.process( - sharedBufferAccessor, - nfaState, - inputEvent.getValue(), - inputEvent.getTimestamp()); - - resultingPatterns.addAll(patterns); - } + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); + Collection>> resultingPatterns = nfaTestHarness.consumeRecords(inputEvents); Assert.assertEquals(1L, resultingPatterns.size()); - Map> match = resultingPatterns.get(0); + Map> match = resultingPatterns.iterator().next(); Assert.assertArrayEquals( match.get("start").toArray(), Lists.newArrayList(startEvent1, startEvent2, startEvent3, startEvent4).toArray()); @@ -2809,25 +2807,12 @@ public boolean filter(Event s) throws Exception { } }); - NFA nfa = compile(pattern, false); - - List>> resultingPatterns = new ArrayList<>(); - - NFAState nfaState = nfa.createInitialNFAState(); - - for (StreamRecord inputEvent : inputEvents) { - Collection>> patterns = nfa.process( - sharedBufferAccessor, - nfaState, - inputEvent.getValue(), - inputEvent.getTimestamp()); - - resultingPatterns.addAll(patterns); - } + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build(); + Collection>> resultingPatterns = nfaTestHarness.consumeRecords(inputEvents); Assert.assertEquals(1L, resultingPatterns.size()); - Map> match = resultingPatterns.get(0); + Map> match = resultingPatterns.iterator().next(); List expectedOrder = Lists.newArrayList("a", "b", "aa", "bb", "ab"); List resultOrder = new ArrayList<>(); @@ -2847,8 +2832,8 @@ public void testSharedBufferClearing() throws Exception { try (SharedBufferAccessor accessor = Mockito.spy(sharedBuffer.getAccessor())) { NFA nfa = compile(pattern, false); - nfa.process(accessor, nfa.createInitialNFAState(), a, 1); - nfa.process(accessor, nfa.createInitialNFAState(), b, 2); + nfa.process(accessor, nfa.createInitialNFAState(), a, 1, AfterMatchSkipStrategy.noSkip()); + nfa.process(accessor, nfa.createInitialNFAState(), b, 2, AfterMatchSkipStrategy.noSkip()); Mockito.verify(accessor, Mockito.never()).advanceTime(anyLong()); nfa.advanceTime(accessor, nfa.createInitialNFAState(), 2); Mockito.verify(accessor, Mockito.times(1)).advanceTime(2); diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAStateAccessTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAStateAccessTest.java index 227fe4e9507f03..1be5f4fb3fd9ab 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAStateAccessTest.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAStateAccessTest.java @@ -20,10 +20,10 @@ import org.apache.flink.cep.Event; import org.apache.flink.cep.SubEvent; -import org.apache.flink.cep.nfa.sharedbuffer.SharedBufferAccessor; import org.apache.flink.cep.pattern.Pattern; import org.apache.flink.cep.pattern.conditions.IterativeCondition; import org.apache.flink.cep.pattern.conditions.SimpleCondition; +import org.apache.flink.cep.utils.NFATestHarness; import org.apache.flink.cep.utils.TestSharedBuffer; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; @@ -32,7 +32,6 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.flink.cep.utils.NFAUtils.compile; import static org.junit.Assert.assertEquals; /** @@ -99,22 +98,13 @@ public boolean filter(Event value) throws Exception { } }); - NFA nfa = compile(pattern, false); - TestSharedBuffer sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer()); - for (StreamRecord inputEvent : inputEvents) { - try (SharedBufferAccessor accessor = sharedBuffer.getAccessor()) { - nfa.process( - accessor, - nfa.createInitialNFAState(), - inputEvent.getValue(), - inputEvent.getTimestamp()); - } - } + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).withSharedBuffer(sharedBuffer).build(); + nfaTestHarness.consumeRecords(inputEvents); - assertEquals(2, sharedBuffer.getStateReads()); - assertEquals(3, sharedBuffer.getStateWrites()); - assertEquals(5, sharedBuffer.getStateAccesses()); + assertEquals(58, sharedBuffer.getStateReads()); + assertEquals(33, sharedBuffer.getStateWrites()); + assertEquals(91, sharedBuffer.getStateAccesses()); } @Test @@ -182,21 +172,12 @@ public boolean filter(Event value) throws Exception { } }); - NFA nfa = compile(pattern, false); - TestSharedBuffer sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer()); - for (StreamRecord inputEvent : inputEvents) { - try (SharedBufferAccessor accessor = sharedBuffer.getAccessor()) { - nfa.process( - accessor, - nfa.createInitialNFAState(), - inputEvent.getValue(), - inputEvent.getTimestamp()); - } - } + NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).withSharedBuffer(sharedBuffer).build(); + nfaTestHarness.consumeRecords(inputEvents); - assertEquals(8, sharedBuffer.getStateReads()); - assertEquals(12, sharedBuffer.getStateWrites()); - assertEquals(20, sharedBuffer.getStateAccesses()); + assertEquals(90, sharedBuffer.getStateReads()); + assertEquals(31, sharedBuffer.getStateWrites()); + assertEquals(121, sharedBuffer.getStateAccesses()); } } diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAStatusChangeITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAStatusChangeITCase.java index 8cbfba1ec6c3cd..d0a78531346455 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAStatusChangeITCase.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAStatusChangeITCase.java @@ -20,6 +20,7 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.cep.Event; +import org.apache.flink.cep.nfa.aftermatch.AfterMatchSkipStrategy; import org.apache.flink.cep.nfa.sharedbuffer.SharedBuffer; import org.apache.flink.cep.nfa.sharedbuffer.SharedBufferAccessor; import org.apache.flink.cep.pattern.Pattern; @@ -47,6 +48,7 @@ public class NFAStatusChangeITCase { private SharedBuffer sharedBuffer; private SharedBufferAccessor sharedBufferAccessor; + private AfterMatchSkipStrategy skipStrategy = AfterMatchSkipStrategy.noSkip(); @Before public void init() { @@ -95,32 +97,32 @@ public boolean filter(Event value, Context ctx) throws Exception { NFAState nfaState = nfa.createInitialNFAState(); - nfa.process(sharedBufferAccessor, nfaState, new Event(1, "b", 1.0), 1L); + nfa.process(sharedBufferAccessor, nfaState, new Event(1, "b", 1.0), 1L, skipStrategy); assertFalse("NFA status should not change as the event does not match the take condition of the 'start' state", nfaState.isStateChanged()); nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(2, "a", 1.0), 2L); + nfa.process(sharedBufferAccessor, nfaState, new Event(2, "a", 1.0), 2L, skipStrategy); assertTrue("NFA status should change as the event matches the take condition of the 'start' state", nfaState.isStateChanged()); // the status of the queue of ComputationStatus changed, // more than one ComputationStatus is generated by the event from some ComputationStatus nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(3, "f", 1.0), 3L); + nfa.process(sharedBufferAccessor, nfaState, new Event(3, "f", 1.0), 3L, skipStrategy); assertTrue("NFA status should change as the event matches the ignore condition and proceed condition of the 'middle:1' state", nfaState.isStateChanged()); // both the queue of ComputationStatus and eventSharedBuffer have not changed nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(4, "f", 1.0), 4L); + nfa.process(sharedBufferAccessor, nfaState, new Event(4, "f", 1.0), 4L, skipStrategy); assertFalse("NFA status should not change as the event only matches the ignore condition of the 'middle:2' state and the target state is still 'middle:2'", nfaState.isStateChanged()); // both the queue of ComputationStatus and eventSharedBuffer have changed nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(5, "b", 1.0), 5L); + nfa.process(sharedBufferAccessor, nfaState, new Event(5, "b", 1.0), 5L, skipStrategy); assertTrue("NFA status should change as the event matches the take condition of 'middle:2' state", nfaState.isStateChanged()); // both the queue of ComputationStatus and eventSharedBuffer have changed nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(6, "d", 1.0), 6L); + nfa.process(sharedBufferAccessor, nfaState, new Event(6, "d", 1.0), 6L, skipStrategy); assertTrue("NFA status should change as the event matches the take condition of 'middle2' state", nfaState.isStateChanged()); // both the queue of ComputationStatus and eventSharedBuffer have not changed @@ -164,10 +166,10 @@ public boolean filter(Event value, Context ctx) throws Exception { NFAState nfaState = nfa.createInitialNFAState(); nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(6, "start", 1.0), 6L); + nfa.process(sharedBufferAccessor, nfaState, new Event(6, "start", 1.0), 6L, skipStrategy); nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(6, "a", 1.0), 7L); + nfa.process(sharedBufferAccessor, nfaState, new Event(6, "a", 1.0), 7L, skipStrategy); assertTrue(nfaState.isStateChanged()); } @@ -193,7 +195,7 @@ public boolean filter(Event value, Context ctx) throws Exception { nfaState.resetStateChanged(); nfa.advanceTime(sharedBufferAccessor, nfaState, 6L); - nfa.process(sharedBufferAccessor, nfaState, new Event(6, "start", 1.0), 6L); + nfa.process(sharedBufferAccessor, nfaState, new Event(6, "start", 1.0), 6L, skipStrategy); nfaState.resetStateChanged(); nfa.advanceTime(sharedBufferAccessor, nfaState, 17L); diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFATest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFATest.java index e5f4c9ee9f6060..638b8b53bc6f6e 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFATest.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFATest.java @@ -19,13 +19,11 @@ package org.apache.flink.cep.nfa; import org.apache.flink.cep.Event; -import org.apache.flink.cep.nfa.sharedbuffer.SharedBuffer; -import org.apache.flink.cep.nfa.sharedbuffer.SharedBufferAccessor; import org.apache.flink.cep.pattern.Pattern; import org.apache.flink.cep.pattern.conditions.BooleanConditions; import org.apache.flink.cep.pattern.conditions.IterativeCondition; import org.apache.flink.cep.pattern.conditions.SimpleCondition; -import org.apache.flink.cep.utils.TestSharedBuffer; +import org.apache.flink.cep.utils.NFATestHarness; import org.apache.flink.core.memory.DataInputViewStreamWrapper; import org.apache.flink.core.memory.DataOutputViewStreamWrapper; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; @@ -39,10 +37,8 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import static org.apache.flink.cep.utils.NFAUtils.compile; import static org.junit.Assert.assertEquals; @@ -91,9 +87,7 @@ public boolean filter(Event value) throws Exception { states.add(endState); states.add(endingState); - NFA nfa = new NFA<>(states, 0, false); - - Set>> expectedPatterns = new HashSet<>(); + List>> expectedPatterns = new ArrayList<>(); Map> firstPattern = new HashMap<>(); firstPattern.put("start", Collections.singletonList(new Event(1, "start", 1.0))); @@ -106,14 +100,16 @@ public boolean filter(Event value) throws Exception { expectedPatterns.add(firstPattern); expectedPatterns.add(secondPattern); - Collection>> actualPatterns = runNFA(nfa, nfa.createInitialNFAState(), streamEvents); + NFA nfa = new NFA<>(states, 0, false); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).build(); + + Collection>> actualPatterns = nfaTestHarness.consumeRecords(streamEvents); assertEquals(expectedPatterns, actualPatterns); } @Test public void testTimeoutWindowPruning() throws Exception { - NFA nfa = createStartEndNFA(); List> streamEvents = new ArrayList<>(); streamEvents.add(new StreamRecord<>(new Event(1, "start", 1.0), 1L)); @@ -121,7 +117,7 @@ public void testTimeoutWindowPruning() throws Exception { streamEvents.add(new StreamRecord<>(new Event(3, "start", 3.0), 3L)); streamEvents.add(new StreamRecord<>(new Event(4, "end", 4.0), 4L)); - Set>> expectedPatterns = new HashSet<>(); + List>> expectedPatterns = new ArrayList<>(); Map> secondPattern = new HashMap<>(); secondPattern.put("start", Collections.singletonList(new Event(3, "start", 3.0))); @@ -129,7 +125,10 @@ public void testTimeoutWindowPruning() throws Exception { expectedPatterns.add(secondPattern); - Collection>> actualPatterns = runNFA(nfa, nfa.createInitialNFAState(), streamEvents); + NFA nfa = createStartEndNFA(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).build(); + + Collection>> actualPatterns = nfaTestHarness.consumeRecords(streamEvents); assertEquals(expectedPatterns, actualPatterns); } @@ -140,15 +139,17 @@ public void testTimeoutWindowPruning() throws Exception { */ @Test public void testWindowBorders() throws Exception { - NFA nfa = createStartEndNFA(); List> streamEvents = new ArrayList<>(); streamEvents.add(new StreamRecord<>(new Event(1, "start", 1.0), 1L)); streamEvents.add(new StreamRecord<>(new Event(2, "end", 2.0), 3L)); - Set>> expectedPatterns = Collections.emptySet(); + List>> expectedPatterns = Collections.emptyList(); + + NFA nfa = createStartEndNFA(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).build(); - Collection>> actualPatterns = runNFA(nfa, nfa.createInitialNFAState(), streamEvents); + Collection>> actualPatterns = nfaTestHarness.consumeRecords(streamEvents); assertEquals(expectedPatterns, actualPatterns); } @@ -159,7 +160,6 @@ public void testWindowBorders() throws Exception { */ @Test public void testTimeoutWindowPruningWindowBorders() throws Exception { - NFA nfa = createStartEndNFA(); List> streamEvents = new ArrayList<>(); streamEvents.add(new StreamRecord<>(new Event(1, "start", 1.0), 1L)); @@ -167,7 +167,7 @@ public void testTimeoutWindowPruningWindowBorders() throws Exception { streamEvents.add(new StreamRecord<>(new Event(3, "foobar", 3.0), 3L)); streamEvents.add(new StreamRecord<>(new Event(4, "end", 4.0), 3L)); - Set>> expectedPatterns = new HashSet<>(); + List>> expectedPatterns = new ArrayList<>(); Map> secondPattern = new HashMap<>(); secondPattern.put("start", Collections.singletonList(new Event(2, "start", 2.0))); @@ -175,30 +175,12 @@ public void testTimeoutWindowPruningWindowBorders() throws Exception { expectedPatterns.add(secondPattern); - Collection>> actualPatterns = runNFA(nfa, nfa.createInitialNFAState(), streamEvents); + NFA nfa = createStartEndNFA(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).build(); - assertEquals(expectedPatterns, actualPatterns); - } + Collection>> actualPatterns = nfaTestHarness.consumeRecords(streamEvents); - public Collection>> runNFA( - NFA nfa, NFAState nfaState, List> inputs) throws Exception { - Set>> actualPatterns = new HashSet<>(); - - SharedBuffer sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer()); - try (SharedBufferAccessor sharedBufferAccessor = sharedBuffer.getAccessor()) { - for (StreamRecord streamEvent : inputs) { - nfa.advanceTime(sharedBufferAccessor, nfaState, streamEvent.getTimestamp()); - Collection>> matchedPatterns = nfa.process( - sharedBufferAccessor, - nfaState, - streamEvent.getValue(), - streamEvent.getTimestamp()); - - actualPatterns.addAll(matchedPatterns); - } - } - - return actualPatterns; + assertEquals(expectedPatterns, actualPatterns); } @Test @@ -289,51 +271,49 @@ public boolean filter(Event value) throws Exception { patterns.add(pattern2); patterns.add(pattern3); - SharedBuffer sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer()); - try (SharedBufferAccessor sharedBufferAccessor = sharedBuffer.getAccessor()) { - - for (Pattern p : patterns) { - NFA nfa = compile(p, false); - - Event a = new Event(40, "a", 1.0); - Event b = new Event(41, "b", 2.0); - Event c = new Event(42, "c", 3.0); - Event b1 = new Event(41, "b", 3.0); - Event b2 = new Event(41, "b", 4.0); - Event b3 = new Event(41, "b", 5.0); - Event d = new Event(43, "d", 4.0); - - NFAState nfaState = nfa.createInitialNFAState(); - - nfa.process(sharedBufferAccessor, nfaState, a, 1); - nfa.process(sharedBufferAccessor, nfaState, b, 2); - nfa.process(sharedBufferAccessor, nfaState, c, 3); - nfa.process(sharedBufferAccessor, nfaState, b1, 4); - nfa.process(sharedBufferAccessor, nfaState, b2, 5); - nfa.process(sharedBufferAccessor, nfaState, b3, 6); - nfa.process(sharedBufferAccessor, nfaState, d, 7); - nfa.process(sharedBufferAccessor, nfaState, a, 8); - - NFAStateSerializer serializer = NFAStateSerializer.INSTANCE; - - //serialize - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - serializer.serialize(nfaState, new DataOutputViewStreamWrapper(baos)); - baos.close(); - - // copy - ByteArrayInputStream in = new ByteArrayInputStream(baos.toByteArray()); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - serializer.duplicate().copy(new DataInputViewStreamWrapper(in), new DataOutputViewStreamWrapper(out)); - in.close(); - out.close(); - - // deserialize - ByteArrayInputStream bais = new ByteArrayInputStream(out.toByteArray()); - NFAState copy = serializer.duplicate().deserialize(new DataInputViewStreamWrapper(bais)); - bais.close(); - assertEquals(nfaState, copy); - } + for (Pattern p : patterns) { + NFA nfa = compile(p, false); + + Event a = new Event(40, "a", 1.0); + Event b = new Event(41, "b", 2.0); + Event c = new Event(42, "c", 3.0); + Event b1 = new Event(41, "b", 3.0); + Event b2 = new Event(41, "b", 4.0); + Event b3 = new Event(41, "b", 5.0); + Event d = new Event(43, "d", 4.0); + + NFAState nfaState = nfa.createInitialNFAState(); + + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); + + nfaTestHarness.consumeRecord(new StreamRecord<>(a, 1)); + nfaTestHarness.consumeRecord(new StreamRecord<>(b, 2)); + nfaTestHarness.consumeRecord(new StreamRecord<>(c, 3)); + nfaTestHarness.consumeRecord(new StreamRecord<>(b1, 4)); + nfaTestHarness.consumeRecord(new StreamRecord<>(b2, 5)); + nfaTestHarness.consumeRecord(new StreamRecord<>(b3, 6)); + nfaTestHarness.consumeRecord(new StreamRecord<>(d, 7)); + nfaTestHarness.consumeRecord(new StreamRecord<>(a, 8)); + + NFAStateSerializer serializer = NFAStateSerializer.INSTANCE; + + //serialize + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + serializer.serialize(nfaState, new DataOutputViewStreamWrapper(baos)); + baos.close(); + + // copy + ByteArrayInputStream in = new ByteArrayInputStream(baos.toByteArray()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + serializer.duplicate().copy(new DataInputViewStreamWrapper(in), new DataOutputViewStreamWrapper(out)); + in.close(); + out.close(); + + // deserialize + ByteArrayInputStream bais = new ByteArrayInputStream(out.toByteArray()); + NFAState copy = serializer.duplicate().deserialize(new DataInputViewStreamWrapper(bais)); + bais.close(); + assertEquals(nfaState, copy); } } diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NotPatternITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NotPatternITCase.java index 11a84841d96b36..6f3321805e892f 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NotPatternITCase.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NotPatternITCase.java @@ -31,8 +31,8 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.flink.cep.nfa.NFATestUtilities.compareMaps; -import static org.apache.flink.cep.nfa.NFATestUtilities.feedNFA; +import static org.apache.flink.cep.utils.NFATestUtilities.compareMaps; +import static org.apache.flink.cep.utils.NFATestUtilities.feedNFA; import static org.apache.flink.cep.utils.NFAUtils.compile; import static org.junit.Assert.assertEquals; diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/SameElementITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/SameElementITCase.java index b7f11779774509..3d277af8ace086 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/SameElementITCase.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/SameElementITCase.java @@ -22,6 +22,7 @@ import org.apache.flink.cep.pattern.Pattern; import org.apache.flink.cep.pattern.conditions.IterativeCondition; import org.apache.flink.cep.pattern.conditions.SimpleCondition; +import org.apache.flink.cep.utils.NFATestHarness; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.util.TestLogger; @@ -33,8 +34,8 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.flink.cep.nfa.NFATestUtilities.compareMaps; -import static org.apache.flink.cep.nfa.NFATestUtilities.feedNFA; +import static org.apache.flink.cep.utils.NFATestUtilities.compareMaps; +import static org.apache.flink.cep.utils.NFATestUtilities.feedNFA; import static org.apache.flink.cep.utils.NFAUtils.compile; import static org.junit.Assert.assertEquals; @@ -141,8 +142,9 @@ public boolean filter(Event value) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - List> resultingPatterns = feedNFA(inputEvents, nfa, nfaState); + List> resultingPatterns = nfaTestHarness.feedRecords(inputEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(a1, b1, c1, d) )); @@ -186,8 +188,9 @@ public boolean filter(Event value, Context ctx) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - List> resultingPatterns = feedNFA(inputEvents, nfa, nfaState); + List> resultingPatterns = nfaTestHarness.feedRecords(inputEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(a1, d1, d2, d3), Lists.newArrayList(a1, d1, d2), diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/TimesOrMoreITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/TimesOrMoreITCase.java index 049c84b3ac35cc..93d8c5683d2b9a 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/TimesOrMoreITCase.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/TimesOrMoreITCase.java @@ -31,8 +31,8 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.flink.cep.nfa.NFATestUtilities.compareMaps; -import static org.apache.flink.cep.nfa.NFATestUtilities.feedNFA; +import static org.apache.flink.cep.utils.NFATestUtilities.compareMaps; +import static org.apache.flink.cep.utils.NFATestUtilities.feedNFA; import static org.apache.flink.cep.utils.NFAUtils.compile; /** diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/TimesRangeITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/TimesRangeITCase.java index 203a1c2fe597f5..dd85d87ae4a41c 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/TimesRangeITCase.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/TimesRangeITCase.java @@ -31,8 +31,8 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.flink.cep.nfa.NFATestUtilities.compareMaps; -import static org.apache.flink.cep.nfa.NFATestUtilities.feedNFA; +import static org.apache.flink.cep.utils.NFATestUtilities.compareMaps; +import static org.apache.flink.cep.utils.NFATestUtilities.feedNFA; import static org.apache.flink.cep.utils.NFAUtils.compile; /** diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/UntilConditionITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/UntilConditionITCase.java index be0f28b52b89b4..39b23b4926a3d2 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/UntilConditionITCase.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/UntilConditionITCase.java @@ -22,6 +22,7 @@ import org.apache.flink.cep.pattern.Pattern; import org.apache.flink.cep.pattern.conditions.IterativeCondition; import org.apache.flink.cep.pattern.conditions.SimpleCondition; +import org.apache.flink.cep.utils.NFATestHarness; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; @@ -31,8 +32,8 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.flink.cep.nfa.NFATestUtilities.compareMaps; -import static org.apache.flink.cep.nfa.NFATestUtilities.feedNFA; +import static org.apache.flink.cep.utils.NFATestUtilities.compareMaps; +import static org.apache.flink.cep.utils.NFATestUtilities.feedNFA; import static org.apache.flink.cep.utils.NFAUtils.compile; import static org.junit.Assert.assertEquals; @@ -92,8 +93,9 @@ public boolean filter(Event value) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - final List> resultingPatterns = feedNFA(inputEvents, nfa, nfaState); + final List> resultingPatterns = nfaTestHarness.feedRecords(inputEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(startEvent, middleEvent1, middleEvent2, breaking), @@ -142,8 +144,9 @@ public boolean filter(Event value) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - final List> resultingPatterns = feedNFA(inputEvents, nfa, nfaState); + final List> resultingPatterns = nfaTestHarness.feedRecords(inputEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(startEvent, middleEvent1, middleEvent2, middleEvent3, breaking), @@ -193,8 +196,9 @@ public boolean filter(Event value) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - final List> resultingPatterns = feedNFA(inputEvents, nfa, nfaState); + final List> resultingPatterns = nfaTestHarness.feedRecords(inputEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(startEvent, middleEvent1, middleEvent2, breaking), @@ -244,8 +248,9 @@ public boolean filter(Event value) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - final List> resultingPatterns = feedNFA(inputEvents, nfa, nfaState); + final List> resultingPatterns = nfaTestHarness.feedRecords(inputEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(startEvent, middleEvent1, breaking) @@ -292,8 +297,9 @@ public boolean filter(Event value) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - final List> resultingPatterns = feedNFA(inputEvents, nfa, nfaState); + final List> resultingPatterns = nfaTestHarness.feedRecords(inputEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(startEvent, middleEvent1, middleEvent2, breaking), @@ -342,8 +348,9 @@ public boolean filter(Event value) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - final List> resultingPatterns = feedNFA(inputEvents, nfa, nfaState); + final List> resultingPatterns = nfaTestHarness.feedRecords(inputEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(startEvent, middleEvent1, middleEvent2, middleEvent3, breaking), @@ -394,8 +401,9 @@ public boolean filter(Event value) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - final List> resultingPatterns = feedNFA(inputEvents, nfa, nfaState); + final List> resultingPatterns = nfaTestHarness.feedRecords(inputEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(startEvent, middleEvent1, middleEvent2, breaking), @@ -525,8 +533,9 @@ public boolean filter(Event value) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - final List> resultingPatterns = feedNFA(inputEvents, nfa, nfaState); + final List> resultingPatterns = nfaTestHarness.feedRecords(inputEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(startEvent, middleEvent1, middleEvent2, middleEvent3), @@ -579,8 +588,9 @@ public boolean filter(Event value, Context ctx) throws Exception { NFA nfa = compile(pattern, false); NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - final List> resultingPatterns = feedNFA(inputEvents, nfa, nfaState); + final List> resultingPatterns = nfaTestHarness.feedRecords(inputEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(startEvent, middleEvent1, middleEvent2, middleEvent3), @@ -631,10 +641,10 @@ public boolean filter(Event value, Context ctx) throws Exception { }); NFA nfa = compile(pattern, false); - NFAState nfaState = nfa.createInitialNFAState(); + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build(); - final List> resultingPatterns = feedNFA(inputEvents, nfa, nfaState); + final List> resultingPatterns = nfaTestHarness.feedRecords(inputEvents); compareMaps(resultingPatterns, Lists.>newArrayList( Lists.newArrayList(startEvent, middleEvent1, middleEvent2, middleEvent3), diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/NFATestHarness.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/NFATestHarness.java new file mode 100644 index 00000000000000..dd080fb4f63a7f --- /dev/null +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/NFATestHarness.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cep.utils; + +import org.apache.flink.cep.Event; +import org.apache.flink.cep.nfa.NFA; +import org.apache.flink.cep.nfa.NFAState; +import org.apache.flink.cep.nfa.aftermatch.AfterMatchSkipStrategy; +import org.apache.flink.cep.nfa.sharedbuffer.SharedBuffer; +import org.apache.flink.cep.nfa.sharedbuffer.SharedBufferAccessor; +import org.apache.flink.cep.pattern.Pattern; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * Test harness for setting up {@link NFA}. + */ +public final class NFATestHarness { + + private final SharedBuffer sharedBuffer; + private final NFA nfa; + private final NFAState nfaState; + private final AfterMatchSkipStrategy afterMatchSkipStrategy; + + private NFATestHarness( + SharedBuffer sharedBuffer, + NFA nfa, + NFAState nfaState, + AfterMatchSkipStrategy afterMatchSkipStrategy) { + this.sharedBuffer = sharedBuffer; + this.nfa = nfa; + this.nfaState = nfaState; + this.afterMatchSkipStrategy = afterMatchSkipStrategy; + } + + /** + * Constructs a test harness starting from a given {@link Pattern}. + */ + public static NFATestHarnessBuilderPattern forPattern(Pattern pattern) { + return new NFATestHarnessBuilderPattern(pattern); + } + + /** + * Constructs a test harness starting from a given {@link NFA}. + */ + public static NFATestHarnessBuilderNFA forNFA(NFA nfa) { + return new NFATestHarnessBuilderNFA(nfa); + } + + public List> feedRecords(List> inputEvents) throws Exception { + List> resultingPatterns = new ArrayList<>(); + for (StreamRecord inputEvent : inputEvents) { + resultingPatterns.addAll(feedRecord(inputEvent)); + } + return resultingPatterns; + } + + public List> feedRecord(StreamRecord inputEvent) throws Exception { + List> resultingPatterns = new ArrayList<>(); + Collection>> matches = consumeRecord(inputEvent); + for (Map> p : matches) { + List res = new ArrayList<>(); + for (List le : p.values()) { + res.addAll(le); + } + resultingPatterns.add(res); + } + return resultingPatterns; + } + + public Collection>> consumeRecords(Collection> inputEvents) throws Exception { + List>> resultingPatterns = new ArrayList<>(); + for (StreamRecord inputEvent : inputEvents) { + resultingPatterns.addAll(consumeRecord(inputEvent)); + } + + return resultingPatterns; + } + + public Collection>> consumeRecord(StreamRecord inputEvent) throws Exception { + try (SharedBufferAccessor sharedBufferAccessor = sharedBuffer.getAccessor()) { + nfa.advanceTime(sharedBufferAccessor, nfaState, inputEvent.getTimestamp()); + return nfa.process( + sharedBufferAccessor, + nfaState, + inputEvent.getValue(), + inputEvent.getTimestamp(), + afterMatchSkipStrategy); + } + } + + /** + * Builder for {@link NFATestHarness} that encapsulates {@link Pattern}. + */ + public static class NFATestHarnessBuilderPattern extends NFATestHarnessBuilderBase { + + private final Pattern pattern; + private boolean timeoutHandling = false; + + NFATestHarnessBuilderPattern(Pattern pattern) { + super(pattern.getAfterMatchSkipStrategy()); + this.pattern = pattern; + } + + public NFATestHarnessBuilderBase withTimeoutHandling() { + this.timeoutHandling = true; + return this; + } + + @Override + public NFATestHarness build() { + NFA nfa = NFAUtils.compile(pattern, timeoutHandling); + return new NFATestHarness( + sharedBuffer, + nfa, + nfa.createInitialNFAState(), + afterMatchSkipStrategy); + } + } + + /** + * Builder for {@link NFATestHarness} that encapsulates {@link NFA}. + */ + public static class NFATestHarnessBuilderNFA extends NFATestHarnessBuilderBase { + + private final NFA nfa; + private NFAState nfaState; + + NFATestHarnessBuilderNFA(NFA nfa) { + super(AfterMatchSkipStrategy.noSkip()); + this.nfa = nfa; + this.nfaState = nfa.createInitialNFAState(); + } + + public NFATestHarnessBuilderBase withNFAState(NFAState nfaState) { + this.nfaState = nfaState; + return this; + } + + @Override + public NFATestHarness build() { + return new NFATestHarness(sharedBuffer, nfa, nfaState, afterMatchSkipStrategy); + } + } + + /** + * Common builder, which can be used independent if we start with {@link Pattern} or {@link NFA}. + * Enables to provide custom services like {@link SharedBuffer} etc. + */ + public abstract static class NFATestHarnessBuilderBase { + + SharedBuffer sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer()); + AfterMatchSkipStrategy afterMatchSkipStrategy; + + NFATestHarnessBuilderBase(AfterMatchSkipStrategy skipStrategy) { + this.afterMatchSkipStrategy = skipStrategy; + } + + public NFATestHarnessBuilderBase withSharedBuffer(SharedBuffer sharedBuffer) { + this.sharedBuffer = sharedBuffer; + return this; + } + + public NFATestHarnessBuilderBase withAfterMatchSkipStrategy(AfterMatchSkipStrategy afterMatchSkipStrategy) { + this.afterMatchSkipStrategy = afterMatchSkipStrategy; + return this; + } + + public abstract NFATestHarness build(); + } +} diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFATestUtilities.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/NFATestUtilities.java similarity index 57% rename from flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFATestUtilities.java rename to flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/NFATestUtilities.java index 91e490ea6e9417..ac4579830c7b50 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFATestUtilities.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/NFATestUtilities.java @@ -16,77 +16,29 @@ * limitations under the License. */ -package org.apache.flink.cep.nfa; +package org.apache.flink.cep.utils; import org.apache.flink.cep.Event; -import org.apache.flink.cep.nfa.aftermatch.AfterMatchSkipStrategy; -import org.apache.flink.cep.nfa.sharedbuffer.SharedBuffer; -import org.apache.flink.cep.nfa.sharedbuffer.SharedBufferAccessor; -import org.apache.flink.cep.utils.TestSharedBuffer; +import org.apache.flink.cep.nfa.NFA; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.junit.Assert; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Map; /** * Base method for IT tests of {@link NFA}. It provides utility methods. */ public class NFATestUtilities { - public static List> feedNFA( - List> inputEvents, - NFA nfa) throws Exception { - return feedNFA(inputEvents, nfa, nfa.createInitialNFAState(), AfterMatchSkipStrategy.noSkip()); - } - - public static List> feedNFA( - List> inputEvents, - NFA nfa, - NFAState nfaState) throws Exception { - return feedNFA(inputEvents, nfa, nfaState, AfterMatchSkipStrategy.noSkip()); - } - - public static List> feedNFA( - List> inputEvents, - NFA nfa, - AfterMatchSkipStrategy afterMatchSkipStrategy) throws Exception { - return feedNFA(inputEvents, nfa, nfa.createInitialNFAState(), afterMatchSkipStrategy); - } - + @Deprecated public static List> feedNFA( List> inputEvents, - NFA nfa, - NFAState nfaState, - AfterMatchSkipStrategy afterMatchSkipStrategy) throws Exception { - List> resultingPatterns = new ArrayList<>(); - - SharedBuffer sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer()); - - for (StreamRecord inputEvent : inputEvents) { - try (SharedBufferAccessor sharedBufferAccessor = sharedBuffer.getAccessor()) { - nfa.advanceTime(sharedBufferAccessor, nfaState, inputEvent.getTimestamp()); - Collection>> patterns = nfa.process( - sharedBufferAccessor, - nfaState, - inputEvent.getValue(), - inputEvent.getTimestamp(), - afterMatchSkipStrategy); - for (Map> p: patterns) { - List res = new ArrayList<>(); - for (List le: p.values()) { - res.addAll(le); - } - resultingPatterns.add(res); - } - } - } - return resultingPatterns; + NFA nfa) throws Exception { + NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).build(); + return nfaTestHarness.feedRecords(inputEvents); } public static void compareMaps(List> actual, List> expected) { From 60f8826de75bdb1294ca4cd26b2af155de62c0df Mon Sep 17 00:00:00 2001 From: Dawid Wysakowicz Date: Wed, 12 Dec 2018 09:17:40 +0100 Subject: [PATCH 074/147] [FLINK-10596][cep] Added access to timer service in IterativeCondition --- ...reamScalaJavaAPIInteroperabilityTest.scala | 2 +- .../cep/functions/PatternProcessFunction.java | 2 +- .../java/org/apache/flink/cep/nfa/NFA.java | 84 +++++++++-------- .../flink/cep/operator/CepOperator.java | 20 +++- .../conditions/IterativeCondition.java | 5 +- .../cep/{context => time}/TimerContext.java | 6 +- .../apache/flink/cep/time/TimerService.java | 35 +++++++ .../org/apache/flink/cep/nfa/NFAITCase.java | 13 ++- ...NFAIterativeConditionTimerContextTest.java | 91 +++++++++++++++++++ .../flink/cep/nfa/NFAStatusChangeITCase.java | 21 +++-- .../flink/cep/operator/CEPOperatorTest.java | 35 +++++++ .../flink/cep/utils/NFATestHarness.java | 35 +++++-- .../flink/cep/utils/TestTimerService.java | 41 +++++++++ 13 files changed, 318 insertions(+), 72 deletions(-) rename flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/{context => time}/TimerContext.java (91%) create mode 100644 flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/TimerService.java create mode 100644 flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAIterativeConditionTimerContextTest.java create mode 100644 flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/TestTimerService.java diff --git a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/PatternStreamScalaJavaAPIInteroperabilityTest.scala b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/PatternStreamScalaJavaAPIInteroperabilityTest.scala index 5f6055d421bec7..80d96aca175d53 100644 --- a/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/PatternStreamScalaJavaAPIInteroperabilityTest.scala +++ b/flink-libraries/flink-cep-scala/src/test/scala/org/apache/flink/cep/scala/PatternStreamScalaJavaAPIInteroperabilityTest.scala @@ -150,7 +150,7 @@ class PatternStreamScalaJavaAPIInteroperabilityTest extends TestLogger { outputs.getOrElseUpdate(outputTag, ListBuffer.empty).append(value) } - override def timestamp(): lang.Long = null + override def timestamp(): Long = 0 override def currentProcessingTime(): Long = System.currentTimeMillis() } diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/PatternProcessFunction.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/PatternProcessFunction.java index b392501a683721..b8e6821c7b4f67 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/PatternProcessFunction.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/PatternProcessFunction.java @@ -20,7 +20,7 @@ import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.api.common.functions.AbstractRichFunction; -import org.apache.flink.cep.context.TimerContext; +import org.apache.flink.cep.time.TimerContext; import org.apache.flink.util.Collector; import org.apache.flink.util.OutputTag; diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java index 48bb587a5d4aaf..869a641cbee565 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java @@ -38,6 +38,7 @@ import org.apache.flink.cep.nfa.sharedbuffer.SharedBuffer; import org.apache.flink.cep.nfa.sharedbuffer.SharedBufferAccessor; import org.apache.flink.cep.pattern.conditions.IterativeCondition; +import org.apache.flink.cep.time.TimerService; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -199,31 +200,6 @@ public void close() throws Exception { } } - /** - * Processes the next input event. If some of the computations reach a final state then the - * resulting event sequences are returned. If computations time out and timeout handling is - * activated, then the timed out event patterns are returned. - * - *

If computations reach a stop state, the path forward is discarded and currently constructed path is returned - * with the element that resulted in the stop state. - * - * @param sharedBufferAccessor the accessor to SharedBuffer object that we need to work upon while processing - * @param nfaState The NFAState object that we need to affect while processing - * @param event The current event to be processed or null if only pruning shall be done - * @param timestamp The timestamp of the current event - * @return Tuple of the collection of matched patterns (e.g. the result of computations which have - * reached a final state) and the collection of timed out patterns (if timeout handling is - * activated) - * @throws Exception Thrown if the system cannot access the state. - */ - public Collection>> process( - final SharedBufferAccessor sharedBufferAccessor, - final NFAState nfaState, - final T event, - final long timestamp) throws Exception { - return process(sharedBufferAccessor, nfaState, event, timestamp, AfterMatchSkipStrategy.noSkip()); - } - /** * Processes the next input event. If some of the computations reach a final state then the * resulting event sequences are returned. If computations time out and timeout handling is @@ -237,6 +213,7 @@ public Collection>> process( * @param event The current event to be processed or null if only pruning shall be done * @param timestamp The timestamp of the current event * @param afterMatchSkipStrategy The skip strategy to use after per match + * @param timerService gives access to processing time and time characteristic, needed for condition evaluation * @return Tuple of the collection of matched patterns (e.g. the result of computations which have * reached a final state) and the collection of timed out patterns (if timeout handling is * activated) @@ -247,9 +224,10 @@ public Collection>> process( final NFAState nfaState, final T event, final long timestamp, - final AfterMatchSkipStrategy afterMatchSkipStrategy) throws Exception { + final AfterMatchSkipStrategy afterMatchSkipStrategy, + final TimerService timerService) throws Exception { try (EventWrapper eventWrapper = new EventWrapper(event, timestamp, sharedBufferAccessor)) { - return doProcess(sharedBufferAccessor, nfaState, eventWrapper, afterMatchSkipStrategy); + return doProcess(sharedBufferAccessor, nfaState, eventWrapper, afterMatchSkipStrategy, timerService); } } @@ -305,7 +283,8 @@ private Collection>> doProcess( final SharedBufferAccessor sharedBufferAccessor, final NFAState nfaState, final EventWrapper event, - final AfterMatchSkipStrategy afterMatchSkipStrategy) throws Exception { + final AfterMatchSkipStrategy afterMatchSkipStrategy, + final TimerService timerService) throws Exception { final PriorityQueue newPartialMatches = new PriorityQueue<>(NFAState.COMPUTATION_STATE_COMPARATOR); final PriorityQueue potentialMatches = new PriorityQueue<>(NFAState.COMPUTATION_STATE_COMPARATOR); @@ -316,7 +295,7 @@ private Collection>> doProcess( sharedBufferAccessor, computationState, event, - event.getTimestamp()); + timerService); if (newComputationStates.size() != 1) { nfaState.setStateChanged(); @@ -555,7 +534,7 @@ public void close() throws Exception { * @param sharedBufferAccessor The accessor to shared buffer that we need to change * @param computationState Current computation state * @param event Current event which is processed - * @param timestamp Timestamp of the current event + * @param timerService timer service which provides access to time related features * @return Collection of computation states which result from the current one * @throws Exception Thrown if the system cannot access the state. */ @@ -563,9 +542,13 @@ private Collection computeNextStates( final SharedBufferAccessor sharedBufferAccessor, final ComputationState computationState, final EventWrapper event, - final long timestamp) throws Exception { + final TimerService timerService) throws Exception { - final ConditionContext context = new ConditionContext<>(this, sharedBufferAccessor, computationState); + final ConditionContext context = new ConditionContext( + sharedBufferAccessor, + computationState, + timerService, + event.getTimestamp()); final OutgoingEdges outgoingEdges = createDecisionGraph(context, computationState, event.getEvent()); @@ -628,7 +611,7 @@ private Collection computeNextStates( final long startTimestamp; final EventId startEventId; if (isStartState(computationState)) { - startTimestamp = timestamp; + startTimestamp = event.getTimestamp(); startEventId = event.getEventId(); } else { startTimestamp = computationState.getStartTimestamp(); @@ -694,7 +677,7 @@ private void addComputationState( } private State findFinalStateAfterProceed( - ConditionContext context, + ConditionContext context, State state, T event) { final Stack> statesToCheck = new Stack<>(); @@ -725,7 +708,7 @@ private int calculateIncreasingSelfState(int ignoreBranches, int takeBranches) { } private OutgoingEdges createDecisionGraph( - ConditionContext context, + ConditionContext context, ComputationState computationState, T event) { State state = getState(computationState); @@ -765,7 +748,7 @@ private OutgoingEdges createDecisionGraph( } private boolean checkFilterCondition( - ConditionContext context, + ConditionContext context, IterativeCondition condition, T event) throws Exception { return condition == null || condition.filter(event, context); @@ -804,7 +787,11 @@ private Map> extractCurrentMatches( /** * The context used when evaluating this computation state. */ - private static class ConditionContext implements IterativeCondition.Context { + private class ConditionContext implements IterativeCondition.Context { + + private final TimerService timerService; + + private final long eventTimestamp; /** The current computation state. */ private ComputationState computationState; @@ -816,17 +803,17 @@ private static class ConditionContext implements IterativeCondition.Context> matchedEvents; - private NFA nfa; - private SharedBufferAccessor sharedBufferAccessor; ConditionContext( - final NFA nfa, final SharedBufferAccessor sharedBufferAccessor, - final ComputationState computationState) { + final ComputationState computationState, + final TimerService timerService, + final long eventTimestamp) { this.computationState = computationState; - this.nfa = nfa; this.sharedBufferAccessor = sharedBufferAccessor; + this.timerService = timerService; + this.eventTimestamp = eventTimestamp; } @Override @@ -837,7 +824,8 @@ public Iterable getEventsForPattern(final String key) throws Exception { // this is to avoid any overheads when using a simple, non-iterative condition. if (matchedEvents == null) { - this.matchedEvents = sharedBufferAccessor.materializeMatch(nfa.extractCurrentMatches(sharedBufferAccessor, + this.matchedEvents = sharedBufferAccessor.materializeMatch(extractCurrentMatches( + sharedBufferAccessor, computationState)); } @@ -851,6 +839,16 @@ public Iterator iterator() { } }; } + + @Override + public long timestamp() { + return eventTimestamp; + } + + @Override + public long currentProcessingTime() { + return timerService.currentProcessingTime(); + } } //////////////////// DEPRECATED/MIGRATION UTILS diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/CepOperator.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/CepOperator.java index 262d89dd7338a7..357fbc78b10c2e 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/CepOperator.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/operator/CepOperator.java @@ -40,6 +40,7 @@ import org.apache.flink.cep.nfa.compiler.NFACompiler; import org.apache.flink.cep.nfa.sharedbuffer.SharedBuffer; import org.apache.flink.cep.nfa.sharedbuffer.SharedBufferAccessor; +import org.apache.flink.cep.time.TimerService; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.state.KeyedStateFunction; import org.apache.flink.runtime.state.StateInitializationContext; @@ -130,6 +131,9 @@ public class CepOperator /** Wrapped RuntimeContext that limits the underlying context features. */ private transient CepRuntimeContext cepRuntimeContext; + /** Thin context passed to NFA that gives access to time related characteristics. */ + private transient TimerService cepTimerService; + public CepOperator( final TypeSerializer inputSerializer, final boolean isProcessingTime, @@ -216,6 +220,7 @@ public void open() throws Exception { context = new ContextFunctionImpl(); collector = new TimestampedCollector<>(output); + cepTimerService = new TimerServiceImpl(); } @Override @@ -415,7 +420,7 @@ private PriorityQueue getSortedTimestamps() throws Exception { private void processEvent(NFAState nfaState, IN event, long timestamp) throws Exception { try (SharedBufferAccessor sharedBufferAccessor = partialMatches.getAccessor()) { Collection>> patterns = - nfa.process(sharedBufferAccessor, nfaState, event, timestamp, afterMatchSkipStrategy); + nfa.process(sharedBufferAccessor, nfaState, event, timestamp, afterMatchSkipStrategy, cepTimerService); processMatchedSequences(patterns, timestamp); } } @@ -464,6 +469,19 @@ private void setTimestamp(long timestamp) { context.setTimestamp(timestamp); } + /** + * Gives {@link NFA} access to {@link InternalTimerService} and tells if {@link CepOperator} works in + * processing time. Should be instantiated once per operator. + */ + private class TimerServiceImpl implements TimerService { + + @Override + public long currentProcessingTime() { + return timerService.currentProcessingTime(); + } + + } + /** * Implementation of {@link PatternProcessFunction.Context}. Design to be instantiated once per operator. * It serves three methods: diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/IterativeCondition.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/IterativeCondition.java index f716ec40f73d07..786cb0834b7b1e 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/IterativeCondition.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/IterativeCondition.java @@ -18,7 +18,9 @@ package org.apache.flink.cep.pattern.conditions; +import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.api.common.functions.Function; +import org.apache.flink.cep.time.TimerContext; import java.io.Serializable; @@ -59,6 +61,7 @@ * the elements that belong to the pattern among the elements stored by the NFA. The cost of this operation can vary, * so when implementing your condition, try to minimize the times the method is called. */ +@PublicEvolving public abstract class IterativeCondition implements Function, Serializable { private static final long serialVersionUID = 7067817235759351255L; @@ -84,7 +87,7 @@ public abstract class IterativeCondition implements Function, Serializable { /** * The context used when evaluating the {@link IterativeCondition condition}. */ - public interface Context { + public interface Context extends TimerContext { /** * @return An {@link Iterable} over the already accepted elements diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/context/TimerContext.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/TimerContext.java similarity index 91% rename from flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/context/TimerContext.java rename to flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/TimerContext.java index 4f0eab2a348392..9b5db520399814 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/context/TimerContext.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/TimerContext.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.cep.context; +package org.apache.flink.cep.time; import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.cep.functions.PatternProcessFunction; @@ -32,8 +32,8 @@ public interface TimerContext { /** * Timestamp of the element currently being processed. * - *

In case of {@link org.apache.flink.streaming.api.TimeCharacteristic#ProcessingTime} this will be set to the - * time when event entered the cep operator. + *

In case of {@link org.apache.flink.streaming.api.TimeCharacteristic#ProcessingTime} this means the + * time when the event entered the cep operator. */ long timestamp(); diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/TimerService.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/TimerService.java new file mode 100644 index 00000000000000..464e267f0fd41f --- /dev/null +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/TimerService.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cep.time; + +import org.apache.flink.annotation.Internal; + +/** + * Enables to provide time characteristic to {@link org.apache.flink.cep.nfa.NFA} for use in + * {@link org.apache.flink.cep.pattern.conditions.IterativeCondition}. + */ +@Internal +public interface TimerService { + + /** + * Current processing time as returned from {@link org.apache.flink.streaming.api.TimerService}. + */ + long currentProcessingTime(); + +} diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java index 62d76db9ea7fff..4b1f6278e3175a 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java @@ -29,6 +29,7 @@ import org.apache.flink.cep.pattern.conditions.SimpleCondition; import org.apache.flink.cep.utils.NFATestHarness; import org.apache.flink.cep.utils.TestSharedBuffer; +import org.apache.flink.cep.utils.TestTimerService; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.util.TestLogger; @@ -420,7 +421,8 @@ public boolean filter(Event value) throws Exception { nfaState, event.getValue(), event.getTimestamp(), - AfterMatchSkipStrategy.noSkip()); + AfterMatchSkipStrategy.noSkip(), + new TestTimerService()); resultingPatterns.addAll(matchedPatterns); resultingTimeoutPatterns.addAll(timeoutPatterns); @@ -2830,10 +2832,13 @@ public void testSharedBufferClearing() throws Exception { Event a = new Event(40, "a", 1.0); Event b = new Event(41, "b", 2.0); + NFA nfa = compile(pattern, false); + TestTimerService timerService = new TestTimerService(); try (SharedBufferAccessor accessor = Mockito.spy(sharedBuffer.getAccessor())) { - NFA nfa = compile(pattern, false); - nfa.process(accessor, nfa.createInitialNFAState(), a, 1, AfterMatchSkipStrategy.noSkip()); - nfa.process(accessor, nfa.createInitialNFAState(), b, 2, AfterMatchSkipStrategy.noSkip()); + nfa.process(accessor, nfa.createInitialNFAState(), a, 1, AfterMatchSkipStrategy.noSkip(), + timerService); + nfa.process(accessor, nfa.createInitialNFAState(), b, 2, AfterMatchSkipStrategy.noSkip(), + timerService); Mockito.verify(accessor, Mockito.never()).advanceTime(anyLong()); nfa.advanceTime(accessor, nfa.createInitialNFAState(), 2); Mockito.verify(accessor, Mockito.times(1)).advanceTime(2); diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAIterativeConditionTimerContextTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAIterativeConditionTimerContextTest.java new file mode 100644 index 00000000000000..3f9b26ad340081 --- /dev/null +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAIterativeConditionTimerContextTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cep.nfa; + +import org.apache.flink.cep.Event; +import org.apache.flink.cep.pattern.Pattern; +import org.apache.flink.cep.pattern.conditions.IterativeCondition; +import org.apache.flink.cep.utils.NFATestHarness; +import org.apache.flink.cep.utils.TestTimerService; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.util.TestLogger; + +import org.junit.Test; + +import java.util.Collections; +import java.util.List; + +import static org.apache.flink.cep.utils.EventBuilder.event; +import static org.apache.flink.cep.utils.NFATestHarness.forPattern; +import static org.apache.flink.cep.utils.NFATestUtilities.compareMaps; + +/** + * Tests for accesing time properties from {@link IterativeCondition}. + */ +public class NFAIterativeConditionTimerContextTest extends TestLogger { + + @Test + public void testEventTimestamp() throws Exception { + final Event event = event().withId(1).build(); + final long timestamp = 3; + + final Pattern pattern = Pattern.begin("start").where(new IterativeCondition() { + @Override + public boolean filter(Event value, Context ctx) throws Exception { + return ctx.timestamp() == timestamp; + } + }); + + final NFATestHarness testHarness = forPattern(pattern).build(); + + final List> resultingPattern = testHarness.feedRecord(new StreamRecord<>(event, timestamp)); + + compareMaps(resultingPattern, Collections.singletonList( + Collections.singletonList(event) + )); + } + + @Test + public void testCurrentProcessingTime() throws Exception { + final Event event1 = event().withId(1).build(); + final Event event2 = event().withId(2).build(); + + final Pattern pattern = Pattern.begin("start").where(new IterativeCondition() { + @Override + public boolean filter(Event value, Context ctx) throws Exception { + return ctx.currentProcessingTime() == 3; + } + }); + + final TestTimerService cepTimerService = new TestTimerService(); + final NFATestHarness testHarness = forPattern(pattern) + .withTimerService(cepTimerService) + .build(); + + cepTimerService.setCurrentProcessingTime(1); + final List> resultingPatterns1 = testHarness.feedRecord(new StreamRecord<>(event1, 7)); + cepTimerService.setCurrentProcessingTime(3); + final List> resultingPatterns2 = testHarness.feedRecord(new StreamRecord<>(event2, 8)); + + compareMaps(resultingPatterns1, Collections.emptyList()); + compareMaps(resultingPatterns2, Collections.singletonList( + Collections.singletonList(event2) + )); + } +} diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAStatusChangeITCase.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAStatusChangeITCase.java index d0a78531346455..c5e668223e6651 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAStatusChangeITCase.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAStatusChangeITCase.java @@ -26,7 +26,9 @@ import org.apache.flink.cep.pattern.Pattern; import org.apache.flink.cep.pattern.conditions.IterativeCondition; import org.apache.flink.cep.pattern.conditions.SimpleCondition; +import org.apache.flink.cep.time.TimerService; import org.apache.flink.cep.utils.TestSharedBuffer; +import org.apache.flink.cep.utils.TestTimerService; import org.apache.flink.streaming.api.windowing.time.Time; import org.junit.After; @@ -49,6 +51,7 @@ public class NFAStatusChangeITCase { private SharedBuffer sharedBuffer; private SharedBufferAccessor sharedBufferAccessor; private AfterMatchSkipStrategy skipStrategy = AfterMatchSkipStrategy.noSkip(); + private TimerService timerService = new TestTimerService(); @Before public void init() { @@ -97,32 +100,32 @@ public boolean filter(Event value, Context ctx) throws Exception { NFAState nfaState = nfa.createInitialNFAState(); - nfa.process(sharedBufferAccessor, nfaState, new Event(1, "b", 1.0), 1L, skipStrategy); + nfa.process(sharedBufferAccessor, nfaState, new Event(1, "b", 1.0), 1L, skipStrategy, timerService); assertFalse("NFA status should not change as the event does not match the take condition of the 'start' state", nfaState.isStateChanged()); nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(2, "a", 1.0), 2L, skipStrategy); + nfa.process(sharedBufferAccessor, nfaState, new Event(2, "a", 1.0), 2L, skipStrategy, timerService); assertTrue("NFA status should change as the event matches the take condition of the 'start' state", nfaState.isStateChanged()); // the status of the queue of ComputationStatus changed, // more than one ComputationStatus is generated by the event from some ComputationStatus nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(3, "f", 1.0), 3L, skipStrategy); + nfa.process(sharedBufferAccessor, nfaState, new Event(3, "f", 1.0), 3L, skipStrategy, timerService); assertTrue("NFA status should change as the event matches the ignore condition and proceed condition of the 'middle:1' state", nfaState.isStateChanged()); // both the queue of ComputationStatus and eventSharedBuffer have not changed nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(4, "f", 1.0), 4L, skipStrategy); + nfa.process(sharedBufferAccessor, nfaState, new Event(4, "f", 1.0), 4L, skipStrategy, timerService); assertFalse("NFA status should not change as the event only matches the ignore condition of the 'middle:2' state and the target state is still 'middle:2'", nfaState.isStateChanged()); // both the queue of ComputationStatus and eventSharedBuffer have changed nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(5, "b", 1.0), 5L, skipStrategy); + nfa.process(sharedBufferAccessor, nfaState, new Event(5, "b", 1.0), 5L, skipStrategy, timerService); assertTrue("NFA status should change as the event matches the take condition of 'middle:2' state", nfaState.isStateChanged()); // both the queue of ComputationStatus and eventSharedBuffer have changed nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(6, "d", 1.0), 6L, skipStrategy); + nfa.process(sharedBufferAccessor, nfaState, new Event(6, "d", 1.0), 6L, skipStrategy, timerService); assertTrue("NFA status should change as the event matches the take condition of 'middle2' state", nfaState.isStateChanged()); // both the queue of ComputationStatus and eventSharedBuffer have not changed @@ -166,10 +169,10 @@ public boolean filter(Event value, Context ctx) throws Exception { NFAState nfaState = nfa.createInitialNFAState(); nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(6, "start", 1.0), 6L, skipStrategy); + nfa.process(sharedBufferAccessor, nfaState, new Event(6, "start", 1.0), 6L, skipStrategy, timerService); nfaState.resetStateChanged(); - nfa.process(sharedBufferAccessor, nfaState, new Event(6, "a", 1.0), 7L, skipStrategy); + nfa.process(sharedBufferAccessor, nfaState, new Event(6, "a", 1.0), 7L, skipStrategy, timerService); assertTrue(nfaState.isStateChanged()); } @@ -195,7 +198,7 @@ public boolean filter(Event value, Context ctx) throws Exception { nfaState.resetStateChanged(); nfa.advanceTime(sharedBufferAccessor, nfaState, 6L); - nfa.process(sharedBufferAccessor, nfaState, new Event(6, "start", 1.0), 6L, skipStrategy); + nfa.process(sharedBufferAccessor, nfaState, new Event(6, "start", 1.0), 6L, skipStrategy, timerService); nfaState.resetStateChanged(); nfa.advanceTime(sharedBufferAccessor, nfaState, 17L); diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPOperatorTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPOperatorTest.java index 2e4b41d6a1cc44..f496dc83a4f11d 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPOperatorTest.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPOperatorTest.java @@ -30,10 +30,14 @@ import org.apache.flink.cep.functions.PatternProcessFunction; import org.apache.flink.cep.functions.TimedOutPartialMatchHandler; import org.apache.flink.cep.nfa.NFA; +import org.apache.flink.cep.nfa.NFAState; +import org.apache.flink.cep.nfa.aftermatch.AfterMatchSkipStrategy; import org.apache.flink.cep.nfa.compiler.NFACompiler; +import org.apache.flink.cep.nfa.sharedbuffer.SharedBufferAccessor; import org.apache.flink.cep.pattern.Pattern; import org.apache.flink.cep.pattern.conditions.IterativeCondition; import org.apache.flink.cep.pattern.conditions.SimpleCondition; +import org.apache.flink.cep.time.TimerService; import org.apache.flink.contrib.streaming.state.RocksDBStateBackend; import org.apache.flink.mock.Whitebox; import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; @@ -64,10 +68,16 @@ import java.util.Map; import java.util.Queue; +import static org.apache.flink.cep.utils.CepOperatorBuilder.createOperatorForNFA; +import static org.apache.flink.cep.utils.EventBuilder.event; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.validateMockitoUsage; +import static org.mockito.Mockito.verify; /** * Tests for {@link CepOperator}. @@ -100,6 +110,31 @@ public void testKeyedCEPOperatorWatermarkForwarding() throws Exception { } } + @Test + public void testProcessingTimestampisPassedToNFA() throws Exception { + + final NFA nfa = NFACompiler.compileFactory(Pattern.begin("begin"), true).createNFA(); + final NFA spyNFA = spy(nfa); + + try ( + OneInputStreamOperatorTestHarness>> harness = + CepOperatorTestUtilities.getCepTestHarness(createOperatorForNFA(spyNFA).build())) { + + long timestamp = 5; + harness.open(); + harness.setProcessingTime(timestamp); + StreamRecord event = event().withTimestamp(3).asStreamRecord(); + harness.processElement(event); + verify(spyNFA).process( + any(SharedBufferAccessor.class), + any(NFAState.class), + eq(event.getValue()), + eq(timestamp), + any(AfterMatchSkipStrategy.class), + any(TimerService.class)); + } + } + @Test public void testKeyedCEPOperatorCheckpointing() throws Exception { diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/NFATestHarness.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/NFATestHarness.java index dd080fb4f63a7f..f8e0f575949cd5 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/NFATestHarness.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/NFATestHarness.java @@ -25,6 +25,7 @@ import org.apache.flink.cep.nfa.sharedbuffer.SharedBuffer; import org.apache.flink.cep.nfa.sharedbuffer.SharedBufferAccessor; import org.apache.flink.cep.pattern.Pattern; +import org.apache.flink.cep.time.TimerService; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import java.util.ArrayList; @@ -41,16 +42,19 @@ public final class NFATestHarness { private final NFA nfa; private final NFAState nfaState; private final AfterMatchSkipStrategy afterMatchSkipStrategy; + private final TimerService timerService; private NFATestHarness( SharedBuffer sharedBuffer, NFA nfa, NFAState nfaState, - AfterMatchSkipStrategy afterMatchSkipStrategy) { + AfterMatchSkipStrategy afterMatchSkipStrategy, + TimerService timerService) { this.sharedBuffer = sharedBuffer; this.nfa = nfa; this.nfaState = nfaState; this.afterMatchSkipStrategy = afterMatchSkipStrategy; + this.timerService = timerService; } /** @@ -68,7 +72,7 @@ public static NFATestHarnessBuilderNFA forNFA(NFA nfa) { } public List> feedRecords(List> inputEvents) throws Exception { - List> resultingPatterns = new ArrayList<>(); + final List> resultingPatterns = new ArrayList<>(); for (StreamRecord inputEvent : inputEvents) { resultingPatterns.addAll(feedRecord(inputEvent)); } @@ -76,8 +80,8 @@ public List> feedRecords(List> inputEvents) thro } public List> feedRecord(StreamRecord inputEvent) throws Exception { - List> resultingPatterns = new ArrayList<>(); - Collection>> matches = consumeRecord(inputEvent); + final List> resultingPatterns = new ArrayList<>(); + final Collection>> matches = consumeRecord(inputEvent); for (Map> p : matches) { List res = new ArrayList<>(); for (List le : p.values()) { @@ -89,7 +93,7 @@ public List> feedRecord(StreamRecord inputEvent) throws Excep } public Collection>> consumeRecords(Collection> inputEvents) throws Exception { - List>> resultingPatterns = new ArrayList<>(); + final List>> resultingPatterns = new ArrayList<>(); for (StreamRecord inputEvent : inputEvents) { resultingPatterns.addAll(consumeRecord(inputEvent)); } @@ -105,7 +109,8 @@ public Collection>> consumeRecord(StreamRecord in nfaState, inputEvent.getValue(), inputEvent.getTimestamp(), - afterMatchSkipStrategy); + afterMatchSkipStrategy, + timerService); } } @@ -129,12 +134,13 @@ public NFATestHarnessBuilderBase withTimeoutHandling() { @Override public NFATestHarness build() { - NFA nfa = NFAUtils.compile(pattern, timeoutHandling); + final NFA nfa = NFAUtils.compile(pattern, timeoutHandling); return new NFATestHarness( sharedBuffer, nfa, nfa.createInitialNFAState(), - afterMatchSkipStrategy); + afterMatchSkipStrategy, + timerService); } } @@ -159,7 +165,12 @@ public NFATestHarnessBuilderBase withNFAState(NFAState nfaState) { @Override public NFATestHarness build() { - return new NFATestHarness(sharedBuffer, nfa, nfaState, afterMatchSkipStrategy); + return new NFATestHarness( + sharedBuffer, + nfa, + nfaState, + afterMatchSkipStrategy, + timerService); } } @@ -171,6 +182,7 @@ public abstract static class NFATestHarnessBuilderBase { SharedBuffer sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer()); AfterMatchSkipStrategy afterMatchSkipStrategy; + TimerService timerService = new TestTimerService(); NFATestHarnessBuilderBase(AfterMatchSkipStrategy skipStrategy) { this.afterMatchSkipStrategy = skipStrategy; @@ -186,6 +198,11 @@ public NFATestHarnessBuilderBase withAfterMatchSkipStrategy(AfterMatchSkipStrate return this; } + public NFATestHarnessBuilderBase withTimerService(TimerService timerService) { + this.timerService = timerService; + return this; + } + public abstract NFATestHarness build(); } } diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/TestTimerService.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/TestTimerService.java new file mode 100644 index 00000000000000..5fd9755d195d0f --- /dev/null +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/utils/TestTimerService.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cep.utils; + +import org.apache.flink.cep.time.TimerService; + +/** + * Test implementation of {@link TimerService}. Provides setters for encapsulated properties. + */ +public class TestTimerService implements TimerService { + + public static final long UNDEFINED_TIME = Long.MIN_VALUE; + + private Long processingTime = UNDEFINED_TIME; + + @Override + public long currentProcessingTime() { + return processingTime; + } + + public void setCurrentProcessingTime(long processingTime) { + this.processingTime = processingTime; + } + +} From 0a46510a7e87f350492f4e4889c006f87ae7c665 Mon Sep 17 00:00:00 2001 From: Dawid Wysakowicz Date: Tue, 8 Jan 2019 12:07:48 +0100 Subject: [PATCH 075/147] [hotfix][cep] Renamed TimerContext to TimeContext --- .../apache/flink/cep/functions/PatternProcessFunction.java | 4 ++-- .../flink/cep/pattern/conditions/IterativeCondition.java | 4 ++-- .../flink/cep/time/{TimerContext.java => TimeContext.java} | 2 +- ...extTest.java => NFAIterativeConditionTimeContextTest.java} | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) rename flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/{TimerContext.java => TimeContext.java} (97%) rename flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/{NFAIterativeConditionTimerContextTest.java => NFAIterativeConditionTimeContextTest.java} (97%) diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/PatternProcessFunction.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/PatternProcessFunction.java index b8e6821c7b4f67..afc798ffa620e8 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/PatternProcessFunction.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/functions/PatternProcessFunction.java @@ -20,7 +20,7 @@ import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.api.common.functions.AbstractRichFunction; -import org.apache.flink.cep.time.TimerContext; +import org.apache.flink.cep.time.TimeContext; import org.apache.flink.util.Collector; import org.apache.flink.util.OutputTag; @@ -64,7 +64,7 @@ public abstract void processMatch( /** * Gives access to time related characteristics as well as enables emitting elements to side outputs. */ - public interface Context extends TimerContext { + public interface Context extends TimeContext { /** * Emits a record to the side output identified by the {@link OutputTag}. * diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/IterativeCondition.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/IterativeCondition.java index 786cb0834b7b1e..ad45dab9d7cf63 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/IterativeCondition.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/pattern/conditions/IterativeCondition.java @@ -20,7 +20,7 @@ import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.api.common.functions.Function; -import org.apache.flink.cep.time.TimerContext; +import org.apache.flink.cep.time.TimeContext; import java.io.Serializable; @@ -87,7 +87,7 @@ public abstract class IterativeCondition implements Function, Serializable { /** * The context used when evaluating the {@link IterativeCondition condition}. */ - public interface Context extends TimerContext { + public interface Context extends TimeContext { /** * @return An {@link Iterable} over the already accepted elements diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/TimerContext.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/TimeContext.java similarity index 97% rename from flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/TimerContext.java rename to flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/TimeContext.java index 9b5db520399814..55e43b8f0ec425 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/TimerContext.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/time/TimeContext.java @@ -27,7 +27,7 @@ * {@link org.apache.flink.cep.pattern.conditions.IterativeCondition} */ @PublicEvolving -public interface TimerContext { +public interface TimeContext { /** * Timestamp of the element currently being processed. diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAIterativeConditionTimerContextTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAIterativeConditionTimeContextTest.java similarity index 97% rename from flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAIterativeConditionTimerContextTest.java rename to flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAIterativeConditionTimeContextTest.java index 3f9b26ad340081..3ebc65f59adc59 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAIterativeConditionTimerContextTest.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAIterativeConditionTimeContextTest.java @@ -38,7 +38,7 @@ /** * Tests for accesing time properties from {@link IterativeCondition}. */ -public class NFAIterativeConditionTimerContextTest extends TestLogger { +public class NFAIterativeConditionTimeContextTest extends TestLogger { @Test public void testEventTimestamp() throws Exception { From 47e9ccf0d3686cfe3e3ffb8f38e606b0f3d0fe0e Mon Sep 17 00:00:00 2001 From: Dawid Wysakowicz Date: Tue, 8 Jan 2019 12:09:15 +0100 Subject: [PATCH 076/147] [FLINK-10596][cep, docs] Added description of TimeContext to docs --- docs/dev/libs/cep.md | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/dev/libs/cep.md b/docs/dev/libs/cep.md index ab44b5c99da01a..a358ba2bc1e0e8 100644 --- a/docs/dev/libs/cep.md +++ b/docs/dev/libs/cep.md @@ -311,6 +311,8 @@ middle.oneOrMore() previously accepted events for a given potential match. The cost of this operation can vary, so when implementing your condition, try to minimize its use. +Described context gives one access to event time characteristics as well. For more info see [Time context](#time-context). + **Simple Conditions:** This type of condition extends the aforementioned `IterativeCondition` class and decides whether to accept an event or not, based *only* on properties of the event itself. @@ -1508,6 +1510,7 @@ class MyPatternProcessFunction extends PatternProcessFunction The `PatternProcessFunction` gives access to a `Context` object. Thanks to it, one can access time related characteristics such as `currentProcessingTime` or `timestamp` of current match (which is the timestamp of the last element assigned to the match). +For more info see [Time context](#time-context). Through this context one can also emit results to a [side-output]({{ site.baseurl }}/dev/stream/side_output.html). @@ -1594,7 +1597,9 @@ val timeoutResult: DataStream = result.getSideOutput(outputTag) -## Handling Lateness in Event Time +## Time in CEP library + +### Handling Lateness in Event Time In `CEP` the order in which elements are processed matters. To guarantee that elements are processed in the correct order when working in event time, an incoming element is initially put in a buffer where elements are *sorted in ascending order based on their timestamp*, and when a watermark arrives, all the elements in this buffer with timestamps smaller than that of the watermark are processed. This implies that elements between watermarks are processed in event-time order. @@ -1620,7 +1625,6 @@ SingleOutputStreamOperator result = patternStream DataStream lateData = result.getSideOutput(lateDataOutputTag); - {% endhighlight %} @@ -1646,6 +1650,39 @@ val lateData: DataStream = result.getSideOutput(lateDataOutputTag) +### Time context + +In [PatternProcessFunction](#selecting-from-patterns) as well as in [IterativeCondition](#conditions) user has access to a context +that implements `TimeContext` as follows: + +{% highlight java %} +/** + * Enables access to time related characteristics such as current processing time or timestamp of + * currently processed element. Used in {@link PatternProcessFunction} and + * {@link org.apache.flink.cep.pattern.conditions.IterativeCondition} + */ +@PublicEvolving +public interface TimeContext { + + /** + * Timestamp of the element currently being processed. + * + *

In case of {@link org.apache.flink.streaming.api.TimeCharacteristic#ProcessingTime} this + * will be set to the time when event entered the cep operator. + */ + long timestamp(); + + /** Returns the current processing time. */ + long currentProcessingTime(); +} +{% endhighlight %} + +This context gives user access to time characteristics of processed events (incoming records in case of `IterativeCondition` and matches in case of `PatternProcessFunction`). +Call to `TimeContext#currentProcessingTime` always gives you the value of current processing time and this call should be preferred to e.g. calling `System.currentTimeMillis()`. + +In case of `TimeContext#timestamp()` the returned value is equal to assigned timestamp in case of `EventTime`. In `ProcessingTime` this will equal to the point of time when said event entered +cep operator (or when the match was generated in case of `PatternProcessFunction`). This means that the value will be consistent across multiple calls to that method. + ## Examples The following example detects the pattern `start, middle(name = "error") -> end(name = "critical")` on a keyed data From 52dd89d3b3369e0f314131993d2b7d110419ccd6 Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 4 Dec 2018 10:55:33 +0800 Subject: [PATCH 077/147] [FLINK-11056] [mesos] Remove MesosApplicationMasterRunner --- .../MesosApplicationMasterRunner.java | 487 ------------------ 1 file changed, 487 deletions(-) delete mode 100755 flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java deleted file mode 100755 index bbecf6abd7fb9a..00000000000000 --- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java +++ /dev/null @@ -1,487 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.mesos.runtime.clusterframework; - -import org.apache.flink.api.common.time.Time; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.JobManagerOptions; -import org.apache.flink.configuration.WebOptions; -import org.apache.flink.core.fs.FileSystem; -import org.apache.flink.mesos.entrypoint.MesosEntrypointUtils; -import org.apache.flink.mesos.runtime.clusterframework.services.MesosServices; -import org.apache.flink.mesos.runtime.clusterframework.services.MesosServicesUtils; -import org.apache.flink.mesos.runtime.clusterframework.store.MesosWorkerStore; -import org.apache.flink.mesos.util.MesosArtifactServer; -import org.apache.flink.mesos.util.MesosConfiguration; -import org.apache.flink.runtime.akka.AkkaUtils; -import org.apache.flink.runtime.clusterframework.BootstrapTools; -import org.apache.flink.runtime.clusterframework.ContainerSpecification; -import org.apache.flink.runtime.concurrent.ScheduledExecutorServiceAdapter; -import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils; -import org.apache.flink.runtime.jobmanager.JobManager; -import org.apache.flink.runtime.jobmanager.MemoryArchivist; -import org.apache.flink.runtime.jobmaster.JobMaster; -import org.apache.flink.runtime.metrics.MetricRegistryConfiguration; -import org.apache.flink.runtime.metrics.MetricRegistryImpl; -import org.apache.flink.runtime.process.ProcessReaper; -import org.apache.flink.runtime.security.SecurityConfiguration; -import org.apache.flink.runtime.security.SecurityUtils; -import org.apache.flink.runtime.util.EnvironmentInformation; -import org.apache.flink.runtime.util.ExecutorThreadFactory; -import org.apache.flink.runtime.util.Hardware; -import org.apache.flink.runtime.util.JvmShutdownSafeguard; -import org.apache.flink.runtime.util.SignalHandler; -import org.apache.flink.runtime.webmonitor.WebMonitor; -import org.apache.flink.runtime.webmonitor.retriever.impl.AkkaJobManagerRetriever; -import org.apache.flink.runtime.webmonitor.retriever.impl.AkkaQueryServiceRetriever; -import org.apache.flink.util.ExecutorUtils; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.actor.Address; -import akka.actor.Props; -import akka.actor.Terminated; -import akka.dispatch.OnComplete; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.PosixParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.URL; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import scala.Option; -import scala.concurrent.Await; -import scala.concurrent.duration.Duration; -import scala.concurrent.duration.FiniteDuration; - -import static org.apache.flink.runtime.concurrent.Executors.directExecutionContext; -import static org.apache.flink.util.Preconditions.checkState; - -/** - * This class is the executable entry point for the Mesos Application Master. - * It starts actor system and the actors for {@link JobManager} - * and {@link MesosFlinkResourceManager}. - * - *

The JobManager handles Flink job execution, while the MesosFlinkResourceManager handles container - * allocation and failure detection. - */ -public class MesosApplicationMasterRunner { - - protected static final Logger LOG = LoggerFactory.getLogger(MesosApplicationMasterRunner.class); - - /** - * The maximum time that TaskManagers may be waiting to register at the JobManager, - * before they quit. - */ - private static final FiniteDuration TASKMANAGER_REGISTRATION_TIMEOUT = new FiniteDuration(5, TimeUnit.MINUTES); - - /** The process environment variables. */ - private static final Map ENV = System.getenv(); - - /** The exit code returned if the initialization of the application master failed. */ - private static final int INIT_ERROR_EXIT_CODE = 31; - - /** The exit code returned if the process exits because a critical actor died. */ - private static final int ACTOR_DIED_EXIT_CODE = 32; - - // ------------------------------------------------------------------------ - // Command-line options - // ------------------------------------------------------------------------ - - private static final Options ALL_OPTIONS; - - static { - ALL_OPTIONS = - new Options() - .addOption(BootstrapTools.newDynamicPropertiesOption()); - } - - // ------------------------------------------------------------------------ - // Program entry point - // ------------------------------------------------------------------------ - - /** - * The entry point for the Mesos AppMaster. - * - * @param args The command line arguments. - */ - public static void main(String[] args) { - EnvironmentInformation.logEnvironmentInfo(LOG, "Mesos AppMaster", args); - SignalHandler.register(LOG); - JvmShutdownSafeguard.installAsShutdownHook(LOG); - - // run and exit with the proper return code - int returnCode = new MesosApplicationMasterRunner().run(args); - System.exit(returnCode); - } - - /** - * The instance entry point for the Mesos AppMaster. Obtains user group - * information and calls the main work method {@link #runPrivileged(Configuration, Configuration)} as a - * privileged action. - * - * @param args The command line arguments. - * @return The process exit code. - */ - protected int run(final String[] args) { - try { - LOG.debug("All environment variables: {}", ENV); - - // loading all config values here has the advantage that the program fails fast, if any - // configuration problem occurs - - CommandLineParser parser = new PosixParser(); - CommandLine cmd = parser.parse(ALL_OPTIONS, args); - - final Configuration dynamicProperties = BootstrapTools.parseDynamicProperties(cmd); - final Configuration config = MesosEntrypointUtils.loadConfiguration(dynamicProperties, LOG); - - // configure the filesystems - try { - FileSystem.initialize(config); - } catch (IOException e) { - throw new IOException("Error while configuring the filesystems.", e); - } - - // configure security - SecurityConfiguration sc = new SecurityConfiguration(config); - SecurityUtils.install(sc); - - // run the actual work in the installed security context - return SecurityUtils.getInstalledContext().runSecured(new Callable() { - @Override - public Integer call() throws Exception { - return runPrivileged(config, dynamicProperties); - } - }); - } - catch (Throwable t) { - // make sure that everything whatever ends up in the log - LOG.error("Mesos AppMaster initialization failed", t); - return INIT_ERROR_EXIT_CODE; - } - } - - // ------------------------------------------------------------------------ - // Core work method - // ------------------------------------------------------------------------ - - /** - * The main work method, must run as a privileged action. - * - * @return The return code for the Java process. - */ - protected int runPrivileged(Configuration config, Configuration dynamicProperties) { - - ActorSystem actorSystem = null; - WebMonitor webMonitor = null; - MesosArtifactServer artifactServer = null; - ScheduledExecutorService futureExecutor = null; - ExecutorService ioExecutor = null; - MesosServices mesosServices = null; - HighAvailabilityServices highAvailabilityServices = null; - MetricRegistryImpl metricRegistry = null; - - try { - // ------- (1) load and parse / validate all configurations ------- - - final String appMasterHostname = config.getString( - JobManagerOptions.ADDRESS, - InetAddress.getLocalHost().getHostName()); - - LOG.info("App Master Hostname to use: {}", appMasterHostname); - - // Mesos configuration - final MesosConfiguration mesosConfig = MesosEntrypointUtils.createMesosSchedulerConfiguration(config, appMasterHostname); - - // JM configuration - int numberProcessors = Hardware.getNumberCPUCores(); - - futureExecutor = Executors.newScheduledThreadPool( - numberProcessors, - new ExecutorThreadFactory("mesos-jobmanager-future")); - - ioExecutor = Executors.newFixedThreadPool( - numberProcessors, - new ExecutorThreadFactory("mesos-jobmanager-io")); - - mesosServices = MesosServicesUtils.createMesosServices(config, appMasterHostname); - - // TM configuration - final MesosTaskManagerParameters taskManagerParameters = MesosEntrypointUtils.createTmParameters(config, LOG); - - // JM endpoint, which should be explicitly configured based on acquired net resources - final int listeningPort = config.getInteger(JobManagerOptions.PORT); - checkState(listeningPort >= 0 && listeningPort <= 65536, "Config parameter \"" + - JobManagerOptions.PORT.key() + "\" is invalid, it must be between 0 and 65536"); - - // ----------------- (2) start the actor system ------------------- - - // try to start the actor system, JobManager and JobManager actor system - // using the configured address and ports - actorSystem = BootstrapTools.startActorSystem(config, appMasterHostname, listeningPort, LOG); - - Address address = AkkaUtils.getAddress(actorSystem); - final String akkaHostname = address.host().get(); - final int akkaPort = (Integer) address.port().get(); - - LOG.info("Actor system bound to hostname {}.", akkaHostname); - - // try to start the artifact server - LOG.debug("Starting Artifact Server"); - artifactServer = mesosServices.getArtifactServer(); - - // ----------------- (3) Generate the configuration for the TaskManagers ------------------- - - // generate a container spec which conveys the artifacts/vars needed to launch a TM - ContainerSpecification taskManagerContainerSpec = new ContainerSpecification(); - - // propagate the AM dynamic configuration to the TM - taskManagerContainerSpec.getDynamicConfiguration().addAll(dynamicProperties); - - // propagate newly-generated configuration elements - final Configuration taskManagerConfig = BootstrapTools.generateTaskManagerConfiguration( - new Configuration(), akkaHostname, akkaPort, taskManagerParameters.containeredParameters().numSlots(), - TASKMANAGER_REGISTRATION_TIMEOUT); - taskManagerContainerSpec.getDynamicConfiguration().addAll(taskManagerConfig); - - // apply the overlays - MesosEntrypointUtils.applyOverlays(config, taskManagerContainerSpec); - - // configure the artifact server to serve the specified artifacts - LaunchableMesosWorker.configureArtifactServer(artifactServer, taskManagerContainerSpec); - - // ----------------- (4) start the actors ------------------- - - // 1) JobManager & Archive (in non-HA case, the leader service takes this) - // 2) Web Monitor (we need its port to register) - // 3) Resource Master for Mesos - // 4) Process reapers for the JobManager and Resource Master - - // 0: Start the JobManager services - highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices( - config, - ioExecutor, - HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION); - - // 1: the web monitor - LOG.debug("Starting Web Frontend"); - - Time webMonitorTimeout = Time.milliseconds(config.getLong(WebOptions.TIMEOUT)); - - webMonitor = BootstrapTools.startWebMonitorIfConfigured( - config, - highAvailabilityServices, - new AkkaJobManagerRetriever(actorSystem, webMonitorTimeout, 10, Time.milliseconds(50L)), - new AkkaQueryServiceRetriever(actorSystem, webMonitorTimeout), - webMonitorTimeout, - new ScheduledExecutorServiceAdapter(futureExecutor), - LOG); - if (webMonitor != null) { - final URL webMonitorURL = new URL(webMonitor.getRestAddress()); - mesosConfig.frameworkInfo().setWebuiUrl(webMonitorURL.toExternalForm()); - } - - // 2: the JobManager - LOG.debug("Starting JobManager actor"); - - metricRegistry = new MetricRegistryImpl( - MetricRegistryConfiguration.fromConfiguration(config)); - - metricRegistry.startQueryService(actorSystem, null); - - // we start the JobManager with its standard name - ActorRef jobManager = JobManager.startJobManagerActors( - config, - actorSystem, - futureExecutor, - ioExecutor, - highAvailabilityServices, - metricRegistry, - webMonitor != null ? Option.apply(webMonitor.getRestAddress()) : Option.empty(), - Option.apply(JobMaster.JOB_MANAGER_NAME), - Option.apply(JobMaster.ARCHIVE_NAME), - getJobManagerClass(), - getArchivistClass())._1(); - - // 3: Flink's Mesos ResourceManager - LOG.debug("Starting Mesos Flink Resource Manager"); - - // create the worker store to persist task information across restarts - MesosWorkerStore workerStore = mesosServices.createMesosWorkerStore( - config, - ioExecutor); - - Props resourceMasterProps = MesosFlinkResourceManager.createActorProps( - getResourceManagerClass(), - config, - mesosConfig, - workerStore, - highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID), - taskManagerParameters, - taskManagerContainerSpec, - artifactServer, - LOG); - - ActorRef resourceMaster = actorSystem.actorOf(resourceMasterProps, "Mesos_Resource_Master"); - - // 4: Process reapers - // The process reapers ensure that upon unexpected actor death, the process exits - // and does not stay lingering around unresponsive - - LOG.debug("Starting process reapers for JobManager"); - - actorSystem.actorOf( - Props.create(ProcessReaper.class, resourceMaster, LOG, ACTOR_DIED_EXIT_CODE), - "Mesos_Resource_Master_Process_Reaper"); - - actorSystem.actorOf( - Props.create(ProcessReaper.class, jobManager, LOG, ACTOR_DIED_EXIT_CODE), - "JobManager_Process_Reaper"); - } - catch (Throwable t) { - // make sure that everything whatever ends up in the log - LOG.error("Mesos JobManager initialization failed", t); - - if (webMonitor != null) { - try { - webMonitor.stop(); - } catch (Throwable ignored) { - LOG.warn("Failed to stop the web frontend", ignored); - } - } - - if (actorSystem != null) { - actorSystem.terminate().onComplete( - new OnComplete() { - public void onComplete(Throwable failure, Terminated success) { - if (failure != null) { - LOG.error("Error shutting down actor system", failure); - } - } - }, directExecutionContext()); - } - - if (futureExecutor != null) { - try { - futureExecutor.shutdownNow(); - } catch (Throwable tt) { - LOG.error("Error shutting down future executor", tt); - } - } - - if (ioExecutor != null) { - try { - ioExecutor.shutdownNow(); - } catch (Throwable tt) { - LOG.error("Error shutting down io executor", tt); - } - } - - if (mesosServices != null) { - try { - mesosServices.close(false); - } catch (Throwable tt) { - LOG.error("Error closing the mesos services.", tt); - } - } - - return INIT_ERROR_EXIT_CODE; - } - - // everything started, we can wait until all is done or the process is killed - LOG.info("Mesos JobManager started"); - - // wait until everything is done - try { - Await.ready(actorSystem.whenTerminated(), Duration.Inf()); - } catch (InterruptedException | TimeoutException e) { - LOG.error("Error shutting down actor system", e); - } - - // if we get here, everything work out jolly all right, and we even exited smoothly - if (webMonitor != null) { - try { - webMonitor.stop(); - } catch (Throwable t) { - LOG.error("Failed to stop the web frontend", t); - } - } - - if (highAvailabilityServices != null) { - try { - highAvailabilityServices.close(); - } catch (Throwable t) { - LOG.error("Could not properly stop the high availability services."); - } - } - - if (metricRegistry != null) { - try { - metricRegistry.shutdown().get(); - } catch (Throwable t) { - LOG.error("Could not shut down metric registry.", t); - } - } - - ExecutorUtils.gracefulShutdown( - AkkaUtils.getTimeout(config).toMillis(), - TimeUnit.MILLISECONDS, - futureExecutor, - ioExecutor); - - try { - mesosServices.close(true); - } catch (Throwable t) { - LOG.error("Failed to clean up and close MesosServices.", t); - } - - return 0; - } - - // ------------------------------------------------------------------------ - // For testing, this allows to override the actor classes used for - // JobManager and the archive of completed jobs - // ------------------------------------------------------------------------ - - protected Class getResourceManagerClass() { - return MesosFlinkResourceManager.class; - } - - protected Class getJobManagerClass() { - return MesosJobManager.class; - } - - protected Class getArchivistClass() { - return MemoryArchivist.class; - } - -} From 64f766f5c75eb040a2927ba47589a6c63ff466c0 Mon Sep 17 00:00:00 2001 From: intsmaze <979222969@qq.com> Date: Wed, 9 Jan 2019 16:55:01 +0800 Subject: [PATCH 078/147] [hotfix][docs] Sync formatting for cli usage examples --- docs/ops/cli.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ops/cli.md b/docs/ops/cli.md index 9af5c5b5333095..0e658c10f27215 100644 --- a/docs/ops/cli.md +++ b/docs/ops/cli.md @@ -127,6 +127,7 @@ The command line can be used to ./bin/flink stop - Modify a running job (streaming jobs only): + ./bin/flink modify -p From 9323ef6ac12e1a44f9745ea7589c526153223cfa Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 8 Jan 2019 13:24:26 +0800 Subject: [PATCH 079/147] [hotfix][test] Remove usage of deprecated method --- .../itcases/AbstractQueryableStateTestBase.java | 8 ++++---- .../itcases/HAQueryableStateFsBackendITCase.java | 4 ++-- .../itcases/HAQueryableStateRocksDBBackendITCase.java | 4 ++-- .../itcases/NonHAQueryableStateFsBackendITCase.java | 4 ++-- .../itcases/NonHAQueryableStateRocksDBBackendITCase.java | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java index eb6b3ef8c50e83..bb4891005cb624 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java @@ -56,8 +56,8 @@ import org.apache.flink.runtime.concurrent.ScheduledExecutorServiceAdapter; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobStatus; -import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.CheckpointListener; +import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.QueryableStateStream; @@ -107,7 +107,7 @@ public abstract class AbstractQueryableStateTestBase extends TestLogger { private static final Duration TEST_TIMEOUT = Duration.ofSeconds(10000L); - public static final long RETRY_TIMEOUT = 50L; + private static final long RETRY_TIMEOUT = 50L; private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(4); private final ScheduledExecutor executor = new ScheduledExecutorServiceAdapter(executorService); @@ -115,7 +115,7 @@ public abstract class AbstractQueryableStateTestBase extends TestLogger { /** * State backend to use. */ - protected AbstractStateBackend stateBackend; + private StateBackend stateBackend; /** * Client shared between all the test. @@ -142,7 +142,7 @@ public void setUp() throws Exception { * * @return a state backend instance for each unit test */ - protected abstract AbstractStateBackend createStateBackend() throws Exception; + protected abstract StateBackend createStateBackend() throws Exception; /** * Runs a simple topology producing random (key, 1) pairs at the sources (where diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java index 9755b52e2f17de..ef6a267df0fbdd 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java @@ -25,7 +25,7 @@ import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.configuration.WebOptions; import org.apache.flink.queryablestate.client.QueryableStateClient; -import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.runtime.state.filesystem.FsStateBackend; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.test.util.MiniClusterWithClientResource; @@ -58,7 +58,7 @@ public class HAQueryableStateFsBackendITCase extends AbstractQueryableStateTestB private static MiniClusterWithClientResource miniClusterResource; @Override - protected AbstractStateBackend createStateBackend() throws Exception { + protected StateBackend createStateBackend() throws Exception { return new FsStateBackend(temporaryFolder.newFolder().toURI().toString()); } diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java index 94baaed37458c7..46202c322cb5c6 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java @@ -26,7 +26,7 @@ import org.apache.flink.configuration.WebOptions; import org.apache.flink.contrib.streaming.state.RocksDBStateBackend; import org.apache.flink.queryablestate.client.QueryableStateClient; -import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.test.util.MiniClusterWithClientResource; @@ -58,7 +58,7 @@ public class HAQueryableStateRocksDBBackendITCase extends AbstractQueryableState private static MiniClusterWithClientResource miniClusterResource; @Override - protected AbstractStateBackend createStateBackend() throws Exception { + protected StateBackend createStateBackend() throws Exception { return new RocksDBStateBackend(temporaryFolder.newFolder().toURI().toString()); } diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java index 36637c0d34200f..f1382556eb7c7d 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java @@ -24,7 +24,7 @@ import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.configuration.WebOptions; import org.apache.flink.queryablestate.client.QueryableStateClient; -import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.runtime.state.filesystem.FsStateBackend; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.test.util.MiniClusterWithClientResource; @@ -61,7 +61,7 @@ public class NonHAQueryableStateFsBackendITCase extends AbstractQueryableStateTe .build()); @Override - protected AbstractStateBackend createStateBackend() throws Exception { + protected StateBackend createStateBackend() throws Exception { return new FsStateBackend(temporaryFolder.newFolder().toURI().toString()); } diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java index ce3e665fce6190..3ad0dda52e2be9 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java @@ -25,7 +25,7 @@ import org.apache.flink.configuration.WebOptions; import org.apache.flink.contrib.streaming.state.RocksDBStateBackend; import org.apache.flink.queryablestate.client.QueryableStateClient; -import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.test.util.MiniClusterWithClientResource; @@ -60,7 +60,7 @@ public class NonHAQueryableStateRocksDBBackendITCase extends AbstractQueryableSt .build()); @Override - protected AbstractStateBackend createStateBackend() throws Exception { + protected StateBackend createStateBackend() throws Exception { return new RocksDBStateBackend(temporaryFolder.newFolder().toURI().toString()); } From 5b5ec591273fc6ce8352f20fd9108315bc0a05d9 Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 8 Jan 2019 13:25:03 +0800 Subject: [PATCH 080/147] [hotfix][test] Reduce test timeout to a reasonable value --- .../queryablestate/itcases/AbstractQueryableStateTestBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java index bb4891005cb624..9efc9e29857893 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java @@ -106,7 +106,7 @@ */ public abstract class AbstractQueryableStateTestBase extends TestLogger { - private static final Duration TEST_TIMEOUT = Duration.ofSeconds(10000L); + private static final Duration TEST_TIMEOUT = Duration.ofSeconds(200L); private static final long RETRY_TIMEOUT = 50L; private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(4); From 4782236aaec550f478ff21f8ae1a08f5c5a6ac2f Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 8 Jan 2019 13:26:44 +0800 Subject: [PATCH 081/147] [hotfix][test] Adjust suppressWarnings --- .../queryablestate/itcases/AbstractQueryableStateTestBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java index 9efc9e29857893..ac4052eceec475 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java @@ -153,7 +153,6 @@ public void setUp() throws Exception { * to query the counts of each key in rounds until all keys have non-zero counts. */ @Test - @SuppressWarnings("unchecked") public void testQueryableState() throws Exception { final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT); final int numKeys = 256; @@ -597,6 +596,7 @@ public Integer getKey(Tuple2 value) { } }).asQueryableState("matata"); + @SuppressWarnings("unchecked") final ValueStateDescriptor> stateDesc = (ValueStateDescriptor>) queryableState.getStateDescriptor(); From a93e91a9994604561ef66dd638d4645947cdd506 Mon Sep 17 00:00:00 2001 From: leesf <490081539@qq.com> Date: Wed, 9 Jan 2019 17:00:26 +0800 Subject: [PATCH 082/147] [hotfix][docs] Update addShutdownHookThread javadocs --- .../main/java/org/apache/flink/util/ShutdownHookUtil.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flink-core/src/main/java/org/apache/flink/util/ShutdownHookUtil.java b/flink-core/src/main/java/org/apache/flink/util/ShutdownHookUtil.java index 7526f65847c681..b87b08e05b96eb 100644 --- a/flink-core/src/main/java/org/apache/flink/util/ShutdownHookUtil.java +++ b/flink-core/src/main/java/org/apache/flink/util/ShutdownHookUtil.java @@ -50,7 +50,12 @@ public static Thread addShutdownHook( } /** - * Adds a shutdown hook to the JVM and returns the Thread, which has been registered. + * Adds a shutdown hook to the JVM. + * + * @param shutdownHook Shutdown hook to be registered. + * @param serviceName The name of service. + * @param logger The logger to log. + * @return Whether the hook has been successfully registered. */ public static boolean addShutdownHookThread( final Thread shutdownHook, From 6a4bc901aac8ce284efdda267fc476cc203bc13d Mon Sep 17 00:00:00 2001 From: zy <602128569@qq.com> Date: Wed, 9 Jan 2019 17:16:19 +0800 Subject: [PATCH 083/147] [hotfix][docs] Fix javadocs of RandomFibonacciSource --- .../flink/streaming/examples/iteration/IterateExample.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java index d123615c7fca23..13db58ae187b10 100644 --- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java +++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java @@ -115,7 +115,7 @@ public static void main(String[] args) throws Exception { // ************************************************************************* /** - * Generate BOUND number of random integer pairs from the range from 0 to BOUND/2. + * Generate BOUND number of random integer pairs from the range from 1 to BOUND/2. */ private static class RandomFibonacciSource implements SourceFunction> { private static final long serialVersionUID = 1L; From 3702029f45b7034b767e2b7eb01601c7f76ab35e Mon Sep 17 00:00:00 2001 From: Matrix42 <934336389@qq.com> Date: Tue, 8 Jan 2019 14:43:19 +0100 Subject: [PATCH 084/147] [FLINK-11140][fs-connector] Fix empty child path check in Buckets. This closes #7287. --- .../functions/sink/filesystem/Buckets.java | 9 ++- .../filesystem/BucketAssignerITCases.java | 66 +++++++++++++++++++ .../sink/filesystem/BucketsTest.java | 10 +-- 3 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssignerITCases.java diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Buckets.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Buckets.java index d08bc2ac0c382a..7b35e502830657 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Buckets.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Buckets.java @@ -254,7 +254,7 @@ private void snapshotActiveBuckets( } } - void onElement(final IN value, final SinkFunction.Context context) throws Exception { + Bucket onElement(final IN value, final SinkFunction.Context context) throws Exception { final long currentProcessingTime = context.currentProcessingTime(); // setting the values in the bucketer context @@ -272,6 +272,7 @@ void onElement(final IN value, final SinkFunction.Context context) throws Except // another part file for the bucket, if we start from 0 we may overwrite previous parts. this.maxPartCounter = Math.max(maxPartCounter, bucket.getPartCounter()); + return bucket; } private Bucket getOrCreateBucketForBucketId(final BucketID bucketId) throws IOException { @@ -304,7 +305,11 @@ void close() { } private Path assembleBucketPath(BucketID bucketId) { - return new Path(basePath, bucketId.toString()); + final String child = bucketId.toString(); + if ("".equals(child)) { + return basePath; + } + return new Path(basePath, child); } /** diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssignerITCases.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssignerITCases.java new file mode 100644 index 00000000000000..5ae57ce43bc21e --- /dev/null +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssignerITCases.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem; + +import org.apache.flink.api.common.serialization.SimpleStringEncoder; +import org.apache.flink.core.fs.Path; +import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.BasePathBucketAssigner; +import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy; + +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; + +/** + * Integration tests for {@link BucketAssigner bucket assigners}. + */ +public class BucketAssignerITCases { + + @ClassRule + public static final TemporaryFolder TEMP_FOLDER = new TemporaryFolder(); + + @Test + public void testAssembleBucketPath() throws Exception { + final File outDir = TEMP_FOLDER.newFolder(); + final Path basePath = new Path(outDir.toURI()); + final long time = 1000L; + + final RollingPolicy rollingPolicy = + DefaultRollingPolicy + .create() + .withMaxPartSize(7L) + .build(); + + final Buckets buckets = new Buckets<>( + basePath, + new BasePathBucketAssigner<>(), + new DefaultBucketFactoryImpl<>(), + new RowWisePartWriter.Factory<>(new SimpleStringEncoder<>()), + rollingPolicy, + 0 + ); + + Bucket bucket = + buckets.onElement("abc", new TestUtils.MockSinkContext(time, time, time)); + Assert.assertEquals(new Path(basePath.toUri()), bucket.getBucketPath()); + } +} diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketsTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketsTest.java index aee362178a76eb..8369b90ac9665e 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketsTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketsTest.java @@ -188,7 +188,7 @@ public void testOnProcessingTime() throws Exception { // it takes the current processing time of the context for the creation time, // and for the last modification time. - buckets.onElement("test", new TestUtils.MockSinkContext(1L, 2L , 3L)); + buckets.onElement("test", new TestUtils.MockSinkContext(1L, 2L, 3L)); // now it should roll buckets.onProcessingTime(7L); @@ -214,13 +214,13 @@ public void testBucketIsRemovedWhenNotActive() throws Exception { final Path path = new Path(outDir.toURI()); final OnProcessingTimePolicy rollOnProcessingTimeCountingPolicy = - new OnProcessingTimePolicy<>(2L); + new OnProcessingTimePolicy<>(2L); final Buckets buckets = createBuckets(path, rollOnProcessingTimeCountingPolicy, 0); // it takes the current processing time of the context for the creation time, and for the last modification time. - buckets.onElement("test", new TestUtils.MockSinkContext(1L, 2L , 3L)); + buckets.onElement("test", new TestUtils.MockSinkContext(1L, 2L, 3L)); // now it should roll buckets.onProcessingTime(7L); @@ -244,7 +244,7 @@ public void testPartCounterAfterBucketResurrection() throws Exception { createBuckets(path, rollOnProcessingTimeCountingPolicy, 0); // it takes the current processing time of the context for the creation time, and for the last modification time. - buckets.onElement("test", new TestUtils.MockSinkContext(1L, 2L , 3L)); + buckets.onElement("test", new TestUtils.MockSinkContext(1L, 2L, 3L)); Assert.assertEquals(1L, buckets.getActiveBuckets().get("test").getPartCounter()); // now it should roll @@ -257,7 +257,7 @@ public void testPartCounterAfterBucketResurrection() throws Exception { Assert.assertTrue(buckets.getActiveBuckets().isEmpty()); - buckets.onElement("test", new TestUtils.MockSinkContext(2L, 3L , 4L)); + buckets.onElement("test", new TestUtils.MockSinkContext(2L, 3L, 4L)); Assert.assertEquals(2L, buckets.getActiveBuckets().get("test").getPartCounter()); } From 6650d74588d326121f9e5925c5e845ece691dc68 Mon Sep 17 00:00:00 2001 From: hequn8128 Date: Wed, 19 Dec 2018 11:26:52 +0800 Subject: [PATCH 085/147] [FLINK-11197][tests] Improve migration test comments about how to generate snapshots --- .../connectors/fs/bucketing/BucketingSinkMigrationTest.java | 1 + .../connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java | 1 + .../connectors/kinesis/FlinkKinesisConsumerMigrationTest.java | 1 + .../flink/hdfstests/ContinuousFileProcessingMigrationTest.java | 1 + .../java/org/apache/flink/cep/operator/CEPMigrationTest.java | 1 + .../runtime/operators/windowing/WindowOperatorMigrationTest.java | 1 + .../utils/LegacyStatefulJobSavepointMigrationITCase.java | 1 + .../checkpointing/utils/StatefulJobSavepointMigrationITCase.java | 1 + .../utils/StatefulJobWBroadcastStateMigrationITCase.java | 1 + .../test/migration/TypeSerializerSnapshotMigrationITCase.java | 1 + .../scala/migration/StatefulJobSavepointMigrationITCase.scala | 1 + .../migration/StatefulJobWBroadcastStateMigrationITCase.scala | 1 + 12 files changed, 12 insertions(+) diff --git a/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java b/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java index fb0796aec6c912..44be702bccdd25 100644 --- a/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java +++ b/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java @@ -69,6 +69,7 @@ public class BucketingSinkMigrationTest { /** * TODO change this to the corresponding savepoint version to be written (e.g. {@link MigrationVersion#v1_3} for 1.3) * TODO and remove all @Ignore annotations on write*Snapshot() methods to generate savepoints + * TODO Note: You should generate the savepoint based on the release branch instead of the master. */ private final MigrationVersion flinkGenerateSavepointVersion = null; diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java index decb5cfebe3257..be468a0237c1f6 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java @@ -79,6 +79,7 @@ public class FlinkKafkaConsumerBaseMigrationTest { /** * TODO change this to the corresponding savepoint version to be written (e.g. {@link MigrationVersion#v1_3} for 1.3) * TODO and remove all @Ignore annotations on write*Snapshot() methods to generate savepoints + * TODO Note: You should generate the savepoint based on the release branch instead of the master. */ private final MigrationVersion flinkGenerateSavepointVersion = null; diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java index 9a6d2d66a6feff..e21a880100cb99 100644 --- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java +++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java @@ -76,6 +76,7 @@ public class FlinkKinesisConsumerMigrationTest { /** * TODO change this to the corresponding savepoint version to be written (e.g. {@link MigrationVersion#v1_3} for 1.3) * TODO and remove all @Ignore annotations on the writeSnapshot() method to generate savepoints + * TODO Note: You should generate the savepoint based on the release branch instead of the master. */ private final MigrationVersion flinkGenerateSavepointVersion = null; diff --git a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java index 1161368f073a89..7f7e0c50dc34b3 100644 --- a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java +++ b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java @@ -87,6 +87,7 @@ public static Collection> parameters () { /** * TODO change this to the corresponding savepoint version to be written (e.g. {@link MigrationVersion#v1_3} for 1.3) * TODO and remove all @Ignore annotations on write*Snapshot() methods to generate savepoints + * TODO Note: You should generate the savepoint based on the release branch instead of the master. */ private final MigrationVersion flinkGenerateSavepointVersion = null; diff --git a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPMigrationTest.java b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPMigrationTest.java index 51c4ad0c1e1ab6..061a3c655a68d1 100644 --- a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPMigrationTest.java +++ b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator/CEPMigrationTest.java @@ -64,6 +64,7 @@ public class CEPMigrationTest { /** * TODO change this to the corresponding savepoint version to be written (e.g. {@link MigrationVersion#v1_3} for 1.3) * TODO and remove all @Ignore annotations on write*Snapshot() methods to generate savepoints + * TODO Note: You should generate the savepoint based on the release branch instead of the master. */ private final MigrationVersion flinkGenerateSavepointVersion = null; diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java index c27d6418ae4901..0168a7b81bebf1 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java @@ -100,6 +100,7 @@ public static Collection parameters () { /** * TODO change this to the corresponding savepoint version to be written (e.g. {@link MigrationVersion#v1_3} for 1.3) * TODO and remove all @Ignore annotations on write*Snapshot() methods to generate savepoints + * TODO Note: You should generate the savepoint based on the release branch instead of the master. */ private final MigrationVersion flinkGenerateSavepointVersion = null; diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/LegacyStatefulJobSavepointMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/LegacyStatefulJobSavepointMigrationITCase.java index eee13500e3c604..641689a5ae8e7f 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/LegacyStatefulJobSavepointMigrationITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/LegacyStatefulJobSavepointMigrationITCase.java @@ -82,6 +82,7 @@ public static Collection> parameters () { * TODO to generate savepoints for a specific Flink version / backend type, * TODO change these values accordingly, e.g. to generate for 1.3 with RocksDB, * TODO set as (MigrationVersion.v1_3, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME) + * TODO Note: You should generate the savepoint based on the release branch instead of the master. */ private final MigrationVersion flinkGenerateSavepointVersion = MigrationVersion.v1_4; private final String flinkGenerateSavepointBackendType = StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME; diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java index 4f3d274a3819f3..e3bc43b749d9fc 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java @@ -72,6 +72,7 @@ public enum ExecutionMode { } // TODO change this to PERFORM_SAVEPOINT to regenerate binary savepoints + // TODO Note: You should generate the savepoint based on the release branch instead of the master. private final ExecutionMode executionMode = ExecutionMode.VERIFY_SAVEPOINT; @Parameterized.Parameters(name = "Migrate Savepoint / Backend: {0}") diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobWBroadcastStateMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobWBroadcastStateMigrationITCase.java index da63b9b3096a3c..456c34f9a1db5d 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobWBroadcastStateMigrationITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobWBroadcastStateMigrationITCase.java @@ -56,6 +56,7 @@ public class StatefulJobWBroadcastStateMigrationITCase extends SavepointMigratio private static final int NUM_SOURCE_ELEMENTS = 4; // TODO change this to PERFORM_SAVEPOINT to regenerate binary savepoints + // TODO Note: You should generate the savepoint based on the release branch instead of the master. private final StatefulJobSavepointMigrationITCase.ExecutionMode executionMode = StatefulJobSavepointMigrationITCase.ExecutionMode.VERIFY_SAVEPOINT; diff --git a/flink-tests/src/test/java/org/apache/flink/test/migration/TypeSerializerSnapshotMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/migration/TypeSerializerSnapshotMigrationITCase.java index ac318722979fb8..377cea09ff423e 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/migration/TypeSerializerSnapshotMigrationITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/migration/TypeSerializerSnapshotMigrationITCase.java @@ -73,6 +73,7 @@ public enum ExecutionMode { } // TODO change this to PERFORM_SAVEPOINT to regenerate binary savepoints + // TODO Note: You should generate the savepoint based on the release branch instead of the master. private final ExecutionMode executionMode = ExecutionMode.VERIFY_SAVEPOINT; @Parameterized.Parameters(name = "Migrate Savepoint / Backend: {0}") diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala index 42db4e47264e6c..71514771454c14 100644 --- a/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala +++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala @@ -66,6 +66,7 @@ object StatefulJobSavepointMigrationITCase { // TODO to generate savepoints for a specific Flink version / backend type, // TODO change these values accordingly, e.g. to generate for 1.3 with RocksDB, // TODO set as (MigrationVersion.v1_3, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME) + // TODO Note: You should generate the savepoint based on the release branch instead of the master. val GENERATE_SAVEPOINT_VER: MigrationVersion = MigrationVersion.v1_4 val GENERATE_SAVEPOINT_BACKEND_TYPE: String = StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobWBroadcastStateMigrationITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobWBroadcastStateMigrationITCase.scala index 4d9af6fca3fbd4..e790a72833f756 100644 --- a/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobWBroadcastStateMigrationITCase.scala +++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobWBroadcastStateMigrationITCase.scala @@ -64,6 +64,7 @@ object StatefulJobWBroadcastStateMigrationITCase { // TODO to generate savepoints for a specific Flink version / backend type, // TODO change these values accordingly, e.g. to generate for 1.3 with RocksDB, // TODO set as (MigrationVersion.v1_3, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME) + // TODO Note: You should generate the savepoint based on the release branch instead of the master. val GENERATE_SAVEPOINT_VER: MigrationVersion = MigrationVersion.v1_7 val GENERATE_SAVEPOINT_BACKEND_TYPE: String = StateBackendLoader.MEMORY_STATE_BACKEND_NAME From e4bc287ea5f3f72f9dfe0af600a2a013cefb0374 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 9 Jan 2019 10:33:57 +0100 Subject: [PATCH 086/147] [FLINK-11262][py] Bump jython-standalone to 2.7.1 Bump the jython dependency because of a security issue. --- NOTICE-binary | 2 +- flink-libraries/flink-streaming-python/pom.xml | 2 +- .../flink-streaming-python/src/main/resources/META-INF/NOTICE | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/NOTICE-binary b/NOTICE-binary index 74df3d1c03ddda..0b10a27b9e9589 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -5797,7 +5797,7 @@ The Apache Software Foundation (http://www.apache.org/). This project bundles the following dependencies under the Python Software Foundation License. (https://opensource.org/licenses/PythonSoftFoundation.php) See bundled license files for details. -- org.python:jython-standalone:2.7.0 +- org.python:jython-standalone:2.7.1 flink-metrics-graphite Copyright 2014-2018 The Apache Software Foundation diff --git a/flink-libraries/flink-streaming-python/pom.xml b/flink-libraries/flink-streaming-python/pom.xml index 1659d83a34edd5..f32056dba6f4ed 100644 --- a/flink-libraries/flink-streaming-python/pom.xml +++ b/flink-libraries/flink-streaming-python/pom.xml @@ -61,7 +61,7 @@ under the License. org.python jython-standalone - 2.7.0 + 2.7.1 diff --git a/flink-libraries/flink-streaming-python/src/main/resources/META-INF/NOTICE b/flink-libraries/flink-streaming-python/src/main/resources/META-INF/NOTICE index f89bbcbf77d066..6fcb19f3bc1466 100644 --- a/flink-libraries/flink-streaming-python/src/main/resources/META-INF/NOTICE +++ b/flink-libraries/flink-streaming-python/src/main/resources/META-INF/NOTICE @@ -7,4 +7,4 @@ The Apache Software Foundation (http://www.apache.org/). This project bundles the following dependencies under the Python Software Foundation License. (https://opensource.org/licenses/PythonSoftFoundation.php) See bundled license files for details. -- org.python:jython-standalone:2.7.0 +- org.python:jython-standalone:2.7.1 From 78cc9bc8b7c9ff2806fa6176ec15f38199fe6b25 Mon Sep 17 00:00:00 2001 From: boshu Zheng Date: Wed, 9 Jan 2019 17:56:49 +0800 Subject: [PATCH 087/147] [FLINK-10783][tests] Update WindowOperatorMigrationTest for 1.7 --- .../windowing/WindowOperatorMigrationTest.java | 3 ++- ...tion-test-apply-event-time-flink1.7-snapshot | Bin 0 -> 6465 bytes ...test-apply-processing-time-flink1.7-snapshot | Bin 0 -> 6357 bytes ...n-test-kryo-serialized-key-flink1.7-snapshot | Bin 0 -> 21981 bytes ...ion-test-reduce-event-time-flink1.7-snapshot | Bin 0 -> 6329 bytes ...est-reduce-processing-time-flink1.7-snapshot | Bin 0 -> 6270 bytes ...sion-with-stateful-trigger-flink1.7-snapshot | Bin 0 -> 9972 bytes ...with-stateful-trigger-mint-flink1.7-snapshot | Bin 0 -> 9112 bytes 8 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.7-snapshot create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.7-snapshot create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.7-snapshot create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.7-snapshot create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.7-snapshot create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.7-snapshot create mode 100644 flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.7-snapshot diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java index 0168a7b81bebf1..f4883c939bb1b8 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java @@ -91,7 +91,8 @@ public static Collection parameters () { MigrationVersion.v1_3, MigrationVersion.v1_4, MigrationVersion.v1_5, - MigrationVersion.v1_6); + MigrationVersion.v1_6, + MigrationVersion.v1_7); } private static final TypeInformation> STRING_INT_TUPLE = diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.7-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.7-snapshot new file mode 100644 index 0000000000000000000000000000000000000000..0051be0b6507eb38692e25748593489d539f078f GIT binary patch literal 6465 zcmeHLOK;Oa5FR@z5>!BWh;rb-A#kfhAfliiP$&jP7^rXn0f%UtZc>9|hxMkfLj|Y) z1^xj7aR=^5{0=0JAh;tav%5|l69+(oP)-Xtm)E22xcOCB` zP8_4=H=Dj^@YNP=bH@#gc@j{==FIULHf0WRoi)mGKQGuGX$6Zum&h~QK!yzLUv|6& ze|f6rdz^Ya5aQya!NgVbu6e0yS4tJLTDd!CLe?tV72MNd;&#cpWmawTX1QdQ@0mB{ zYUvLov7CTM8ulop!QngKqyR#Qaw6mdvP61}#@Zftn$)PYTP}@2XML~kG&<00_FLf& z^kSy)@)L{&43aH395_OWN7DgT7+lzfm>1FI3QVA7%P>Zm!&l3&%cWj}FW!fVx?`pG`m0Z{#5>e0AA^g- zsVI@+g4uX8UCMoL_4&q&cgC_Q1bhVRkVOH{{Q+#*>*0T|bOLPm5`&Y8V2Yi>g^KPN zX5@p^(Jj#Hr#Kw!aq5lv2eih~?Rvc+fcK$)PYeYd;VK7+@Q1p}=OYq-1E=Vij93$U zAAWhMc7T)8BKA%V<;o53LpP*C#ndlKYCgE0$l+v`#coJ&y{BTvh-#Or2=|$?I*aU`6JF?<4=SAWeFK9_+|L95De`Rg9_)3R%{C|VE*13Z z);kvLeMna$={TI-YOQm~Lkb62boZ1%A`6O3wf!nb!xCq6r^a1fdlaHOb$?pn<9IC5(IxCQjB=dn z@QiX?Vk0ch%z>}oqa4?56(-s!jbaDG!j5fXj&j`Z+RCGsxmX+Y@8vjwENWuaE6Y5V z!|_*R_(oHZc(M!7 X*~oMT(NDW8NoRx(@{AbBGx5O>1^_-H literal 0 HcmV?d00001 diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.7-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.7-snapshot new file mode 100644 index 0000000000000000000000000000000000000000..38bbc7ea3b0971e6f090ef7a866875412bc99a5d GIT binary patch literal 6357 zcmeHL&2G~`5FWcJ5}-nXigMtK~yaRk8~L7Cl6W1BcMZH=fze2I5A{yaPL?ab!^=)RPp_M9Sr$G<6% zoti4tYW7(>H#=chbA{^JLasVJF_D{^F3il;?3vlQ$r*qX&+w6b{4o{EKCNq3)2b~I z&2}B{0S+9k<~JI?r?IsrX)(tQw5k;l&1BT^>L#I%uK!OT4aUzwML$_ql_z{yKOkGlj&>bWt^q9PR)fHF2PBY;62&(YB#|Fh34AKlk+H%Xi1Wzu^8Acs=9QWmw`F zirPovLVsHnNO8kVJee+GzPI-3`Rfnbs?H632_xsAV_Hud#|Q)8HSJTFr-4o)DKE(F}Rk<;bfM@PFglL+AI#pUv=lFm%IFL z9av?D2X^dQ@v3ev78kmz^yU6sw`ftTK^UPf34(YIl!Y}csPMH-x=UTq0i(ofV9peB zM@kJol6Qet#=3o52#{=8r9_VD)Yg=9bxIUeAg#@wu5j@}D*>|@$v3HALx-b}Ds5pV zR=F$-g=n(;y0wM({U)In^QrVY4gHiQ literal 0 HcmV?d00001 diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.7-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.7-snapshot new file mode 100644 index 0000000000000000000000000000000000000000..df7866b61641dd2e05fc47e131566b6633e6801c GIT binary patch literal 21981 zcmeHP&2Jk;6o0mpex)TJl%`0i;@~1UWbL-0NeNPM6Q`KEX%r`o`XPmlJ$5#=cbD05 z>hutm1EL51fc^&v#HDbc+=@Wr%#|B|0D==7DiPky?t0hRb>k$qaMH1q-R#cne7!gC zy?MVk0}%X0V$QpWeV_j>0`UvUYJP<%#j>W~!v{@SRo7I7t(&CEw6dkps?M|uQSQ-oL%D{R z9HE+8)*cWVZ+x6I^rBYE>8fe180)8-~0i+7KO`utdF-Ef3YYW&cf*e zDXP^n^O`eRRxOKI3|`OjJ|uYi6ItvQDSW6}E7w(XA`3%-&#X^i10q=%2tgXlCID;sKITFos%ozHtv1$lHQg}a5 z$_r)y=XO7TeDwIeNBUZ1ZsXCR#CM4`h zm=Kc?AGCx`!9==7@<L#H7Bh!4 zoS6LyXZ9zy_PyQ`LK_R7sF`cD9n`VeL|kj8X>L$h*bA|mHrTCa2sx3$D}t^098=PI zwc<}S8T2j88ndwY=iorX+_NZ&rP)bBcYUh=!_I*laYCJ`$x-J;}IF8GXy1?a; z;!>PYD?U}NRMt~ilg))(APHi-t;9PH6;^Z0*Rs|GduTPB=w3pwstwNMLkxN+v#DIJ z21q9UW)RyEX%h=X+fq&&goPO6)+ydON=G_L-5B)SqH-i}=#1#ha(F8O1J}~G(o;*h z`P6)RY5vx18jj4RrxqqNGgmQB6b5dlvJ2^@TzW2(%4Tk-=fvC0PB?MCYWjs_GkfC5 zhGS1m?4fYCUt9<7&@$Q5%qyxzywZ9y(5q67t!Lm+ndl|9at8*A8YvgNnS>U^*qw0k z>&Sgi0CQpOeTkC(7~`WPCS%8J0#3Tk&CP$S)M;qYO5Dc8n7H)m8(;o#>iJ_~WAO62 z+x|P+%>Kg!pK9mSXqWwmCAVPzVJ>0+345Z(uRYyju^o3TwurFH#8Sku<=m8*Na^9^ zmlm5}v`LGtF5&9dddnuJBW$7L7TeIlwAfAuEVf}3DdVBaG$7`eQQIjQ-M8#^$RSpf zVU|B%QH2-NsM~V5JS!7s=<8p7{`D`){WMn{Lf|18-U@AzOlSY@_daREuP~Azm1yzY)b`u<)Vd+4_w$`^Z2j#AAna*;NRZ>{g3d4^ms^* z$BsN6LH+6aLZtqzFWaTZV><_J_vz1<%Htv3={6|fdab29y(`@5Qn^TXI=8cht!iJS zJDr1<(w**LzB|Uv+TxPuta#wv>GneLcgJ1boh$5iV8}?zD9nVeq{CPK4Gx5O!;>37 zAj;hJknth!l-nS(OSi$vnbh@kZZg?Gi9`Zt$5Ts`$VN!!f zKA?MQ$pObiDu>3rIGk|!>*hA2??ou^QKufz6W{XO3=2EF+ElnS(z$6(g06zHJ-6Y6 zR;g4OUu$#rszX>=IB}1xd#z(|akuc*t2(l*>{dJ3~2j5Lxp%eomG$PYmDfTp+auZ zlp`uT4?OiBOyKFY0Pu7YMM}GrwoRSRSSUjLVFe+$Pd~!j1CQ-A8R_cT2qHp-xZc1P z^bdyMHrGGxFb=v!eYxn4bj?ZEoOI3A_ZE1fV_&)EB<>?|pDx@+A(e|%E>gMdcjY3t zQ3M>Q+-Ysc{h^0aY*MO6knUu|Oh>m-*bBv%&;=g_31x(GYybvpgl`X*K!55@n(hMoMaoznNzz7Lyrvir4@Y$sx&b~a~Yhxd_FmRF>6{D=2l UAcwm-#L3e4)Xwq5x#8>Bf81XA5C8xG literal 0 HcmV?d00001 diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.7-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.7-snapshot new file mode 100644 index 0000000000000000000000000000000000000000..6fe289640b39b2a2768a1a5084601ae02ab42718 GIT binary patch literal 6329 zcmeHLL2nX46n-qNQ5$Qm#(MDJfqG+Cv{+0JhPJCAVr>eoQ4eIG1FSB)I6Jh~L!&4E zf>%#Q<6XU*_&ZEIYV>YmeY3N$Y*`dr8cAf5(w!aVf&d7D*KQYWP-B3 ztaW=cUa~DtE$#?&u|;6whJHt1EadX(yk5xPS=Qn3ioUqIkXgEp;|ffyrL(JgA*Zio z(%H;ieZ_s73AN@v*&uCRvWCTt8ddX+x=DRM7i_C+RGQ8`&VBgq8UoS8o3AkHFbH?G zH*ka!EusKCVsLdIX3k@bB$z-O$-pRK2H(oSA(L7aUcCnsWrLdQzGDuD*?U?1Nlh|9 za~XggLz4{9S`5(g$OV7qTo#UUu}TXjaS-N2R(PwK#HwiSt2uOLCe)b@ZUvEj58zQ9 zJIc<^&d&z(4DErcLAbdqPtU%5fBMIBvA!-|*LZap7E1bd`xwmk=Aw%fG%SZ($)w!2 zwq8AZ{ZZZ2g@F%Y8%EH;)L+16Ebhmp-8unwWP`!^P%z0(5kg7#i~dw43rhI{hsm~} z-YVXwC5~ZN$_W8Hj`5xD3E0P#0TAaE`s%hXNc?Bm#lS?xJ2Ce8x0`AJg48c!59-+~ z!|=%qLtm)a_l?rG7%YWyIGkmHofZr9n#lqAtL%OEiX}g&4IA0xhCQ{EzNzPy(+eFb z^?Gkhw@etRQBIjj9Vd7O%Eam?RQTqR?pg=5!x<$tJTqN=BDI0&$tOq6W8S{)Du!72 zX*j49j6lDfoh}IR!U_&2oN9GumvF&xjS8NUaz}-Z@|9z_R$BYR>f3e72)CJBb>^>k zOhlor-%U6v<6RGzV4S=D5_%z;&^dSEm{F@WI4PP`(5t0)vT*DH-SVyDaCxs<=cI)c zc2Fg^Ng$C0#irW7%88)F*%@?MqM(8OVrnaRQt^z6NaEjaRj>^(GU<&CRlz^Lk;xGK zhX=F*!$jAS5JHOt!Q?JCxxCnli&y^=-MK)bqZ?Qr80I)xk{RZ>(8^VinO(2m!yMPD z?S-yUKg5ok=LfXvILvW>ssj%%bAdAG!R0uCEGzp$6-S?lK8(E`#T!k+eQte>y?f$5 zPuXJ7E8~s5ljBo$IVfi~YkYDHyf1mzL4S&i(*$dZ^46TdX!9x=7)^+;2rHw%0I$p( AGynhq literal 0 HcmV?d00001 diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.7-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.7-snapshot new file mode 100644 index 0000000000000000000000000000000000000000..a6d1863eb5887dfc0e0d394e064b1daeafd29603 GIT binary patch literal 6270 zcmeHLO-~a+7=CvF<3~V5BOW|>DBilIfPmp(3aq5DBB2EX9GYo6EeqS-Iy>d-K=j04 z@aoBE`~iA5@pqVb)aczrd}pV0yL2lRN^D{$A#`Rk&%Phe%=^p$)HkK8GN#D?gRc|_ z4d=;3Za6m<9-(8|@YrZ}BAm%&X!zcEE;Eu2U^< z8rCLu3@vLHi?*fl-4ZQx!*sL^aj2H$%&-bc$_!!}dz6Ka4W7=U|Kj%R4>y0l6!CTO8s@9cut3t++Q(qJITu|dzhOz-N+#vD zwfpAa?I&$p7Y06qedtC5C;tLAZgDR!wblu+OIr-?27*a;iWEw^k9bSf1gQC84g)nw zeUjOrS&nH}^HBlZhxr|93fRMy2@vNM`s%i?Nc;orVq&_)n3((W$4%7+Nop0b+x6^~ zY4}`8LrbXG@{Q869LxuDIGAO=ofh)-j*0{FS3Uji6(K*b4Xd1T!`_&WJ=T*8v6;G( zI@et276>Cn$|*Ccwf6+aGS}fGcVpD zk%gLmBjKd>H9TDWd2aY6^g=YDb5^0(C>G0{WK1gPRn3?PpfsVTS#FC zRdS645>-%ash7h7@hYF(kb<12J@3!eu%HBLUsbZT7S%axy*4X@sv8drVW z3ye`K#Ex6%+w|43Q{zs02kuOcRh+;QY+k>^X$IiCvT{AmQ zNe@-qOAmYjZXj_${P_TIMM!)Mgv1d92P6&%!kd|0uQ%SLi(`XJccsKTfBfdny!Yn) z<~Ifyev}!%PLZFBUnr0(7Mu0rO0${2+$T2X4l(wnUK7G0?CktY{Tt2ToQj2n15Tpsb8&Cs->!4tKQtw;e~De>h+EC<~3|r zVScA%T-R$AeY;#T%D41wQM>gIbTHh=+@i(E%+^Mh3gt$cATi56&%SX%ysrZd-C~r+C5zgEG82el_gb@I(4~n zW1H-pT#J0ebr^A&7mO6+i5a6@sbc$4URX*tGZABk<*{(d2i#79b2t7JGvu=@$BMjT z?wJEYOEG}ZkXp6ZwTJv95FPr<%#!af@sXQMc(6>;rf;4=%`3R&gwir)$ePq*`(-#~ z6Q|8~Z^L}cB6c%eEsw$Lw+$S!AUCvv*wC&a@m>~*v|OYVBnNanb@b@vOFpT9RYnZh6o3)>Yyhx@4xw#? zig_Q1dE~g$!2bk;slOeYE>aAZY=4V_ror?nFw+Du*t2lP5^#+Fo1N6I{L+ZKvdqx| zjQSn3&Ibsg9s^|0qwzS5dJfOBDx{piIKq8=oIvV|PvZ$wdMZN69nL&sbH(Q#T7U23 zq}8DvB4E8dW-_AfxK?u$8B-4_?g0*L&@j__np0rZ$t~76??-E52?|~6HjpevC|&cR zv}omwW_X7X&!6uMt!|V#cDsa{%%zesrXdxV;|%gZS#s0Cu`=Ww)w8}GhtK5+M+z;O zxdmsePN&Drx=lC&>{EpXoViQ(L+u!>j!F~0;tV?zhgAGLdUAqLL1cO`?Tj9$947#G z`6_17nsJ2M1Axqe-iblC2{$&b@tiJn~Dpl5#=zhhtNzX3A%&)EO~ literal 0 HcmV?d00001 diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.7-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.7-snapshot new file mode 100644 index 0000000000000000000000000000000000000000..e93debd2bf19d98d3372f71603624856c1474e43 GIT binary patch literal 9112 zcmeHN!EW0|5FOHTfCNeEIzdi71c=g0A<7nPxrPsg;xK_w)-^)aL6U>8l2*e-kUuC|rPQvh($doM!eWV3Xr)wLURYYLw3b(wwp##KZsIGa z#A_-vT-Mb{pR{+V*0C+;0S+v!?e==Eqw)Pd9dOI`v=;HGW^!gZU6V44*w!P;im%@= z9n$x9T<#;U9RnE(@X?;-Y`c4UZ?(O|wyJ=*`BRO&tnG-E819 z73Q{T#y5J?)Nj>mM*XgS%fDND2TKggr$opiML>7$=9B%cf z))@3{8iKC7PRHsFp&yw){wZN+7GM4jvmS%=l^YElp~R)A01p{_aSSuBq01$hKr7T? zmN1L&*Wr{+oi5+G2Xh^Z+S}p8JPs@O4E)NnBvjca6b`JiBvkp3P~`?vD|~a!fHPdI z(z;DNg!$MIvn|`i>6ZOlYdD#u)X5~M8b=Oi01x{(P!A3c{u{8s&=Hs)hg(~H^2v9v zF8%XDtgnxEq`wk|rN&UyJ_grDo1%{tH_XJyv?+I;{qLXs@RPQu3j@D}$5226KYs_< zx)b4l)H(rndY8ecsbG?w!iAFVmGD*70JXEgVLsy2uUq$No1@#+&Y}Q5kN&+n5^#ts z9U#&l&egwuL*hTfAv$J8+!N1!ePd8ffs?W#_OzyQ>4skdH)KM^%mXDeA8e#@IGts& zNGrxh!{UJaRmYEBvE-8qSY^zBy|_{PN;fxa>xn9TbF`(~B#iVZr_83F7tevRu!aQ{ z{s>5SBLVGkMoABIrZ0D-)DR>2$y zlma2xA}Mvd5RXgA8mUPqrNOddy|guu&l!ifFUQ26Gn2~7 zXUELSvPKVO)ZbWjo}+|%3NU-gk9RnUsZ@o$BxMoq*_9+o(-TzP~H{hgYQ=tG4TZIOkdqDR?>o`=$Z4+K_l4KH+O6*6E-yl@5J9sed9X%BdoPgXF zt2i5|y`d5uHq;&i6y}3xc|+|zpN-@y=xwJ}d5>>=;$sU)a5l! z9**WUF7*sK-WfXb)GDuWho|hRYm|l9!L%^7=l^+)%WK?Qev%T;B$KOg0(q#1#b>!^ I^M7^oKO-sO$N&HU literal 0 HcmV?d00001 From f817537f51d09ac618138e0d3aecf015bb22b55b Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Wed, 9 Jan 2019 11:00:43 +0100 Subject: [PATCH 088/147] [FLINK-11207][build] Bump commons-compress to 1.18 This addresses CVE-2018-11771. --- NOTICE-binary | 6 +++--- flink-dist/src/main/resources/META-INF/NOTICE | 2 +- .../src/main/resources/META-INF/NOTICE | 2 +- .../src/main/resources/META-INF/NOTICE | 2 +- pom.xml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NOTICE-binary b/NOTICE-binary index 0b10a27b9e9589..2619dbab859a64 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -64,7 +64,7 @@ This project bundles the following dependencies under the Apache Software Licens - commons-collections:commons-collections:3.2.2 - commons-io:commons-io:2.4 - org.apache.camel:camel-core:2.17.7 -- org.apache.commons:commons-compress:1.4.1 +- org.apache.commons:commons-compress:1.18 - org.apache.commons:commons-lang3:3.3.2 - org.apache.commons:commons-math3:3.5 - org.javassist:javassist:3.19.0-GA @@ -344,7 +344,7 @@ This project bundles the following dependencies under the Apache Software Licens - org.apache.htrace:htrace-core4:4.0.1-incubating - org.apache.httpcomponents:httpclient:4.5.3 - org.apache.httpcomponents:httpcore:4.4.6 -- org.apache.commons:commons-compress:1.4.1 +- org.apache.commons:commons-compress:1.18 - org.apache.commons:commons-math3:3.5 - commons-beanutils:commons-beanutils:1.8.3 - commons-cli:commons-cli:1.3.1 @@ -2412,7 +2412,7 @@ This project bundles the following dependencies under the Apache Software Licens - commons-logging:commons-logging:1.1.3 - commons-net:commons-net:3.1 - org.apache.avro:avro:1.8.2 -- org.apache.commons:commons-compress:1.4.1 +- org.apache.commons:commons-compress:1.18 - org.apache.commons:commons-math3:3.5 - org.apache.zookeeper:zookeeper:3.4.10 - org.codehaus.jackson:jackson-core-asl:1.9.13 diff --git a/flink-dist/src/main/resources/META-INF/NOTICE b/flink-dist/src/main/resources/META-INF/NOTICE index 7088b1ab775bcb..442787a8b516d2 100644 --- a/flink-dist/src/main/resources/META-INF/NOTICE +++ b/flink-dist/src/main/resources/META-INF/NOTICE @@ -20,7 +20,7 @@ This project bundles the following dependencies under the Apache Software Licens - commons-collections:commons-collections:3.2.2 - commons-io:commons-io:2.4 - org.apache.camel:camel-core:2.17.7 -- org.apache.commons:commons-compress:1.4.1 +- org.apache.commons:commons-compress:1.18 - org.apache.commons:commons-lang3:3.3.2 - org.apache.commons:commons-math3:3.5 - org.javassist:javassist:3.19.0-GA diff --git a/flink-filesystems/flink-swift-fs-hadoop/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-swift-fs-hadoop/src/main/resources/META-INF/NOTICE index 1d4bcc1601f627..9f2c6350cf3be7 100644 --- a/flink-filesystems/flink-swift-fs-hadoop/src/main/resources/META-INF/NOTICE +++ b/flink-filesystems/flink-swift-fs-hadoop/src/main/resources/META-INF/NOTICE @@ -16,7 +16,7 @@ This project bundles the following dependencies under the Apache Software Licens - org.apache.htrace:htrace-core4:4.0.1-incubating - org.apache.httpcomponents:httpclient:4.5.3 - org.apache.httpcomponents:httpcore:4.4.6 -- org.apache.commons:commons-compress:1.4.1 +- org.apache.commons:commons-compress:1.18 - org.apache.commons:commons-math3:3.5 - commons-beanutils:commons-beanutils:1.8.3 - commons-cli:commons-cli:1.3.1 diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/NOTICE b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/NOTICE index ad28a9bbcad64f..f6d007d94d15fc 100644 --- a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/NOTICE +++ b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/src/main/resources/META-INF/NOTICE @@ -19,7 +19,7 @@ This project bundles the following dependencies under the Apache Software Licens - commons-logging:commons-logging:1.1.3 - commons-net:commons-net:3.1 - org.apache.avro:avro:1.8.2 -- org.apache.commons:commons-compress:1.4.1 +- org.apache.commons:commons-compress:1.18 - org.apache.commons:commons-math3:3.5 - org.apache.zookeeper:zookeeper:3.4.10 - org.codehaus.jackson:jackson-core-asl:1.9.13 diff --git a/pom.xml b/pom.xml index 60d9a988f9f098..46a3a85f445e91 100644 --- a/pom.xml +++ b/pom.xml @@ -429,7 +429,7 @@ under the License. org.apache.commons commons-compress - 1.4.1 + 1.18 From 0315f2066c0b56a7ebfe1e8cd8c8b2eca9acca9c Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 12 Sep 2018 16:13:55 +0200 Subject: [PATCH 089/147] [FLINK-10326] Simplify ZooKeeperSubmittedJobGraphStore#constructor Move initialization logic out of the ZooKeeperSubmittedJobGraphStore constructor. --- .../ZooKeeperSubmittedJobGraphStore.java | 36 ++++---------- .../flink/runtime/util/ZooKeeperUtils.java | 21 +++++++-- .../ZooKeeperSubmittedJobGraphStoreTest.java | 8 ++-- ...ooKeeperSubmittedJobGraphsStoreITCase.java | 47 ++++++++++++------- 4 files changed, 61 insertions(+), 51 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStore.java index a0301843a6d4fe..81584293e74d09 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStore.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStore.java @@ -20,7 +20,6 @@ import org.apache.flink.api.common.JobID; import org.apache.flink.runtime.state.RetrievableStateHandle; -import org.apache.flink.runtime.zookeeper.RetrievableStateStorageHelper; import org.apache.flink.runtime.zookeeper.ZooKeeperStateHandleStore; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; @@ -69,9 +68,6 @@ public class ZooKeeperSubmittedJobGraphStore implements SubmittedJobGraphStore { /** Lock to synchronize with the {@link SubmittedJobGraphListener}. */ private final Object cacheLock = new Object(); - /** Client (not a namespace facade). */ - private final CuratorFramework client; - /** The set of IDs of all added job graphs. */ private final Set addedJobGraphs = new HashSet<>(); @@ -96,34 +92,20 @@ public class ZooKeeperSubmittedJobGraphStore implements SubmittedJobGraphStore { /** * Submitted job graph store backed by ZooKeeper. * - * @param client ZooKeeper client - * @param currentJobsPath ZooKeeper path for current job graphs - * @param stateStorage State storage used to persist the submitted jobs - * @throws Exception + * @param zooKeeperFullBasePath ZooKeeper path for current job graphs + * @param zooKeeperStateHandleStore State storage used to persist the submitted jobs */ public ZooKeeperSubmittedJobGraphStore( - CuratorFramework client, - String currentJobsPath, - RetrievableStateStorageHelper stateStorage) throws Exception { - - checkNotNull(currentJobsPath, "Current jobs path"); - checkNotNull(stateStorage, "State storage"); - - // Keep a reference to the original client and not the namespace facade. The namespace - // facade cannot be closed. - this.client = checkNotNull(client, "Curator client"); - - // Ensure that the job graphs path exists - client.newNamespaceAwareEnsurePath(currentJobsPath) - .ensure(client.getZookeeperClient()); + String zooKeeperFullBasePath, + ZooKeeperStateHandleStore zooKeeperStateHandleStore, + PathChildrenCache pathCache) { - // All operations will have the path as root - CuratorFramework facade = client.usingNamespace(client.getNamespace() + currentJobsPath); + checkNotNull(zooKeeperFullBasePath, "Current jobs path"); - this.zooKeeperFullBasePath = client.getNamespace() + currentJobsPath; - this.jobGraphsInZooKeeper = new ZooKeeperStateHandleStore<>(facade, stateStorage); + this.zooKeeperFullBasePath = zooKeeperFullBasePath; + this.jobGraphsInZooKeeper = checkNotNull(zooKeeperStateHandleStore); - this.pathCache = new PathChildrenCache(facade, "/", false); + this.pathCache = checkNotNull(pathCache); pathCache.getListenable().addListener(new SubmittedJobGraphsPathCacheListener()); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java index cc1ec7044c45f3..039dcf41d8d999 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java @@ -33,6 +33,7 @@ import org.apache.flink.runtime.leaderelection.ZooKeeperLeaderElectionService; import org.apache.flink.runtime.leaderretrieval.ZooKeeperLeaderRetrievalService; import org.apache.flink.runtime.zookeeper.RetrievableStateStorageHelper; +import org.apache.flink.runtime.zookeeper.ZooKeeperStateHandleStore; import org.apache.flink.runtime.zookeeper.filesystem.FileSystemStateStorageHelper; import org.apache.flink.util.Preconditions; @@ -41,6 +42,7 @@ import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.api.ACLProvider; import org.apache.curator.framework.imps.DefaultACLProvider; +import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.data.ACL; @@ -244,10 +246,23 @@ public static ZooKeeperSubmittedJobGraphStore createSubmittedJobGraphs( // ZooKeeper submitted jobs root dir String zooKeeperSubmittedJobsPath = configuration.getString(HighAvailabilityOptions.HA_ZOOKEEPER_JOBGRAPHS_PATH); + // Ensure that the job graphs path exists + client.newNamespaceAwareEnsurePath(zooKeeperSubmittedJobsPath) + .ensure(client.getZookeeperClient()); + + // All operations will have the path as root + CuratorFramework facade = client.usingNamespace(client.getNamespace() + zooKeeperSubmittedJobsPath); + + final String zooKeeperFullSubmittedJobsPath = client.getNamespace() + zooKeeperSubmittedJobsPath; + + final ZooKeeperStateHandleStore zooKeeperStateHandleStore = new ZooKeeperStateHandleStore<>(facade, stateStorage); + + final PathChildrenCache pathCache = new PathChildrenCache(facade, "/", false); + return new ZooKeeperSubmittedJobGraphStore( - client, - zooKeeperSubmittedJobsPath, - stateStorage); + zooKeeperFullSubmittedJobsPath, + zooKeeperStateHandleStore, + pathCache); } /** diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStoreTest.java index fae84591b556b2..04b7792be6a2c8 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStoreTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStoreTest.java @@ -24,9 +24,11 @@ import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.util.ZooKeeperUtils; import org.apache.flink.runtime.zookeeper.ZooKeeperResource; +import org.apache.flink.runtime.zookeeper.ZooKeeperStateHandleStore; import org.apache.flink.util.TestLogger; import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -98,9 +100,9 @@ public void testJobGraphRemovalFailureAndLockRelease() throws Exception { @Nonnull public ZooKeeperSubmittedJobGraphStore createSubmittedJobGraphStore(CuratorFramework client, TestingRetrievableStateStorageHelper stateStorage) throws Exception { return new ZooKeeperSubmittedJobGraphStore( - client, - "/foobar", - stateStorage); + client.getNamespace(), + new ZooKeeperStateHandleStore<>(client, stateStorage), + new PathChildrenCache(client, "/", false)); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphsStoreITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphsStoreITCase.java index e9be145c37fa88..23c272528889b3 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphsStoreITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphsStoreITCase.java @@ -27,17 +27,22 @@ import org.apache.flink.runtime.state.RetrievableStreamStateHandle; import org.apache.flink.runtime.state.memory.ByteStreamStateHandle; import org.apache.flink.runtime.zookeeper.RetrievableStateStorageHelper; +import org.apache.flink.runtime.zookeeper.ZooKeeperStateHandleStore; import org.apache.flink.runtime.zookeeper.ZooKeeperTestEnvironment; import org.apache.flink.util.InstantiationUtil; import org.apache.flink.util.TestLogger; import akka.actor.ActorRef; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; +import javax.annotation.Nonnull; + import java.io.IOException; import java.util.Collection; import java.util.HashMap; @@ -59,9 +64,9 @@ */ public class ZooKeeperSubmittedJobGraphsStoreITCase extends TestLogger { - private final static ZooKeeperTestEnvironment ZooKeeper = new ZooKeeperTestEnvironment(1); + private static final ZooKeeperTestEnvironment ZooKeeper = new ZooKeeperTestEnvironment(1); - private final static RetrievableStateStorageHelper localStateStorage = new RetrievableStateStorageHelper() { + private static final RetrievableStateStorageHelper localStateStorage = new RetrievableStateStorageHelper() { @Override public RetrievableStateHandle store(SubmittedJobGraph state) throws IOException { ByteStreamStateHandle byteStreamStateHandle = new ByteStreamStateHandle( @@ -71,7 +76,6 @@ public RetrievableStateHandle store(SubmittedJobGraph state) } }; - @AfterClass public static void tearDown() throws Exception { if (ZooKeeper != null) { @@ -86,10 +90,7 @@ public void cleanUp() throws Exception { @Test public void testPutAndRemoveJobGraph() throws Exception { - ZooKeeperSubmittedJobGraphStore jobGraphs = new ZooKeeperSubmittedJobGraphStore( - ZooKeeper.createClient(), - "/testPutAndRemoveJobGraph", - localStateStorage); + ZooKeeperSubmittedJobGraphStore jobGraphs = createZooKeeperSubmittedJobGraphStore("/testPutAndRemoveJobGraph"); try { SubmittedJobGraphListener listener = mock(SubmittedJobGraphListener.class); @@ -142,10 +143,25 @@ public void testPutAndRemoveJobGraph() throws Exception { } } + @Nonnull + private ZooKeeperSubmittedJobGraphStore createZooKeeperSubmittedJobGraphStore(String fullPath) throws Exception { + final CuratorFramework client = ZooKeeper.getClient(); + // Ensure that the job graphs path exists + client.newNamespaceAwareEnsurePath(fullPath).ensure(client.getZookeeperClient()); + + // All operations will have the path as root + CuratorFramework facade = client.usingNamespace(client.getNamespace() + fullPath); + return new ZooKeeperSubmittedJobGraphStore( + fullPath, + new ZooKeeperStateHandleStore<>( + facade, + localStateStorage), + new PathChildrenCache(facade, "/", false)); + } + @Test public void testRecoverJobGraphs() throws Exception { - ZooKeeperSubmittedJobGraphStore jobGraphs = new ZooKeeperSubmittedJobGraphStore( - ZooKeeper.createClient(), "/testRecoverJobGraphs", localStateStorage); + ZooKeeperSubmittedJobGraphStore jobGraphs = createZooKeeperSubmittedJobGraphStore("/testRecoverJobGraphs"); try { SubmittedJobGraphListener listener = mock(SubmittedJobGraphListener.class); @@ -195,12 +211,9 @@ public void testConcurrentAddJobGraph() throws Exception { ZooKeeperSubmittedJobGraphStore otherJobGraphs = null; try { - jobGraphs = new ZooKeeperSubmittedJobGraphStore( - ZooKeeper.createClient(), "/testConcurrentAddJobGraph", localStateStorage); - - otherJobGraphs = new ZooKeeperSubmittedJobGraphStore( - ZooKeeper.createClient(), "/testConcurrentAddJobGraph", localStateStorage); + jobGraphs = createZooKeeperSubmittedJobGraphStore("/testConcurrentAddJobGraph"); + otherJobGraphs = createZooKeeperSubmittedJobGraphStore("/testConcurrentAddJobGraph"); SubmittedJobGraph jobGraph = createSubmittedJobGraph(new JobID(), 0); SubmittedJobGraph otherJobGraph = createSubmittedJobGraph(new JobID(), 0); @@ -254,11 +267,9 @@ public Void answer(InvocationOnMock invocation) throws Throwable { @Test(expected = IllegalStateException.class) public void testUpdateJobGraphYouDidNotGetOrAdd() throws Exception { - ZooKeeperSubmittedJobGraphStore jobGraphs = new ZooKeeperSubmittedJobGraphStore( - ZooKeeper.createClient(), "/testUpdateJobGraphYouDidNotGetOrAdd", localStateStorage); + ZooKeeperSubmittedJobGraphStore jobGraphs = createZooKeeperSubmittedJobGraphStore("/testUpdateJobGraphYouDidNotGetOrAdd"); - ZooKeeperSubmittedJobGraphStore otherJobGraphs = new ZooKeeperSubmittedJobGraphStore( - ZooKeeper.createClient(), "/testUpdateJobGraphYouDidNotGetOrAdd", localStateStorage); + ZooKeeperSubmittedJobGraphStore otherJobGraphs = createZooKeeperSubmittedJobGraphStore("/testUpdateJobGraphYouDidNotGetOrAdd"); jobGraphs.start(null); otherJobGraphs.start(null); From 1e2aa8e9f35e7a943a4ed56a47834ee50bab3b47 Mon Sep 17 00:00:00 2001 From: ZILI CHEN Date: Wed, 9 Jan 2019 18:33:36 +0800 Subject: [PATCH 090/147] [FLINK-10436] Add ConfigOption#withFallbackKeys (#6872) * [FLINK-10436] Add ConfigOption#withFallbackKeys * [hotfix] correct javadoc * [FLINK-10436] Code quality improvement --- .../flink/configuration/ConfigOption.java | 89 ++++++++++++------- .../flink/configuration/Configuration.java | 37 ++++---- .../DelegatingConfiguration.java | 12 +-- .../flink/configuration/FallbackKey.java | 80 +++++++++++++++++ .../flink/configuration/RestOptions.java | 2 +- .../configuration/ConfigurationTest.java | 33 +++++++ 6 files changed, 198 insertions(+), 55 deletions(-) create mode 100644 flink-core/src/main/java/org/apache/flink/configuration/FallbackKey.java diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigOption.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigOption.java index 9e6fc324670344..d5fb5551767d06 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigOption.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigOption.java @@ -39,7 +39,7 @@ @PublicEvolving public class ConfigOption { - private static final String[] EMPTY = new String[0]; + private static final FallbackKey[] EMPTY = new FallbackKey[0]; // ------------------------------------------------------------------------ @@ -47,7 +47,7 @@ public class ConfigOption { private final String key; /** The list of deprecated keys, in the order to be checked. */ - private final String[] deprecatedKeys; + private final FallbackKey[] fallbackKeys; /** The default value for this option. */ private final T defaultValue; @@ -58,52 +58,70 @@ public class ConfigOption { // ------------------------------------------------------------------------ /** - * Creates a new config option with no deprecated keys. + * Creates a new config option with no fallback keys. * - * @param key The current key for that config option - * @param defaultValue The default value for this option + * @param key The current key for that config option + * @param defaultValue The default value for this option */ ConfigOption(String key, T defaultValue) { this.key = checkNotNull(key); this.description = Description.builder().text("").build(); this.defaultValue = defaultValue; - this.deprecatedKeys = EMPTY; + this.fallbackKeys = EMPTY; } /** - * Creates a new config option with deprecated keys. + * Creates a new config option with fallback keys. * - * @param key The current key for that config option - * @param description Description for that option - * @param defaultValue The default value for this option - * @param deprecatedKeys The list of deprecated keys, in the order to be checked + * @param key The current key for that config option + * @param description Description for that option + * @param defaultValue The default value for this option + * @param fallbackKeys The list of fallback keys, in the order to be checked * @deprecated use version with {@link Description} instead */ @Deprecated - ConfigOption(String key, String description, T defaultValue, String... deprecatedKeys) { + ConfigOption(String key, String description, T defaultValue, FallbackKey... fallbackKeys) { this.key = checkNotNull(key); this.description = Description.builder().text(description).build(); this.defaultValue = defaultValue; - this.deprecatedKeys = deprecatedKeys == null || deprecatedKeys.length == 0 ? EMPTY : deprecatedKeys; + this.fallbackKeys = fallbackKeys == null || fallbackKeys.length == 0 ? EMPTY : fallbackKeys; } /** - * Creates a new config option with deprecated keys. + * Creates a new config option with fallback keys. * - * @param key The current key for that config option - * @param description Description for that option - * @param defaultValue The default value for this option - * @param deprecatedKeys The list of deprecated keys, in the order to be checked + * @param key The current key for that config option + * @param description Description for that option + * @param defaultValue The default value for this option + * @param fallbackKeys The list of fallback keys, in the order to be checked */ - ConfigOption(String key, Description description, T defaultValue, String... deprecatedKeys) { + ConfigOption(String key, Description description, T defaultValue, FallbackKey... fallbackKeys) { this.key = checkNotNull(key); this.description = description; this.defaultValue = defaultValue; - this.deprecatedKeys = deprecatedKeys == null || deprecatedKeys.length == 0 ? EMPTY : deprecatedKeys; + this.fallbackKeys = fallbackKeys == null || fallbackKeys.length == 0 ? EMPTY : fallbackKeys; } // ------------------------------------------------------------------------ + /** + * Creates a new config option, using this option's key and default value, and + * adding the given fallback keys. + * + *

When obtaining a value from the configuration via {@link Configuration#getValue(ConfigOption)}, + * the fallback keys will be checked in the order provided to this method. The first key for which + * a value is found will be used - that value will be returned. + * + * @param fallbackKeys The fallback keys, in the order in which they should be checked. + * @return A new config options, with the given fallback keys. + */ + public ConfigOption withFallbackKeys(String... fallbackKeys) { + FallbackKey[] fallbackKeyArray = Arrays.stream(fallbackKeys) + .map(FallbackKey::createFallbackKey) + .toArray(FallbackKey[]::new); + return new ConfigOption<>(key, description, defaultValue, fallbackKeyArray); + } + /** * Creates a new config option, using this option's key and default value, and * adding the given deprecated keys. @@ -116,7 +134,10 @@ public class ConfigOption { * @return A new config options, with the given deprecated keys. */ public ConfigOption withDeprecatedKeys(String... deprecatedKeys) { - return new ConfigOption<>(key, description, defaultValue, deprecatedKeys); + FallbackKey[] fallbackKeys = Arrays.stream(deprecatedKeys) + .map(FallbackKey::createDeprecatedKey) + .toArray(FallbackKey[]::new); + return new ConfigOption<>(key, description, defaultValue, fallbackKeys); } /** @@ -138,7 +159,7 @@ public ConfigOption withDescription(final String description) { * @return A new config option, with given description. */ public ConfigOption withDescription(final Description description) { - return new ConfigOption<>(key, description, defaultValue, deprecatedKeys); + return new ConfigOption<>(key, description, defaultValue, fallbackKeys); } // ------------------------------------------------------------------------ @@ -168,19 +189,19 @@ public T defaultValue() { } /** - * Checks whether this option has deprecated keys. - * @return True if the option has deprecated keys, false if not. + * Checks whether this option has fallback keys. + * @return True if the option has fallback keys, false if not. */ - public boolean hasDeprecatedKeys() { - return deprecatedKeys != EMPTY; + public boolean hasFallbackKeys() { + return fallbackKeys != EMPTY; } /** - * Gets the deprecated keys, in the order to be checked. - * @return The option's deprecated keys. + * Gets the fallback keys, in the order to be checked. + * @return The option's fallback keys. */ - public Iterable deprecatedKeys() { - return deprecatedKeys == EMPTY ? Collections.emptyList() : Arrays.asList(deprecatedKeys); + public Iterable fallbackKeys() { + return (fallbackKeys == EMPTY) ? Collections.emptyList() : Arrays.asList(fallbackKeys); } /** @@ -201,7 +222,7 @@ public boolean equals(Object o) { else if (o != null && o.getClass() == ConfigOption.class) { ConfigOption that = (ConfigOption) o; return this.key.equals(that.key) && - Arrays.equals(this.deprecatedKeys, that.deprecatedKeys) && + Arrays.equals(this.fallbackKeys, that.fallbackKeys) && (this.defaultValue == null ? that.defaultValue == null : (that.defaultValue != null && this.defaultValue.equals(that.defaultValue))); } @@ -213,13 +234,13 @@ else if (o != null && o.getClass() == ConfigOption.class) { @Override public int hashCode() { return 31 * key.hashCode() + - 17 * Arrays.hashCode(deprecatedKeys) + + 17 * Arrays.hashCode(fallbackKeys) + (defaultValue != null ? defaultValue.hashCode() : 0); } @Override public String toString() { - return String.format("Key: '%s' , default: %s (deprecated keys: %s)", - key, defaultValue, Arrays.toString(deprecatedKeys)); + return String.format("Key: '%s' , default: %s (fallback keys: %s)", + key, defaultValue, Arrays.toString(fallbackKeys)); } } diff --git a/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java b/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java index 00c4c38e820094..07c0290d66c3bc 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java @@ -701,12 +701,11 @@ public boolean contains(ConfigOption configOption) { if (this.confData.containsKey(configOption.key())) { return true; } - else if (configOption.hasDeprecatedKeys()) { - // try the deprecated keys - for (String deprecatedKey : configOption.deprecatedKeys()) { - if (this.confData.containsKey(deprecatedKey)) { - LOG.warn("Config uses deprecated configuration key '{}' instead of proper key '{}'", - deprecatedKey, configOption.key()); + else if (configOption.hasFallbackKeys()) { + // try the fallback keys + for (FallbackKey fallbackKey : configOption.fallbackKeys()) { + if (this.confData.containsKey(fallbackKey.getKey())) { + loggingFallback(fallbackKey, configOption); return true; } } @@ -741,11 +740,10 @@ public boolean removeConfig(ConfigOption configOption){ // try the current key Object oldValue = this.confData.remove(configOption.key()); if (oldValue == null){ - for (String deprecatedKey : configOption.deprecatedKeys()){ - oldValue = this.confData.remove(deprecatedKey); + for (FallbackKey fallbackKey : configOption.fallbackKeys()){ + oldValue = this.confData.remove(fallbackKey.getKey()); if (oldValue != null){ - LOG.warn("Config uses deprecated configuration key '{}' instead of proper key '{}'", - deprecatedKey, configOption.key()); + loggingFallback(fallbackKey, configOption); return true; } } @@ -789,13 +787,12 @@ private Object getRawValueFromOption(ConfigOption configOption) { // found a value for the current proper key return o; } - else if (configOption.hasDeprecatedKeys()) { + else if (configOption.hasFallbackKeys()) { // try the deprecated keys - for (String deprecatedKey : configOption.deprecatedKeys()) { - Object oo = getRawValue(deprecatedKey); + for (FallbackKey fallbackKey : configOption.fallbackKeys()) { + Object oo = getRawValue(fallbackKey.getKey()); if (oo != null) { - LOG.warn("Config uses deprecated configuration key '{}' instead of proper key '{}'", - deprecatedKey, configOption.key()); + loggingFallback(fallbackKey, configOption); return oo; } } @@ -809,6 +806,16 @@ private Object getValueOrDefaultFromOption(ConfigOption configOption) { return o != null ? o : configOption.defaultValue(); } + private void loggingFallback(FallbackKey fallbackKey, ConfigOption configOption) { + if (fallbackKey.isDeprecated()) { + LOG.warn("Config uses deprecated configuration key '{}' instead of proper key '{}'", + fallbackKey.getKey(), configOption.key()); + } else { + LOG.info("Config uses fallback configuration key '{}' instead of key '{}'", + fallbackKey.getKey(), configOption.key()); + } + } + // -------------------------------------------------------------------------------------------- // Type conversion // -------------------------------------------------------------------------------------------- diff --git a/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java b/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java index 1a637f65626ceb..0a0a7773e2b09d 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java @@ -32,6 +32,8 @@ import java.util.Properties; import java.util.Set; +import static org.apache.flink.configuration.FallbackKey.createDeprecatedKey; + /** * A configuration that manages a subset of keys with a common prefix from a given configuration. */ @@ -361,17 +363,17 @@ public boolean equals(Object obj) { private static ConfigOption prefixOption(ConfigOption option, String prefix) { String key = prefix + option.key(); - List deprecatedKeys; - if (option.hasDeprecatedKeys()) { + List deprecatedKeys; + if (option.hasFallbackKeys()) { deprecatedKeys = new ArrayList<>(); - for (String dk : option.deprecatedKeys()) { - deprecatedKeys.add(prefix + dk); + for (FallbackKey dk : option.fallbackKeys()) { + deprecatedKeys.add(createDeprecatedKey(prefix + dk.getKey())); } } else { deprecatedKeys = Collections.emptyList(); } - String[] deprecated = deprecatedKeys.toArray(new String[deprecatedKeys.size()]); + FallbackKey[] deprecated = deprecatedKeys.toArray(new FallbackKey[0]); return new ConfigOption<>(key, option.description(), option.defaultValue(), diff --git a/flink-core/src/main/java/org/apache/flink/configuration/FallbackKey.java b/flink-core/src/main/java/org/apache/flink/configuration/FallbackKey.java new file mode 100644 index 00000000000000..6c67727ce94be3 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/configuration/FallbackKey.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.configuration; + +/** + * A key with FallbackKeys will fall back to the FallbackKeys if it itself is not configured. + */ +public class FallbackKey { + + // ------------------------- + // Factory methods + // ------------------------- + + static FallbackKey createFallbackKey(String key) { + return new FallbackKey(key, false); + } + + static FallbackKey createDeprecatedKey(String key) { + return new FallbackKey(key, true); + } + + // ------------------------------------------------------------------------ + + private final String key; + + private final boolean isDeprecated; + + public String getKey() { + return key; + } + + public boolean isDeprecated() { + return isDeprecated; + } + + private FallbackKey(String key, boolean isDeprecated) { + this.key = key; + this.isDeprecated = isDeprecated; + } + + // ------------------------------------------------------------------------ + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (o != null && o.getClass() == FallbackKey.class) { + FallbackKey that = (FallbackKey) o; + return this.key.equals(that.key) && (this.isDeprecated == that.isDeprecated); + } else { + return false; + } + } + + @Override + public int hashCode() { + return 31 * key.hashCode() + (isDeprecated ? 1 : 0); + } + + @Override + public String toString() { + return String.format("{key=%s, isDeprecated=%s}", key, isDeprecated); + } +} diff --git a/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java index 5c1b6d681fab49..d4f5a740a76858 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java @@ -43,7 +43,7 @@ public class RestOptions { public static final ConfigOption ADDRESS = key("rest.address") .noDefaultValue() - .withDeprecatedKeys(JobManagerOptions.ADDRESS.key()) + .withFallbackKeys(JobManagerOptions.ADDRESS.key()) .withDescription("The address that should be used by clients to connect to the server."); /** diff --git a/flink-core/src/test/java/org/apache/flink/configuration/ConfigurationTest.java b/flink-core/src/test/java/org/apache/flink/configuration/ConfigurationTest.java index 3b98a4490891f9..d0d382fe6e47c4 100644 --- a/flink-core/src/test/java/org/apache/flink/configuration/ConfigurationTest.java +++ b/flink-core/src/test/java/org/apache/flink/configuration/ConfigurationTest.java @@ -305,6 +305,39 @@ public void testDeprecatedKeys() { assertEquals(-1, cfg.getInteger(notContained)); } + @Test + public void testFallbackKeys() { + Configuration cfg = new Configuration(); + cfg.setInteger("the-key", 11); + cfg.setInteger("old-key", 12); + cfg.setInteger("older-key", 13); + + ConfigOption matchesFirst = ConfigOptions + .key("the-key") + .defaultValue(-1) + .withFallbackKeys("old-key", "older-key"); + + ConfigOption matchesSecond = ConfigOptions + .key("does-not-exist") + .defaultValue(-1) + .withFallbackKeys("old-key", "older-key"); + + ConfigOption matchesThird = ConfigOptions + .key("does-not-exist") + .defaultValue(-1) + .withFallbackKeys("foo", "older-key"); + + ConfigOption notContained = ConfigOptions + .key("does-not-exist") + .defaultValue(-1) + .withFallbackKeys("not-there", "also-not-there"); + + assertEquals(11, cfg.getInteger(matchesFirst)); + assertEquals(12, cfg.getInteger(matchesSecond)); + assertEquals(13, cfg.getInteger(matchesThird)); + assertEquals(-1, cfg.getInteger(notContained)); + } + @Test public void testRemove(){ Configuration cfg = new Configuration(); From e5ed8c85139d1dfdb4b6a47a63ac36143a7b5c64 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Tue, 8 Jan 2019 15:02:43 +0100 Subject: [PATCH 091/147] [FLINK-11287] [rocksdb] RocksDBListState should be using registered serializer in state meta infos This closes #7434. --- .../flink/contrib/streaming/state/RocksDBListState.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java index 13f5559405e8bf..72a5bc60b20ca6 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java @@ -19,7 +19,6 @@ package org.apache.flink.contrib.streaming.state; import org.apache.flink.api.common.state.ListState; -import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.api.common.state.State; import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; @@ -77,7 +76,6 @@ class RocksDBListState * @param namespaceSerializer The serializer for the namespace. * @param valueSerializer The serializer for the state. * @param defaultValue The default value for the state. - * @param elementSerializer The serializer for elements of the list state. * @param backend The backend for which this state is bind to. */ private RocksDBListState( @@ -85,11 +83,12 @@ private RocksDBListState( TypeSerializer namespaceSerializer, TypeSerializer> valueSerializer, List defaultValue, - TypeSerializer elementSerializer, RocksDBKeyedStateBackend backend) { super(columnFamily, namespaceSerializer, valueSerializer, defaultValue, backend); - this.elementSerializer = elementSerializer; + + ListSerializer castedListSerializer = (ListSerializer) valueSerializer; + this.elementSerializer = castedListSerializer.getElementSerializer(); } @Override @@ -281,7 +280,6 @@ static IS create( registerResult.f1.getNamespaceSerializer(), (TypeSerializer>) registerResult.f1.getStateSerializer(), (List) stateDesc.getDefaultValue(), - ((ListStateDescriptor) stateDesc).getElementSerializer(), backend); } From cad0509a752be57eb995a95ccebf00012e443ed5 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Tue, 8 Jan 2019 17:41:38 +0100 Subject: [PATCH 092/147] [FLINK-11073] [state backends, tests] Activate ignored testKeyedListStateSerializerReconfiguration test in StateBackendMigrationTestBase This test was previously ignored due to 2 missing changes: - ListSerializerSnapshot was not respecting serializer reconfiguration (fixed by FLINK-11073), and - RocksDBListState did not use the correct registered state serializer in backend's state meta info (fixed by FLINK-11287) With those fixes in, the test can now be activated and expected to pass. --- .../flink/runtime/state/StateBackendMigrationTestBase.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java index 8d85e74aca7eca..b636ed08bd24a1 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendMigrationTestBase.java @@ -40,7 +40,6 @@ import org.apache.flink.util.TestLogger; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -192,7 +191,6 @@ public void testKeyedListStateMigration() throws Exception { } @Test - @Ignore("This currently doesn't pass because the ListSerializer doesn't respect the reconfigured case, yet.") public void testKeyedListStateSerializerReconfiguration() throws Exception { final String stateName = "test-name"; From 43a74977de27869e41e97878e821dcf363d81911 Mon Sep 17 00:00:00 2001 From: Yangze Guo Date: Wed, 9 Jan 2019 21:37:44 +0800 Subject: [PATCH 093/147] [hotfix][docs] Add missing config option descriptions --- docs/_includes/generated/common_section.html | 2 +- .../generated/core_configuration.html | 4 +- .../history_server_configuration.html | 2 +- .../generated/job_manager_configuration.html | 2 +- .../generated/metric_configuration.html | 8 +-- .../resource_manager_configuration.html | 2 +- .../generated/task_manager_configuration.html | 2 +- .../generated/web_configuration.html | 30 +++++----- .../flink/configuration/CoreOptions.java | 6 +- .../configuration/HistoryServerOptions.java | 3 +- .../configuration/JobManagerOptions.java | 3 +- .../flink/configuration/MetricOptions.java | 16 ++++-- .../configuration/ResourceManagerOptions.java | 11 +++- .../configuration/TaskManagerOptions.java | 8 ++- .../flink/configuration/WebOptions.java | 57 ++++++++++++------- 15 files changed, 98 insertions(+), 58 deletions(-) diff --git a/docs/_includes/generated/common_section.html b/docs/_includes/generated/common_section.html index 65804db2283d9c..1c6685b2f4c657 100644 --- a/docs/_includes/generated/common_section.html +++ b/docs/_includes/generated/common_section.html @@ -20,7 +20,7 @@

- + diff --git a/docs/_includes/generated/core_configuration.html b/docs/_includes/generated/core_configuration.html index 4366e8b246f82e..540290045a8309 100644 --- a/docs/_includes/generated/core_configuration.html +++ b/docs/_includes/generated/core_configuration.html @@ -25,12 +25,12 @@ - + - +
After found matching a b c1 c2 c3 d, the match process will not discard any result.
SKIP_TO_FIRST[b*]SKIP_TO_FIRST[c*] a b c1 c2 c3 d
c1 c2 c3 d
@@ -1383,7 +1382,7 @@ Pattern: `a b+` and sequence: `a b1 b2 b3` Then the results will be: 		After found matching a b1, the match process will not discard any result.
SKIP_TO_NEXT[b*]SKIP_TO_NEXT a b1
parallelism.default
 1Default parallelism for jobs.
taskmanager.numberOfTaskSlots
io.tmp.dirs
 'LOCAL_DIRS' on Yarn. '_FLINK_TMP_DIR' on Mesos. System.getProperty("java.io.tmpdir") in standalone.Directories for temporary files, separated by",", "|", or the system's java.io.File.pathSeparator.
parallelism.default
 1Default parallelism for jobs.
diff --git a/docs/_includes/generated/history_server_configuration.html b/docs/_includes/generated/history_server_configuration.html index 272606e2884b68..7691c0abe62a40 100644 --- a/docs/_includes/generated/history_server_configuration.html +++ b/docs/_includes/generated/history_server_configuration.html @@ -30,7 +30,7 @@
historyserver.web.refresh-interval
10000 - + The refresh interval for the HistoryServer web-frontend in milliseconds.
historyserver.web.ssl.enabled
diff --git a/docs/_includes/generated/job_manager_configuration.html b/docs/_includes/generated/job_manager_configuration.html index 99eec1dcec7afe..177c362fb8bf93 100644 --- a/docs/_includes/generated/job_manager_configuration.html +++ b/docs/_includes/generated/job_manager_configuration.html @@ -10,7 +10,7 @@
jobmanager.archive.fs.dir
(none) - + Dictionary for JobManager to store the archives of completed jobs.
jobmanager.execution.attempts-history-size
diff --git a/docs/_includes/generated/metric_configuration.html b/docs/_includes/generated/metric_configuration.html index 420bb7f1da2a50..39a76cef165cae 100644 --- a/docs/_includes/generated/metric_configuration.html +++ b/docs/_includes/generated/metric_configuration.html @@ -50,12 +50,12 @@
metrics.reporters
(none) - + An optional list of reporter names. If configured, only reporters whose name matches any of the names in the list will be started. Otherwise, all reporters that could be found in the configuration will be started.
metrics.scope.delimiter
"." - + Delimiter used to assemble the metric identifier.
metrics.scope.jm
@@ -90,12 +90,12 @@
metrics.system-resource
false - + Flag indicating whether Flink should report system resource metrics such as machine's CPU, memory or network usage.
metrics.system-resource-probing-interval
5000 - + Interval between probing of system resource metrics specified in milliseconds. Has an effect only when 'metrics.system-resource' is enabled. diff --git a/docs/_includes/generated/resource_manager_configuration.html b/docs/_includes/generated/resource_manager_configuration.html index 9243fcd3cb9777..3448aba8f52337 100644 --- a/docs/_includes/generated/resource_manager_configuration.html +++ b/docs/_includes/generated/resource_manager_configuration.html @@ -20,7 +20,7 @@
local.number-resourcemanager
1 - + The number of resource managers start.
resourcemanager.job.timeout
diff --git a/docs/_includes/generated/task_manager_configuration.html b/docs/_includes/generated/task_manager_configuration.html index 63bfb043ea44a6..2093531211d4b1 100644 --- a/docs/_includes/generated/task_manager_configuration.html +++ b/docs/_includes/generated/task_manager_configuration.html @@ -20,7 +20,7 @@
task.cancellation.timers.timeout
7500 - + Time we wait for the timers in milliseconds to finish all pending timer threads when the stream task is cancelled.
task.checkpoint.alignment.max-size
diff --git a/docs/_includes/generated/web_configuration.html b/docs/_includes/generated/web_configuration.html index dbc353dbd56775..6fb35e9284ec9a 100644 --- a/docs/_includes/generated/web_configuration.html +++ b/docs/_includes/generated/web_configuration.html @@ -10,77 +10,77 @@
web.access-control-allow-origin
"*" - + Access-Control-Allow-Origin header for all responses from the web-frontend.
web.address
(none) - + Address for runtime monitor web-frontend server.
web.backpressure.cleanup-interval
600000 - + Time, in milliseconds, after which cached stats are cleaned up if not accessed.
web.backpressure.delay-between-samples
50 - + Delay between stack trace samples to determine back pressure in milliseconds.
web.backpressure.num-samples
100 - + Number of stack trace samples to take to determine back pressure.
web.backpressure.refresh-interval
60000 - + Time, in milliseconds, after which available stats are deprecated and need to be refreshed (by resampling).
web.checkpoints.history
10 - + Number of checkpoints to remember for recent history.
web.history
5 - + Number of archived jobs for the JobManager.
web.log.path
(none) - + Path to the log file (may be in /log for standalone but under log directory when using YARN).
web.refresh-interval
3000 - + Refresh interval for the web-frontend in milliseconds.
web.ssl.enabled
true - + Flag indicating whether to override SSL support for the JobManager Web UI.
web.submit.enable
true - + Flag indicating whether jobs can be uploaded and run from the web-frontend.
web.timeout
10000 - + Timeout for asynchronous operations by the WebRuntimeMonitor in milliseconds.
web.tmpdir
System.getProperty("java.io.tmpdir") - + Flink web directory which is used by the webmonitor.
web.upload.dir
(none) - + Directory for uploading the job jars. If not specified a dynamic directory will be used under the directory specified by JOB_MANAGER_WEB_TMPDIR_KEY. diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java index 4c928fef686bae..a70c020e21c87b 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java @@ -216,7 +216,8 @@ public static String[] getParentFirstLoaderPatterns(Configuration config) { public static final ConfigOption TMP_DIRS = key("io.tmp.dirs") .defaultValue(System.getProperty("java.io.tmpdir")) - .withDeprecatedKeys("taskmanager.tmp.dirs"); + .withDeprecatedKeys("taskmanager.tmp.dirs") + .withDescription("Directories for temporary files, separated by\",\", \"|\", or the system's java.io.File.pathSeparator."); // ------------------------------------------------------------------------ // program @@ -225,7 +226,8 @@ public static String[] getParentFirstLoaderPatterns(Configuration config) { @Documentation.CommonOption(position = Documentation.CommonOption.POSITION_PARALLELISM_SLOTS) public static final ConfigOption DEFAULT_PARALLELISM = ConfigOptions .key("parallelism.default") - .defaultValue(1); + .defaultValue(1) + .withDescription("Default parallelism for jobs."); // ------------------------------------------------------------------------ // file systems diff --git a/flink-core/src/main/java/org/apache/flink/configuration/HistoryServerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/HistoryServerOptions.java index 12e7f24697e254..a4df23ce148679 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/HistoryServerOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/HistoryServerOptions.java @@ -76,7 +76,8 @@ public class HistoryServerOptions { */ public static final ConfigOption HISTORY_SERVER_WEB_REFRESH_INTERVAL = key("historyserver.web.refresh-interval") - .defaultValue(10000L); + .defaultValue(10000L) + .withDescription("The refresh interval for the HistoryServer web-frontend in milliseconds."); /** * Enables/Disables SSL support for the HistoryServer web-frontend. Only relevant if diff --git a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java index cb2dbe61631899..a91b931d1098be 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java @@ -136,7 +136,8 @@ public class JobManagerOptions { */ public static final ConfigOption ARCHIVE_DIR = key("jobmanager.archive.fs.dir") - .noDefaultValue(); + .noDefaultValue() + .withDescription("Dictionary for JobManager to store the archives of completed jobs."); /** * The job store cache size in bytes which is used to keep completed diff --git a/flink-core/src/main/java/org/apache/flink/configuration/MetricOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/MetricOptions.java index 0785b3473359c3..2c22d926da45a3 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/MetricOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/MetricOptions.java @@ -47,7 +47,10 @@ public class MetricOptions { */ public static final ConfigOption REPORTERS_LIST = key("metrics.reporters") - .noDefaultValue(); + .noDefaultValue() + .withDescription("An optional list of reporter names. If configured, only reporters whose name matches" + + " any of the names in the list will be started. Otherwise, all reporters that could be found in" + + " the configuration will be started."); public static final ConfigOption REPORTER_CLASS = key("metrics.reporter..class") @@ -68,7 +71,8 @@ public class MetricOptions { /** The delimiter used to assemble the metric identifier. */ public static final ConfigOption SCOPE_DELIMITER = key("metrics.scope.delimiter") - .defaultValue("."); + .defaultValue(".") + .withDescription("Delimiter used to assemble the metric identifier."); /** The scope format string that is applied to all metrics scoped to a JobManager. */ public static final ConfigOption SCOPE_NAMING_JM = @@ -135,14 +139,18 @@ public class MetricOptions { */ public static final ConfigOption SYSTEM_RESOURCE_METRICS = key("metrics.system-resource") - .defaultValue(false); + .defaultValue(false) + .withDescription("Flag indicating whether Flink should report system resource metrics such as machine's CPU," + + " memory or network usage."); /** * Interval between probing of system resource metrics specified in milliseconds. Has an effect only when * {@link #SYSTEM_RESOURCE_METRICS} is enabled. */ public static final ConfigOption SYSTEM_RESOURCE_METRICS_PROBING_INTERVAL = key("metrics.system-resource-probing-interval") - .defaultValue(5000L); + .defaultValue(5000L) + .withDescription("Interval between probing of system resource metrics specified in milliseconds. Has an effect" + + " only when '" + SYSTEM_RESOURCE_METRICS.key() + "' is enabled."); /** * The default network port range for Flink's internal metric query service. The {@code "0"} means that diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ResourceManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/ResourceManagerOptions.java index 5a203e3372b462..ca6cc146123ae3 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/ResourceManagerOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/ResourceManagerOptions.java @@ -35,10 +35,19 @@ public class ResourceManagerOptions { .defaultValue("5 minutes") .withDescription("Timeout for jobs which don't have a job manager as leader assigned."); + /** + * The number of resource managers start. + */ public static final ConfigOption LOCAL_NUMBER_RESOURCE_MANAGER = ConfigOptions .key("local.number-resourcemanager") - .defaultValue(1); + .defaultValue(1) + .withDescription("The number of resource managers start."); + /** + * Defines the network port to connect to for communication with the resource manager. + * By default, the port of the JobManager, because the same ActorSystem is used. Its not + * possible to use this configuration key to define port ranges. + */ public static final ConfigOption IPC_PORT = ConfigOptions .key("resourcemanager.rpc.port") .defaultValue(0) diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java index 6d3ac4515b20f6..6844ee4cd4840f 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java @@ -375,13 +375,15 @@ public class TaskManagerOptions { " leads to a fatal TaskManager error. A value of 0 deactivates" + " the watch dog."); /** - * This configures how long we wait for the timers to finish all pending timer threads - * when the stream task is cancelled . + * This configures how long we wait for the timers in milliseconds to finish all pending timer threads + * when the stream task is cancelled. */ public static final ConfigOption TASK_CANCELLATION_TIMEOUT_TIMERS = ConfigOptions .key("task.cancellation.timers.timeout") .defaultValue(7500L) - .withDeprecatedKeys("timerservice.exceptional.shutdown.timeout"); + .withDeprecatedKeys("timerservice.exceptional.shutdown.timeout") + .withDescription("Time we wait for the timers in milliseconds to finish all pending timer threads" + + " when the stream task is cancelled."); /** * The maximum number of bytes that a checkpoint alignment may buffer. diff --git a/flink-core/src/main/java/org/apache/flink/configuration/WebOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/WebOptions.java index 6d0a6ec9e93570..ba341619051474 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/WebOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/WebOptions.java @@ -35,7 +35,8 @@ public class WebOptions { public static final ConfigOption ADDRESS = key("web.address") .noDefaultValue() - .withDeprecatedKeys("jobmanager.web.address"); + .withDeprecatedKeys("jobmanager.web.address") + .withDescription("Address for runtime monitor web-frontend server."); /** * The port for the runtime monitor web-frontend server. @@ -55,15 +56,17 @@ public class WebOptions { public static final ConfigOption ACCESS_CONTROL_ALLOW_ORIGIN = key("web.access-control-allow-origin") .defaultValue("*") - .withDeprecatedKeys("jobmanager.web.access-control-allow-origin"); + .withDeprecatedKeys("jobmanager.web.access-control-allow-origin") + .withDescription("Access-Control-Allow-Origin header for all responses from the web-frontend."); /** - * The config parameter defining the refresh interval for the web-frontend. + * The config parameter defining the refresh interval for the web-frontend in milliseconds. */ public static final ConfigOption REFRESH_INTERVAL = key("web.refresh-interval") .defaultValue(3000L) - .withDeprecatedKeys("jobmanager.web.refresh-interval"); + .withDeprecatedKeys("jobmanager.web.refresh-interval") + .withDescription("Refresh interval for the web-frontend in milliseconds."); /** * Config parameter to override SSL support for the JobManager Web UI. @@ -71,7 +74,8 @@ public class WebOptions { public static final ConfigOption SSL_ENABLED = key("web.ssl.enabled") .defaultValue(true) - .withDeprecatedKeys("jobmanager.web.ssl.enabled"); + .withDeprecatedKeys("jobmanager.web.ssl.enabled") + .withDescription("Flag indicating whether to override SSL support for the JobManager Web UI."); /** * The config parameter defining the flink web directory to be used by the webmonitor. @@ -80,7 +84,8 @@ public class WebOptions { public static final ConfigOption TMP_DIR = key("web.tmpdir") .defaultValue(System.getProperty("java.io.tmpdir")) - .withDeprecatedKeys("jobmanager.web.tmpdir"); + .withDeprecatedKeys("jobmanager.web.tmpdir") + .withDescription("Flink web directory which is used by the webmonitor."); /** * The config parameter defining the directory for uploading the job jars. If not specified a dynamic directory @@ -89,15 +94,18 @@ public class WebOptions { public static final ConfigOption UPLOAD_DIR = key("web.upload.dir") .noDefaultValue() - .withDeprecatedKeys("jobmanager.web.upload.dir"); + .withDeprecatedKeys("jobmanager.web.upload.dir") + .withDescription("Directory for uploading the job jars. If not specified a dynamic directory will be used" + + " under the directory specified by JOB_MANAGER_WEB_TMPDIR_KEY."); /** - * The config parameter defining the number of archived jobs for the jobmanager. + * The config parameter defining the number of archived jobs for the JobManager. */ public static final ConfigOption ARCHIVE_COUNT = key("web.history") .defaultValue(5) - .withDeprecatedKeys("jobmanager.web.history"); + .withDeprecatedKeys("jobmanager.web.history") + .withDescription("Number of archived jobs for the JobManager."); /** * The log file location (may be in /log for standalone but under log directory when using YARN). @@ -105,7 +113,8 @@ public class WebOptions { public static final ConfigOption LOG_PATH = key("web.log.path") .noDefaultValue() - .withDeprecatedKeys("jobmanager.web.log.path"); + .withDeprecatedKeys("jobmanager.web.log.path") + .withDescription("Path to the log file (may be in /log for standalone but under log directory when using YARN)."); /** * Config parameter indicating whether jobs can be uploaded and run from the web-frontend. @@ -113,7 +122,8 @@ public class WebOptions { public static final ConfigOption SUBMIT_ENABLE = key("web.submit.enable") .defaultValue(true) - .withDeprecatedKeys("jobmanager.web.submit.enable"); + .withDeprecatedKeys("jobmanager.web.submit.enable") + .withDescription("Flag indicating whether jobs can be uploaded and run from the web-frontend."); /** * Config parameter defining the number of checkpoints to remember for recent history. @@ -121,23 +131,27 @@ public class WebOptions { public static final ConfigOption CHECKPOINTS_HISTORY_SIZE = key("web.checkpoints.history") .defaultValue(10) - .withDeprecatedKeys("jobmanager.web.checkpoints.history"); + .withDeprecatedKeys("jobmanager.web.checkpoints.history") + .withDescription("Number of checkpoints to remember for recent history."); /** - * Time after which cached stats are cleaned up if not accessed. + * Time, in milliseconds, after which cached stats are cleaned up if not accessed. */ public static final ConfigOption BACKPRESSURE_CLEANUP_INTERVAL = key("web.backpressure.cleanup-interval") .defaultValue(10 * 60 * 1000) - .withDeprecatedKeys("jobmanager.web.backpressure.cleanup-interval"); + .withDeprecatedKeys("jobmanager.web.backpressure.cleanup-interval") + .withDescription("Time, in milliseconds, after which cached stats are cleaned up if not accessed."); /** - * Time after which available stats are deprecated and need to be refreshed (by resampling). + * Time, in milliseconds, after which available stats are deprecated and need to be refreshed (by resampling). */ public static final ConfigOption BACKPRESSURE_REFRESH_INTERVAL = key("web.backpressure.refresh-interval") .defaultValue(60 * 1000) - .withDeprecatedKeys("jobmanager.web.backpressure.refresh-interval"); + .withDeprecatedKeys("jobmanager.web.backpressure.refresh-interval") + .withDescription("Time, in milliseconds, after which available stats are deprecated and need to be refreshed" + + " (by resampling)."); /** * Number of stack trace samples to take to determine back pressure. @@ -145,22 +159,25 @@ public class WebOptions { public static final ConfigOption BACKPRESSURE_NUM_SAMPLES = key("web.backpressure.num-samples") .defaultValue(100) - .withDeprecatedKeys("jobmanager.web.backpressure.num-samples"); + .withDeprecatedKeys("jobmanager.web.backpressure.num-samples") + .withDescription("Number of stack trace samples to take to determine back pressure."); /** - * Delay between stack trace samples to determine back pressure. + * Delay between stack trace samples to determine back pressure in milliseconds. */ public static final ConfigOption BACKPRESSURE_DELAY = key("web.backpressure.delay-between-samples") .defaultValue(50) - .withDeprecatedKeys("jobmanager.web.backpressure.delay-between-samples"); + .withDeprecatedKeys("jobmanager.web.backpressure.delay-between-samples") + .withDescription("Delay between stack trace samples to determine back pressure in milliseconds."); /** * Timeout for asynchronous operations by the WebRuntimeMonitor in milliseconds. */ public static final ConfigOption TIMEOUT = key("web.timeout") - .defaultValue(10L * 1000L); + .defaultValue(10L * 1000L) + .withDescription("Timeout for asynchronous operations by the WebRuntimeMonitor in milliseconds."); // ------------------------------------------------------------------------ From 7a9a6adf18bc6232e15e42252807793c8a94a131 Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 17 Oct 2018 11:58:27 +0200 Subject: [PATCH 094/147] [FLINK-10571][storm] Remove topology support --- docs/dev/libs/storm_compatibility.md | 53 +- flink-contrib/flink-storm-examples/pom.xml | 109 ---- .../storm/exclamation/ExclamationLocal.java | 73 --- .../exclamation/ExclamationTopology.java | 121 ----- .../flink/storm/join/SingleJoinExample.java | 88 ---- .../flink/storm/print/PrintSampleStream.java | 69 --- .../flink/storm/wordcount/WordCountLocal.java | 70 --- .../storm/wordcount/WordCountLocalByName.java | 70 --- .../wordcount/WordCountRemoteByClient.java | 84 --- .../wordcount/WordCountRemoteBySubmitter.java | 82 --- .../storm/wordcount/WordCountTopology.java | 135 ----- .../StormExclamationLocalITCase.java | 42 -- .../flink/storm/join/SingleJoinITCase.java | 55 -- .../apache/flink/storm/split/SplitBolt.java | 65 --- .../flink/storm/split/SplitBoltTopology.java | 92 ---- .../apache/flink/storm/split/SplitITCase.java | 12 - .../flink/storm/split/SplitSpoutTopology.java | 91 ---- .../storm/split/SplitStreamBoltLocal.java | 54 -- .../storm/split/SplitStreamSpoutLocal.java | 54 -- .../tests/StormFieldsGroupingITCase.java | 98 ---- .../storm/tests/StormMetaDataITCase.java | 63 --- .../flink/storm/tests/StormUnionITCase.java | 82 --- .../tests/operators/FiniteRandomSpout.java | 91 ---- .../storm/tests/operators/MergerBolt.java | 54 -- .../storm/tests/operators/MetaDataSpout.java | 59 --- .../storm/tests/operators/TaskIdBolt.java | 57 --- .../tests/operators/VerifyMetaDataBolt.java | 73 --- .../storm/wordcount/WordCountLocalITCase.java | 41 -- .../wordcount/WordCountLocalNamedITCase.java | 41 -- .../apache/flink/storm/api/FlinkClient.java | 365 ------------- .../flink/storm/api/FlinkLocalCluster.java | 201 -------- .../storm/api/FlinkOutputFieldsDeclarer.java | 166 ------ .../flink/storm/api/FlinkSubmitter.java | 200 -------- .../apache/flink/storm/api/FlinkTopology.java | 480 ------------------ .../storm/api/StormFlinkStreamMerger.java | 51 -- .../storm/api/TwoFlinkStreamsMerger.java | 59 --- .../flink/storm/util/StormStreamSelector.java | 3 +- .../flink/storm/wrappers/BoltWrapper.java | 51 +- .../flink/storm/wrappers/SpoutWrapper.java | 15 +- .../storm/wrappers/WrapperSetupHelper.java | 137 +---- .../api/FlinkOutputFieldsDeclarerTest.java | 189 ------- .../flink/storm/api/FlinkTopologyTest.java | 92 ---- .../org/apache/flink/storm/api/TestBolt.java | 52 -- .../org/apache/flink/storm/api/TestSpout.java | 63 --- .../WrapperSetupInLocalClusterTest.java | 198 -------- 45 files changed, 29 insertions(+), 4371 deletions(-) delete mode 100644 flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationLocal.java delete mode 100644 flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationTopology.java delete mode 100644 flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/join/SingleJoinExample.java delete mode 100644 flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/print/PrintSampleStream.java delete mode 100644 flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocal.java delete mode 100644 flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocalByName.java delete mode 100644 flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteByClient.java delete mode 100644 flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteBySubmitter.java delete mode 100644 flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountTopology.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/StormExclamationLocalITCase.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/join/SingleJoinITCase.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBolt.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBoltTopology.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitSpoutTopology.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamBoltLocal.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamSpoutLocal.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormFieldsGroupingITCase.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormMetaDataITCase.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormUnionITCase.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/FiniteRandomSpout.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MergerBolt.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MetaDataSpout.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/TaskIdBolt.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/VerifyMetaDataBolt.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalITCase.java delete mode 100644 flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalNamedITCase.java delete mode 100644 flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java delete mode 100644 flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkLocalCluster.java delete mode 100644 flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarer.java delete mode 100644 flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkSubmitter.java delete mode 100644 flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkTopology.java delete mode 100644 flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/StormFlinkStreamMerger.java delete mode 100644 flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/TwoFlinkStreamsMerger.java delete mode 100644 flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarerTest.java delete mode 100644 flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkTopologyTest.java delete mode 100644 flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestBolt.java delete mode 100644 flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestSpout.java delete mode 100644 flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupInLocalClusterTest.java diff --git a/docs/dev/libs/storm_compatibility.md b/docs/dev/libs/storm_compatibility.md index 7d7e6c8178cddd..8847c5bac5e315 100644 --- a/docs/dev/libs/storm_compatibility.md +++ b/docs/dev/libs/storm_compatibility.md @@ -26,10 +26,7 @@ under the License. [Flink streaming]({{ site.baseurl }}/dev/datastream_api.html) is compatible with Apache Storm interfaces and therefore allows reusing code that was implemented for Storm. -You can: - -- execute a whole Storm `Topology` in Flink. -- use Storm `Spout`/`Bolt` as source/operator in Flink streaming programs. +You can use Storm `Spout`/`Bolt` as source/operator in Flink streaming programs. This document shows how to use existing Storm code with Flink. @@ -60,48 +57,9 @@ See *WordCount Storm* within `flink-storm-examples/pom.xml` for an example how t If you want to avoid large uber-jars, you can manually copy `storm-core-0.9.4.jar`, `json-simple-1.1.jar` and `flink-storm-{{site.version}}.jar` into Flink's `lib/` folder of each cluster node (*before* the cluster is started). For this case, it is sufficient to include only your own Spout and Bolt classes (and their internal dependencies) into the program jar. -# Execute Storm Topologies - -Flink provides a Storm compatible API (`org.apache.flink.storm.api`) that offers replacements for the following classes: - -- `StormSubmitter` replaced by `FlinkSubmitter` -- `NimbusClient` and `Client` replaced by `FlinkClient` -- `LocalCluster` replaced by `FlinkLocalCluster` - -In order to submit a Storm topology to Flink, it is sufficient to replace the used Storm classes with their Flink replacements in the Storm *client code that assembles* the topology. -The actual runtime code, ie, Spouts and Bolts, can be used *unmodified*. -If a topology is executed in a remote cluster, parameters `nimbus.host` and `nimbus.thrift.port` are used as `jobmanger.rpc.address` and `jobmanger.rpc.port`, respectively. If a parameter is not specified, the value is taken from `flink-conf.yaml`. - -
-
-{% highlight java %} -TopologyBuilder builder = new TopologyBuilder(); // the Storm topology builder - -// actual topology assembling code and used Spouts/Bolts can be used as-is -builder.setSpout("source", new FileSpout(inputFilePath)); -builder.setBolt("tokenizer", new BoltTokenizer()).shuffleGrouping("source"); -builder.setBolt("counter", new BoltCounter()).fieldsGrouping("tokenizer", new Fields("word")); -builder.setBolt("sink", new BoltFileSink(outputFilePath)).shuffleGrouping("counter"); - -Config conf = new Config(); -if(runLocal) { // submit to test cluster - // replaces: LocalCluster cluster = new LocalCluster(); - FlinkLocalCluster cluster = new FlinkLocalCluster(); - cluster.submitTopology("WordCount", conf, FlinkTopology.createTopology(builder)); -} else { // submit to remote cluster - // optional - // conf.put(Config.NIMBUS_HOST, "remoteHost"); - // conf.put(Config.NIMBUS_THRIFT_PORT, 6123); - // replaces: StormSubmitter.submitTopology(topologyId, conf, builder.createTopology()); - FlinkSubmitter.submitTopology("WordCount", conf, FlinkTopology.createTopology(builder)); -} -{% endhighlight %} -
-
- # Embed Storm Operators in Flink Streaming Programs -As an alternative, Spouts and Bolts can be embedded into regular streaming programs. +Spouts and Bolts can be embedded into regular streaming programs. The Storm compatibility layer offers a wrapper classes for each, namely `SpoutWrapper` and `BoltWrapper` (`org.apache.flink.storm.wrappers`). Per default, both wrappers convert Storm output tuples to Flink's [Tuple]({{site.baseurl}}/dev/api_concepts.html#tuples-and-case-classes) types (ie, `Tuple0` to `Tuple25` according to the number of fields of the Storm tuples). @@ -182,9 +140,8 @@ See [BoltTokenizerWordCountPojo](https://github.com/apache/flink/tree/master/fli In Storm, Spouts and Bolts can be configured with a globally distributed `Map` object that is given to `submitTopology(...)` method of `LocalCluster` or `StormSubmitter`. This `Map` is provided by the user next to the topology and gets forwarded as a parameter to the calls `Spout.open(...)` and `Bolt.prepare(...)`. -If a whole topology is executed in Flink using `FlinkTopologyBuilder` etc., there is no special attention required – it works as in regular Storm. -For embedded usage, Flink's configuration mechanism must be used. +To replicate this functionality Flink's configuration mechanism must be used. A global configuration can be set in a `StreamExecutionEnvironment` via `.getConfig().setGlobalJobParameters(...)`. Flink's regular `Configuration` class can be used to configure Spouts and Bolts. However, `Configuration` does not support arbitrary key data types as Storm does (only `String` keys are allowed). @@ -211,9 +168,8 @@ env.getConfig().setGlobalJobParameters(config); ## Multiple Output Streams Flink can also handle the declaration of multiple output streams for Spouts and Bolts. -If a whole topology is executed in Flink using `FlinkTopologyBuilder` etc., there is no special attention required – it works as in regular Storm. -For embedded usage, the output stream will be of data type `SplitStreamType` and must be split by using `DataStream.split(...)` and `SplitStream.select(...)`. +The output stream will be of data type `SplitStreamType` and must be split by using `DataStream.split(...)` and `SplitStream.select(...)`. Flink provides the predefined output selector `StormStreamSelector` for `.split(...)` already. Furthermore, the wrapper type `SplitStreamTuple` can be removed using `SplitStreamMapper`. @@ -282,7 +238,6 @@ To run the examples, you need to assemble a correct jar file. There are example jars for embedded Spout and Bolt, namely `WordCount-SpoutSource.jar` and `WordCount-BoltTokenizer.jar`, respectively. Compare `pom.xml` to see how both jars are built. -Furthermore, there is one example for whole Storm topologies (`WordCount-StormTopology.jar`). You can run each of those examples via `bin/flink run .jar`. The correct entry point class is contained in each jar's manifest file. diff --git a/flink-contrib/flink-storm-examples/pom.xml b/flink-contrib/flink-storm-examples/pom.xml index 47323c05b706ec..6c8bcab7061cec 100644 --- a/flink-contrib/flink-storm-examples/pom.xml +++ b/flink-contrib/flink-storm-examples/pom.xml @@ -133,16 +133,6 @@ under the License. - - org.apache.flink - flink-examples-batch_${scala.binary.version} - ${project.version} - jar - false - ${project.build.directory}/classes - org/apache/flink/examples/java/wordcount/util/WordCountData.class - - org.apache.flink flink-storm_${scala.binary.version} @@ -284,25 +274,6 @@ under the License. - - - - - WordCount-StormTopology - package - - jar - - - WordCount - StormTopology - - - test-jar @@ -310,86 +281,6 @@ under the License. - - - - - - maven-shade-plugin - org.apache.maven.plugins - - - WordCount-StormTopology - package - - shade - - - WordCount-StormTopology - - - - org.apache.storm:storm-core - - org.yaml:snakeyaml - com.googlecode.json-simple:json-simple - org.apache.flink:flink-storm_${scala.binary.version} - org.apache.flink:flink-storm-examples_${scala.binary.version} - - - - - org.apache.storm:storm-core - - defaults.yaml - org/apache/storm/*.class - org/apache/storm/topology/*.class - org/apache/storm/spout/*.class - org/apache/storm/task/*.class - org/apache/storm/tuple/*.class - org/apache/storm/generated/*.class - org/apache/storm/metric/**/*.class - org/apache/storm/utils/*.class - org/apache/storm/serialization/*.class - org/apache/storm/curator/**/*.class - org/apache/storm/grouping/**/*.class - org/apache/storm/thrift/**/*.class - - org/json/simple/**/*.class - org/yaml/snakeyaml/**/*.class - org/apache/storm/shade/**/*.class - - - - org.apache.flink:flink-storm-examples_${scala.binary.version} - - org/apache/flink/storm/wordcount/WordCountRemoteBySubmitter.class - - org/apache/flink/storm/wordcount/WordCountTopology.class - org/apache/flink/storm/wordcount/operators/*.class - org/apache/flink/storm/util/*.class - org/apache/flink/storm/wordcount/util/WordCountData.class - - - - org.apache.flink:flink-storm_${scala.binary.version} - - org/apache/flink/storm/api/*.class - org/apache/flink/storm/util/*.class - org/apache/flink/storm/wrappers/*.class - - - - - - org.apache.flink.storm.wordcount.WordCountRemoteBySubmitter - - - - - - diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationLocal.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationLocal.java deleted file mode 100644 index 6108f791a9ab45..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationLocal.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.exclamation; - -import org.apache.flink.storm.api.FlinkLocalCluster; -import org.apache.flink.storm.api.FlinkTopology; -import org.apache.flink.storm.exclamation.operators.ExclamationBolt; - -import org.apache.storm.Config; -import org.apache.storm.topology.TopologyBuilder; - -/** - * Implements the "Exclamation" program that attaches five exclamation mark to every line of a text files in a streaming - * fashion. The program is constructed as a regular {@link org.apache.storm.generated.StormTopology} and submitted to - * Flink for execution in the same way as to a Storm {@link org.apache.storm.LocalCluster}. - * - *

This example shows how to run program directly within Java, thus it cannot be used to submit a - * {@link org.apache.storm.generated.StormTopology} via Flink command line clients (ie, bin/flink). - * - *

The input is a plain text file with lines separated by newline characters. - * - *

Usage: ExclamationLocal <text path> <result path>
- * If no parameters are provided, the program is run with default data from - * {@link org.apache.flink.examples.java.wordcount.util.WordCountData}. - * - *

This example shows how to: - *

    - *
  • run a regular Storm program locally on Flink
    • - *
- */ -public class ExclamationLocal { - - public static final String TOPOLOGY_ID = "Streaming Exclamation"; - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws Exception { - - if (!ExclamationTopology.parseParameters(args)) { - return; - } - - // build Topology the Storm way - final TopologyBuilder builder = ExclamationTopology.buildTopology(); - - // execute program locally - Config conf = new Config(); - conf.put(ExclamationBolt.EXCLAMATION_COUNT, ExclamationTopology.getExclamation()); - conf.put(FlinkLocalCluster.SUBMIT_BLOCKING, true); // only required to stabilize integration test - - final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster(); - cluster.submitTopology(TOPOLOGY_ID, conf, FlinkTopology.createTopology(builder)); - cluster.shutdown(); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationTopology.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationTopology.java deleted file mode 100644 index 9de1684dba12db..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationTopology.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.exclamation; - -import org.apache.flink.storm.exclamation.operators.ExclamationBolt; -import org.apache.flink.storm.util.BoltFileSink; -import org.apache.flink.storm.util.BoltPrintSink; -import org.apache.flink.storm.util.FiniteFileSpout; -import org.apache.flink.storm.util.FiniteInMemorySpout; -import org.apache.flink.storm.util.OutputFormatter; -import org.apache.flink.storm.util.SimpleOutputFormatter; -import org.apache.flink.storm.wordcount.util.WordCountData; - -import org.apache.storm.topology.TopologyBuilder; - -/** - * Implements the "Exclamation" program that attaches two exclamation marks to every line of a text files in a streaming - * fashion. The program is constructed as a regular {@link org.apache.storm.generated.StormTopology}. - * - *

The input is a plain text file with lines separated by newline characters. - * - *

Usage: Exclamation[Local|RemoteByClient|RemoteBySubmitter] <text path> - * <result path>
- * If no parameters are provided, the program is run with default data from {@link WordCountData}. - * - *

This example shows how to: - *

    - *
  • construct a regular Storm topology as Flink program
    • - *
  • make use of the FiniteSpout interface
    • - *
- */ -public class ExclamationTopology { - - private static final String spoutId = "source"; - private static final String firstBoltId = "exclamation1"; - private static final String secondBoltId = "exclamation2"; - private static final String sinkId = "sink"; - private static final OutputFormatter formatter = new SimpleOutputFormatter(); - - public static TopologyBuilder buildTopology() { - final TopologyBuilder builder = new TopologyBuilder(); - - // get input data - if (fileInputOutput) { - // read the text file from given input path - final String[] tokens = textPath.split(":"); - final String inputFile = tokens[tokens.length - 1]; - builder.setSpout(spoutId, new FiniteFileSpout(inputFile)); - } else { - builder.setSpout(spoutId, new FiniteInMemorySpout(WordCountData.WORDS)); - } - - builder.setBolt(firstBoltId, new ExclamationBolt(), 3).shuffleGrouping(spoutId); - builder.setBolt(secondBoltId, new ExclamationBolt(), 2).shuffleGrouping(firstBoltId); - - // emit result - if (fileInputOutput) { - // read the text file from given input path - final String[] tokens = outputPath.split(":"); - final String outputFile = tokens[tokens.length - 1]; - builder.setBolt(sinkId, new BoltFileSink(outputFile, formatter)) - .shuffleGrouping(secondBoltId); - } else { - builder.setBolt(sinkId, new BoltPrintSink(formatter), 4) - .shuffleGrouping(secondBoltId); - } - - return builder; - } - - // ************************************************************************* - // UTIL METHODS - // ************************************************************************* - - private static boolean fileInputOutput = false; - private static String textPath; - private static String outputPath; - private static int exclamationNum = 3; - - static int getExclamation() { - return exclamationNum; - } - - static boolean parseParameters(final String[] args) { - - if (args.length > 0) { - // parse input arguments - fileInputOutput = true; - if (args.length == 3) { - textPath = args[0]; - outputPath = args[1]; - exclamationNum = Integer.parseInt(args[2]); - } else { - System.err.println("Usage: StormExclamation* "); - return false; - } - } else { - System.out.println("Executing StormExclamation example with built-in default data"); - System.out.println(" Provide parameters to read input data from a file"); - System.out.println(" Usage: StormExclamation "); - } - - return true; - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/join/SingleJoinExample.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/join/SingleJoinExample.java deleted file mode 100644 index b2ad05f1bb5a76..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/join/SingleJoinExample.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.join; - -import org.apache.flink.storm.api.FlinkLocalCluster; -import org.apache.flink.storm.api.FlinkTopology; -import org.apache.flink.storm.util.BoltFileSink; -import org.apache.flink.storm.util.NullTerminatingSpout; -import org.apache.flink.storm.util.TupleOutputFormatter; - -import org.apache.storm.Config; -import org.apache.storm.starter.bolt.PrinterBolt; -import org.apache.storm.starter.bolt.SingleJoinBolt; -import org.apache.storm.testing.FeederSpout; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Values; - -/** - * Implements a simple example where 2 input streams are being joined. - */ -public class SingleJoinExample { - - public static void main(String[] args) throws Exception { - final FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender", "hobbies")); - final FeederSpout ageSpout = new FeederSpout(new Fields("id", "age")); - - Config conf = new Config(); - TopologyBuilder builder = new TopologyBuilder(); - - // only required to stabilize integration test - conf.put(FlinkLocalCluster.SUBMIT_BLOCKING, true); - final NullTerminatingSpout finalGenderSpout = new NullTerminatingSpout(genderSpout); - final NullTerminatingSpout finalAgeSpout = new NullTerminatingSpout(ageSpout); - - builder.setSpout("gender", finalGenderSpout); - builder.setSpout("age", finalAgeSpout); - builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))) - .fieldsGrouping("gender", new Fields("id")) - .fieldsGrouping("age", new Fields("id")); - - // emit result - if (args.length > 0) { - // read the text file from given input path - builder.setBolt("fileOutput", new BoltFileSink(args[0], new TupleOutputFormatter())) - .shuffleGrouping("join"); - } else { - builder.setBolt("print", new PrinterBolt()).shuffleGrouping("join"); - } - - String[] hobbies = new String[] {"reading", "biking", "travelling", "watching tv"}; - - for (int i = 0; i < 10; i++) { - String gender; - if (i % 2 == 0) { - gender = "male"; - } - else { - gender = "female"; - } - genderSpout.feed(new Values(i, gender, hobbies[i % hobbies.length])); - } - - for (int i = 9; i >= 0; i--) { - ageSpout.feed(new Values(i, i + 20)); - } - - final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster(); - cluster.submitTopology("joinTopology", conf, FlinkTopology.createTopology(builder)); - cluster.shutdown(); - } -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/print/PrintSampleStream.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/print/PrintSampleStream.java deleted file mode 100644 index 6157e2cf495d87..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/print/PrintSampleStream.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.print; - -import org.apache.flink.storm.api.FlinkLocalCluster; -import org.apache.flink.storm.api.FlinkTopology; - -import org.apache.storm.Config; -import org.apache.storm.starter.bolt.PrinterBolt; -import org.apache.storm.starter.spout.TwitterSampleSpout; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.utils.Utils; - -import java.util.Arrays; - -/** - * Prints incoming tweets. Tweets can be filtered by keywords. - */ -public class PrintSampleStream { - - public static void main(String[] args) throws Exception { - - if (args.length < 4) { - System.err.println( - "Usage: PrintSampleStream "); - return; - } - - String consumerKey = args[0]; - String consumerSecret = args[1]; - String accessToken = args[2]; - String accessTokenSecret = args[3]; - - // keywords start with the 5th parameter - String[] keyWords = Arrays.copyOfRange(args, 4, args.length); - - TopologyBuilder builder = new TopologyBuilder(); - - builder.setSpout("twitter", new TwitterSampleSpout(consumerKey, consumerSecret, - accessToken, accessTokenSecret, keyWords)); - builder.setBolt("print", new PrinterBolt()) - .shuffleGrouping("twitter"); - - Config conf = new Config(); - - final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster(); - cluster.submitTopology("Print", conf, FlinkTopology.createTopology(builder)); - - Utils.sleep(10 * 1000); - - cluster.shutdown(); - } -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocal.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocal.java deleted file mode 100644 index 71983274f4d9a7..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocal.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.storm.api.FlinkLocalCluster; -import org.apache.flink.storm.api.FlinkTopology; -import org.apache.flink.storm.wordcount.util.WordCountData; - -import org.apache.storm.Config; -import org.apache.storm.LocalCluster; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.topology.TopologyBuilder; - -/** - * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming - * fashion. The program is constructed as a regular {@link StormTopology} and submitted to Flink for execution in the - * same way as to a Storm {@link LocalCluster}. - * - *

This example shows how to run program directly within Java, thus it cannot be used to submit a {@link StormTopology} - * via Flink command line clients (ie, bin/flink). - * - *

The input is a plain text file with lines separated by newline characters. - * - *

Usage: WordCount <text path> <result path>
- * If no parameters are provided, the program is run with default data from {@link WordCountData}. - * - *

This example shows how to: - *

    - *
  • run a regular Storm program locally on Flink
    • - *
- */ -public class WordCountLocal { - private static final String topologyId = "Storm WordCount"; - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws Exception { - - if (!WordCountTopology.parseParameters(args)) { - return; - } - - // build Topology the Storm way - final TopologyBuilder builder = WordCountTopology.buildTopology(); - - final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster(); - Config conf = new Config(); - conf.put(FlinkLocalCluster.SUBMIT_BLOCKING, true); // only required to stabilize integration test - cluster.submitTopology(topologyId, conf, FlinkTopology.createTopology(builder)); - cluster.shutdown(); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocalByName.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocalByName.java deleted file mode 100644 index dcad567e4b7942..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocalByName.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.storm.api.FlinkLocalCluster; -import org.apache.flink.storm.api.FlinkTopology; -import org.apache.flink.storm.wordcount.util.WordCountData; - -import org.apache.storm.Config; -import org.apache.storm.LocalCluster; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.topology.TopologyBuilder; - -/** - * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming - * fashion. The program is constructed as a regular {@link StormTopology} and submitted to Flink for execution in the - * same way as to a Storm {@link LocalCluster}. In contrast to {@link WordCountLocal} all bolts access the field of - * input tuples by name instead of index. - * - *

This example shows how to run program directly within Java, thus it cannot be used to submit a {@link StormTopology} - * via Flink command line clients (ie, bin/flink). - * - *

The input is a plain text file with lines separated by newline characters. - * - *

Usage: WordCountLocalByName <text path> <result path>
- * If no parameters are provided, the program is run with default data from {@link WordCountData}. - * - *

This example shows how to: - *

    - *
  • run a regular Storm program locally on Flink - *
- */ -public class WordCountLocalByName { - private static final String topologyId = "Storm WordCountName"; - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws Exception { - - if (!WordCountTopology.parseParameters(args)) { - return; - } - - // build Topology the Storm way - final TopologyBuilder builder = WordCountTopology.buildTopology(false); - - final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster(); - Config conf = new Config(); - conf.put(FlinkLocalCluster.SUBMIT_BLOCKING, true); // only required to stabilize integration test - cluster.submitTopology(topologyId, conf, FlinkTopology.createTopology(builder)); - cluster.shutdown(); - } -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteByClient.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteByClient.java deleted file mode 100644 index 9ad0d633e04774..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteByClient.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.storm.api.FlinkClient; -import org.apache.flink.storm.api.FlinkTopology; -import org.apache.flink.storm.wordcount.util.WordCountData; - -import org.apache.storm.Config; -import org.apache.storm.generated.AlreadyAliveException; -import org.apache.storm.generated.InvalidTopologyException; -import org.apache.storm.generated.NotAliveException; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.utils.NimbusClient; -import org.apache.storm.utils.Utils; - -/** - * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming - * fashion. The program is constructed as a regular {@link StormTopology} and submitted to Flink for execution in the - * same way as to a Storm cluster similar to {@link NimbusClient}. The Flink cluster can be local or remote. - * - *

This example shows how to submit the program via Java, thus it cannot be used to submit a {@link StormTopology} via - * Flink command line clients (ie, bin/flink). - * - *

The input is a plain text file with lines separated by newline characters. - * - *

Usage: WordCountRemoteByClient <text path> <result path>
- * If no parameters are provided, the program is run with default data from {@link WordCountData}. - * - *

This example shows how to: - *

    - *
  • submit a regular Storm program to a local or remote Flink cluster.
    • - *
- */ -public class WordCountRemoteByClient { - private static final String topologyId = "Storm WordCount"; - private static final String uploadedJarLocation = "WordCount-StormTopology.jar"; - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws AlreadyAliveException, InvalidTopologyException, - NotAliveException { - - if (!WordCountTopology.parseParameters(args)) { - return; - } - - // build Topology the Storm way - final TopologyBuilder builder = WordCountTopology.buildTopology(); - - // execute program on Flink cluster - final Config conf = new Config(); - // can be changed to remote address - conf.put(Config.NIMBUS_HOST, "localhost"); - // use default flink jobmanger.rpc.port - conf.put(Config.NIMBUS_THRIFT_PORT, 6123); - - final FlinkClient cluster = FlinkClient.getConfiguredClient(conf); - cluster.submitTopology(topologyId, uploadedJarLocation, FlinkTopology.createTopology(builder)); - - Utils.sleep(5 * 1000); - - cluster.killTopology(topologyId); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteBySubmitter.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteBySubmitter.java deleted file mode 100644 index 2d1a0b5f610b2a..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteBySubmitter.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.storm.api.FlinkClient; -import org.apache.flink.storm.api.FlinkSubmitter; -import org.apache.flink.storm.api.FlinkTopology; -import org.apache.flink.storm.wordcount.util.WordCountData; - -import org.apache.storm.Config; -import org.apache.storm.StormSubmitter; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.topology.TopologyBuilder; - -/** - * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming - * fashion. The program is constructed as a regular {@link StormTopology} and submitted to Flink for execution in the - * same way as to a Storm cluster similar to {@link StormSubmitter}. The Flink cluster can be local or remote. - * - *

This example shows how to submit the program via Java as well as Flink's command line client (ie, bin/flink). - * - *

The input is a plain text file with lines separated by newline characters. - * - *

Usage: WordCountRemoteBySubmitter <text path> <result path>
- * If no parameters are provided, the program is run with default data from {@link WordCountData}. - * - *

This example shows how to: - *

    - *
  • submit a regular Storm program to a local or remote Flink cluster.
    • - *
- */ -public class WordCountRemoteBySubmitter { - private static final String topologyId = "Storm WordCount"; - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws Exception { - - if (!WordCountTopology.parseParameters(args)) { - return; - } - - // build Topology the Storm way - final TopologyBuilder builder = WordCountTopology.buildTopology(); - - // execute program on Flink cluster - final Config conf = new Config(); - // We can set Jobmanager host/port values manually or leave them blank - // if not set and - // - executed within Java, default values "localhost" and "6123" are set by FlinkSubmitter - // - executed via bin/flink values from flink-conf.yaml are set by FlinkSubmitter. - // conf.put(Config.NIMBUS_HOST, "localhost"); - // conf.put(Config.NIMBUS_THRIFT_PORT, new Integer(6123)); - - // The user jar file must be specified via JVM argument if executed via Java. - // => -Dstorm.jar=target/WordCount-StormTopology.jar - // If bin/flink is used, the jar file is detected automatically. - FlinkSubmitter.submitTopology(topologyId, conf, FlinkTopology.createTopology(builder)); - - Thread.sleep(5 * 1000); - - FlinkClient.getConfiguredClient(conf).killTopology(topologyId); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountTopology.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountTopology.java deleted file mode 100644 index 3ae7c5ce5616a2..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountTopology.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.storm.util.BoltFileSink; -import org.apache.flink.storm.util.BoltPrintSink; -import org.apache.flink.storm.util.NullTerminatingSpout; -import org.apache.flink.storm.util.OutputFormatter; -import org.apache.flink.storm.util.TupleOutputFormatter; -import org.apache.flink.storm.wordcount.operators.BoltCounter; -import org.apache.flink.storm.wordcount.operators.BoltCounterByName; -import org.apache.flink.storm.wordcount.operators.BoltTokenizer; -import org.apache.flink.storm.wordcount.operators.BoltTokenizerByName; -import org.apache.flink.storm.wordcount.operators.WordCountFileSpout; -import org.apache.flink.storm.wordcount.operators.WordCountInMemorySpout; -import org.apache.flink.storm.wordcount.util.WordCountData; - -import org.apache.storm.generated.StormTopology; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.tuple.Fields; - -/** - * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming - * fashion. The program is constructed as a regular {@link StormTopology}. - * - *

The input is a plain text file with lines separated by newline characters. - * - *

Usage: - * WordCount[Local|LocalByName|RemoteByClient|RemoteBySubmitter] <text path> <result path>
- * If no parameters are provided, the program is run with default data from {@link WordCountData}. - * - *

This example shows how to: - *

    - *
  • how to construct a regular Storm topology as Flink program
    • - *
- */ -public class WordCountTopology { - private static final String spoutId = "source"; - private static final String tokenierzerId = "tokenizer"; - private static final String counterId = "counter"; - private static final String sinkId = "sink"; - private static final OutputFormatter formatter = new TupleOutputFormatter(); - - public static TopologyBuilder buildTopology() { - return buildTopology(true); - } - - public static TopologyBuilder buildTopology(boolean indexOrName) { - - final TopologyBuilder builder = new TopologyBuilder(); - - // get input data - if (fileInputOutput) { - // read the text file from given input path - final String[] tokens = textPath.split(":"); - final String inputFile = tokens[tokens.length - 1]; - // inserting NullTerminatingSpout only required to stabilize integration test - builder.setSpout(spoutId, new NullTerminatingSpout(new WordCountFileSpout(inputFile))); - } else { - builder.setSpout(spoutId, new WordCountInMemorySpout()); - } - - if (indexOrName) { - // split up the lines in pairs (2-tuples) containing: (word,1) - builder.setBolt(tokenierzerId, new BoltTokenizer(), 4).shuffleGrouping(spoutId); - // group by the tuple field "0" and sum up tuple field "1" - builder.setBolt(counterId, new BoltCounter(), 4).fieldsGrouping(tokenierzerId, - new Fields(BoltTokenizer.ATTRIBUTE_WORD)); - } else { - // split up the lines in pairs (2-tuples) containing: (word,1) - builder.setBolt(tokenierzerId, new BoltTokenizerByName(), 4).shuffleGrouping( - spoutId); - // group by the tuple field "0" and sum up tuple field "1" - builder.setBolt(counterId, new BoltCounterByName(), 4).fieldsGrouping( - tokenierzerId, new Fields(BoltTokenizerByName.ATTRIBUTE_WORD)); - } - - // emit result - if (fileInputOutput) { - // read the text file from given input path - final String[] tokens = outputPath.split(":"); - final String outputFile = tokens[tokens.length - 1]; - builder.setBolt(sinkId, new BoltFileSink(outputFile, formatter)).shuffleGrouping(counterId); - } else { - builder.setBolt(sinkId, new BoltPrintSink(formatter), 4).shuffleGrouping(counterId); - } - - return builder; - } - - // ************************************************************************* - // UTIL METHODS - // ************************************************************************* - - private static boolean fileInputOutput = false; - private static String textPath; - private static String outputPath; - - static boolean parseParameters(final String[] args) { - - if (args.length > 0) { - // parse input arguments - fileInputOutput = true; - if (args.length == 2) { - textPath = args[0]; - outputPath = args[1]; - } else { - System.err.println("Usage: WordCount* "); - return false; - } - } else { - System.out.println("Executing WordCount example with built-in default data"); - System.out.println(" Provide parameters to read input data from a file"); - System.out.println(" Usage: WordCount* "); - } - - return true; - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/StormExclamationLocalITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/StormExclamationLocalITCase.java deleted file mode 100644 index c82da37c893cf9..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/StormExclamationLocalITCase.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.exclamation; - -import org.apache.flink.storm.exclamation.util.ExclamationData; -import org.apache.flink.test.testdata.WordCountData; -import org.apache.flink.test.util.AbstractTestBase; - -import org.junit.Test; - -/** - * Test for the ExclamationLocal example. - */ -public class StormExclamationLocalITCase extends AbstractTestBase { - - @Test - public void testProgram() throws Exception { - String textPath = createTempFile("text.txt", WordCountData.TEXT); - String resultPath = getTempDirPath("result"); - String exclamationNum = "3"; - - ExclamationLocal.main(new String[]{textPath, resultPath, exclamationNum}); - - compareResultsByLinesInMemory(ExclamationData.TEXT_WITH_EXCLAMATIONS, resultPath); - } -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/join/SingleJoinITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/join/SingleJoinITCase.java deleted file mode 100644 index c00c1542e3ee68..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/join/SingleJoinITCase.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.join; - -import org.apache.flink.test.util.AbstractTestBase; - -import org.apache.flink.shaded.guava18.com.google.common.base.Joiner; - -import org.junit.Test; - -/** - * Test for the SingleJoin example. - */ -public class SingleJoinITCase extends AbstractTestBase { - - protected static String[] expectedOutput = { - "(male,20)", - "(female,21)", - "(male,22)", - "(female,23)", - "(male,24)", - "(female,25)", - "(male,26)", - "(female,27)", - "(male,28)", - "(female,29)" - }; - - @Test - public void testProgram() throws Exception { - String resultPath = getTempDirPath("result"); - // We need to remove the file scheme because we can't use the Flink file system. - // (to remain compatible with Storm) - SingleJoinExample.main(new String[]{resultPath.replace("file:", "")}); - - compareResultsByLinesInMemory(Joiner.on("\n").join(expectedOutput), resultPath); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBolt.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBolt.java deleted file mode 100644 index 90ee795317f81c..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBolt.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.split; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichBolt; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -import java.util.Map; - -/** - * A bolt for splitting an input stream containing numbers based on whether they are even or odd. - */ -public class SplitBolt extends BaseRichBolt { - private static final long serialVersionUID = -6627606934204267173L; - - public static final String EVEN_STREAM = "even"; - public static final String ODD_STREAM = "odd"; - - private OutputCollector collector; - - @SuppressWarnings("rawtypes") - @Override - public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { - this.collector = collector; - - } - - @Override - public void execute(Tuple input) { - if (input.getInteger(0) % 2 == 0) { - this.collector.emit(EVEN_STREAM, new Values(input.getInteger(0))); - } else { - this.collector.emit(ODD_STREAM, new Values(input.getInteger(0))); - } - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - Fields schema = new Fields("number"); - declarer.declareStream(EVEN_STREAM, schema); - declarer.declareStream(ODD_STREAM, schema); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBoltTopology.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBoltTopology.java deleted file mode 100644 index c00284035db08b..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBoltTopology.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.split; - -import org.apache.flink.storm.split.operators.RandomSpout; -import org.apache.flink.storm.split.operators.VerifyAndEnrichBolt; -import org.apache.flink.storm.util.BoltFileSink; -import org.apache.flink.storm.util.BoltPrintSink; -import org.apache.flink.storm.util.OutputFormatter; -import org.apache.flink.storm.util.TupleOutputFormatter; - -import org.apache.storm.topology.TopologyBuilder; - -/** - * A simple topology that splits a stream of numbers based on their parity, and verifies the result. - */ -public class SplitBoltTopology { - private static final String spoutId = "randomSource"; - private static final String boltId = "splitBolt"; - private static final String evenVerifierId = "evenVerifier"; - private static final String oddVerifierId = "oddVerifier"; - private static final String sinkId = "sink"; - private static final OutputFormatter formatter = new TupleOutputFormatter(); - - public static TopologyBuilder buildTopology() { - final TopologyBuilder builder = new TopologyBuilder(); - - builder.setSpout(spoutId, new RandomSpout(false, seed)); - builder.setBolt(boltId, new SplitBolt()).shuffleGrouping(spoutId); - builder.setBolt(evenVerifierId, new VerifyAndEnrichBolt(true)).shuffleGrouping(boltId, - SplitBolt.EVEN_STREAM); - builder.setBolt(oddVerifierId, new VerifyAndEnrichBolt(false)).shuffleGrouping(boltId, - SplitBolt.ODD_STREAM); - - // emit result - if (outputPath != null) { - // read the text file from given input path - final String[] tokens = outputPath.split(":"); - final String outputFile = tokens[tokens.length - 1]; - builder.setBolt(sinkId, new BoltFileSink(outputFile, formatter)) - .shuffleGrouping(evenVerifierId).shuffleGrouping(oddVerifierId); - } else { - builder.setBolt(sinkId, new BoltPrintSink(formatter), 4) - .shuffleGrouping(evenVerifierId).shuffleGrouping(oddVerifierId); - } - - return builder; - } - - // ************************************************************************* - // UTIL METHODS - // ************************************************************************* - - private static long seed = System.currentTimeMillis(); - private static String outputPath = null; - - static boolean parseParameters(final String[] args) { - - if (args.length > 0) { - // parse input arguments - if (args.length == 2) { - seed = Long.parseLong(args[0]); - outputPath = args[1]; - } else { - System.err.println("Usage: SplitStreamBoltLocal "); - return false; - } - } else { - System.out.println("Executing SplitBoltTopology example with random data"); - System.out.println(" Usage: SplitStreamBoltLocal "); - } - - return true; - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitITCase.java index 7152cf2e6ccdaf..944000ca369d33 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitITCase.java @@ -53,16 +53,4 @@ public void testEmbeddedSpout() throws Exception { Assert.assertFalse(Enrich.errorOccured); } - @Test - public void testSpoutSplitTopology() throws Exception { - SplitStreamSpoutLocal.main(new String[] { "0", output }); - Assert.assertFalse(VerifyAndEnrichBolt.errorOccured); - } - - @Test - public void testBoltSplitTopology() throws Exception { - SplitStreamBoltLocal.main(new String[] { "0", output }); - Assert.assertFalse(VerifyAndEnrichBolt.errorOccured); - } - } diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitSpoutTopology.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitSpoutTopology.java deleted file mode 100644 index aa92a959995626..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitSpoutTopology.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.split; - -import org.apache.flink.storm.split.operators.RandomSpout; -import org.apache.flink.storm.split.operators.VerifyAndEnrichBolt; -import org.apache.flink.storm.util.BoltFileSink; -import org.apache.flink.storm.util.BoltPrintSink; -import org.apache.flink.storm.util.OutputFormatter; -import org.apache.flink.storm.util.TupleOutputFormatter; - -import org.apache.storm.topology.TopologyBuilder; - -/** - * A simple topology similar to the {@link SplitBoltTopology}, except that the split streams are generated directly in - * a spout. - */ -public class SplitSpoutTopology { - private static final String spoutId = "randomSplitSource"; - private static final String evenVerifierId = "evenVerifier"; - private static final String oddVerifierId = "oddVerifier"; - private static final String sinkId = "sink"; - private static final OutputFormatter formatter = new TupleOutputFormatter(); - - public static TopologyBuilder buildTopology() { - final TopologyBuilder builder = new TopologyBuilder(); - - builder.setSpout(spoutId, new RandomSpout(true, seed)); - builder.setBolt(evenVerifierId, new VerifyAndEnrichBolt(true)).shuffleGrouping(spoutId, - RandomSpout.EVEN_STREAM); - builder.setBolt(oddVerifierId, new VerifyAndEnrichBolt(false)).shuffleGrouping(spoutId, - RandomSpout.ODD_STREAM); - - // emit result - if (outputPath != null) { - // read the text file from given input path - final String[] tokens = outputPath.split(":"); - final String outputFile = tokens[tokens.length - 1]; - builder.setBolt(sinkId, new BoltFileSink(outputFile, formatter)) - .shuffleGrouping(evenVerifierId).shuffleGrouping(oddVerifierId); - } else { - builder.setBolt(sinkId, new BoltPrintSink(formatter), 4) - .shuffleGrouping(evenVerifierId).shuffleGrouping(oddVerifierId); - } - - return builder; - } - - // ************************************************************************* - // UTIL METHODS - // ************************************************************************* - - private static long seed = System.currentTimeMillis(); - private static String outputPath = null; - - static boolean parseParameters(final String[] args) { - - if (args.length > 0) { - // parse input arguments - if (args.length == 2) { - seed = Long.parseLong(args[0]); - outputPath = args[1]; - } else { - System.err.println("Usage: SplitStreamSpoutLocal "); - return false; - } - } else { - System.out.println("Executing SplitSpoutTopology example with random data"); - System.out.println(" Usage: SplitStreamSpoutLocal "); - } - - return true; - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamBoltLocal.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamBoltLocal.java deleted file mode 100644 index 55c3bd39fb7fca..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamBoltLocal.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.split; - -import org.apache.flink.storm.api.FlinkLocalCluster; -import org.apache.flink.storm.api.FlinkTopology; - -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.utils.Utils; - -/** - * An example using the {@link SplitBoltTopology}. - */ -public class SplitStreamBoltLocal { - private static final String topologyId = "Bolt split stream example"; - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws Exception { - - if (!SplitBoltTopology.parseParameters(args)) { - return; - } - - // build Topology the Storm way - final TopologyBuilder builder = SplitBoltTopology.buildTopology(); - - final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster(); - cluster.submitTopology(topologyId, null, FlinkTopology.createTopology(builder)); - - // run topology for 5 seconds - Utils.sleep(5 * 1000); - - cluster.shutdown(); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamSpoutLocal.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamSpoutLocal.java deleted file mode 100644 index da6e574131a742..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamSpoutLocal.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.split; - -import org.apache.flink.storm.api.FlinkLocalCluster; -import org.apache.flink.storm.api.FlinkTopology; - -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.utils.Utils; - -/** - * An example using the {@link SplitSpoutTopology}. - */ -public class SplitStreamSpoutLocal { - private static final String topologyId = "Spout split stream example"; - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws Exception { - - if (!SplitSpoutTopology.parseParameters(args)) { - return; - } - - // build Topology the Storm way - final TopologyBuilder builder = SplitSpoutTopology.buildTopology(); - - final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster(); - cluster.submitTopology(topologyId, null, FlinkTopology.createTopology(builder)); - - // run topology for 5 seconds - Utils.sleep(5 * 1000); - - cluster.shutdown(); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormFieldsGroupingITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormFieldsGroupingITCase.java deleted file mode 100644 index 69059537875b18..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormFieldsGroupingITCase.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.tests; - -import org.apache.flink.storm.api.FlinkLocalCluster; -import org.apache.flink.storm.api.FlinkTopology; -import org.apache.flink.storm.tests.operators.FiniteRandomSpout; -import org.apache.flink.storm.tests.operators.TaskIdBolt; -import org.apache.flink.storm.util.BoltFileSink; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.test.util.AbstractTestBase; -import org.apache.flink.util.MathUtils; - -import org.apache.storm.Config; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.tuple.Fields; -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -/** - * This test relies on the hash function used by the {@link DataStream#keyBy}, which is - * assumed to be {@link MathUtils#murmurHash}. - */ -public class StormFieldsGroupingITCase extends AbstractTestBase { - - private static final String topologyId = "FieldsGrouping Test"; - private static final String spoutId = "spout"; - private static final String boltId = "bolt"; - private static final String sinkId = "sink"; - - @Test - public void testProgram() throws Exception { - String resultPath = this.getTempDirPath("result"); - - final String[] tokens = resultPath.split(":"); - final String outputFile = tokens[tokens.length - 1]; - - final TopologyBuilder builder = new TopologyBuilder(); - - builder.setSpout(spoutId, new FiniteRandomSpout(0, 10, 2)); - builder.setBolt(boltId, new TaskIdBolt(), 2).fieldsGrouping( - spoutId, FiniteRandomSpout.STREAM_PREFIX + 0, new Fields("number")); - builder.setBolt(sinkId, new BoltFileSink(outputFile)).shuffleGrouping(boltId); - - final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster(); - Config conf = new Config(); - conf.put(FlinkLocalCluster.SUBMIT_BLOCKING, true); // only required to stabilize integration test - cluster.submitTopology(topologyId, conf, FlinkTopology.createTopology(builder)); - cluster.shutdown(); - - List expectedResults = Arrays.asList( - "-1155484576", "1033096058", "-1930858313", "1431162155", "-1557280266", "-1728529858", "1654374947", - "-65105105", "-518907128", "-252332814"); - - List actualResults = new ArrayList<>(); - readAllResultLines(actualResults, resultPath, new String[0], false); - - //remove potential operator id prefix - for (int i = 0; i < actualResults.size(); ++i) { - String s = actualResults.get(i); - if (s.contains(">")) { - s = s.substring(s.indexOf(">") + 2); - actualResults.set(i, s); - } - } - - Assert.assertEquals(expectedResults.size(), actualResults.size()); - Collections.sort(actualResults); - Collections.sort(expectedResults); - System.out.println(actualResults); - for (int i = 0; i < actualResults.size(); ++i) { - //compare against actual results with removed prefix (as it depends e.g. on the hash function used) - Assert.assertEquals(expectedResults.get(i), actualResults.get(i)); - } - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormMetaDataITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormMetaDataITCase.java deleted file mode 100644 index c24a95e30bbcdf..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormMetaDataITCase.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.tests; - -import org.apache.flink.storm.api.FlinkLocalCluster; -import org.apache.flink.storm.api.FlinkTopology; -import org.apache.flink.storm.tests.operators.MetaDataSpout; -import org.apache.flink.storm.tests.operators.VerifyMetaDataBolt; -import org.apache.flink.test.util.AbstractTestBase; - -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.utils.Utils; -import org.junit.Assert; -import org.junit.Test; - -/** - * Test for meta data spouts/bolts. - */ -public class StormMetaDataITCase extends AbstractTestBase { - - private static final String topologyId = "FieldsGrouping Test"; - private static final String spoutId = "spout"; - private static final String boltId1 = "bolt1"; - private static final String boltId2 = "bolt2"; - - @Test - public void testProgram() throws Exception { - final TopologyBuilder builder = new TopologyBuilder(); - - builder.setSpout(spoutId, new MetaDataSpout(), 2); - builder.setBolt(boltId1, new VerifyMetaDataBolt(), 2).localOrShuffleGrouping(spoutId, - MetaDataSpout.STREAM_ID); - builder.setBolt(boltId2, new VerifyMetaDataBolt()).shuffleGrouping(boltId1, - VerifyMetaDataBolt.STREAM_ID); - - final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster(); - cluster.submitTopology(topologyId, null, FlinkTopology.createTopology(builder)); - - // run topology for 5 seconds - Utils.sleep(5 * 1000); - - cluster.shutdown(); - - Assert.assertFalse(VerifyMetaDataBolt.errorOccured); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormUnionITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormUnionITCase.java deleted file mode 100644 index 6f6e47fab0ff31..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormUnionITCase.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.tests; - -import org.apache.flink.storm.api.FlinkLocalCluster; -import org.apache.flink.storm.api.FlinkTopology; -import org.apache.flink.storm.tests.operators.FiniteRandomSpout; -import org.apache.flink.storm.tests.operators.MergerBolt; -import org.apache.flink.storm.util.BoltFileSink; -import org.apache.flink.test.util.AbstractTestBase; - -import org.apache.storm.Config; -import org.apache.storm.topology.TopologyBuilder; -import org.junit.Test; - -/** - * Test for the {@link MergerBolt}. - */ -public class StormUnionITCase extends AbstractTestBase { - - private static final String RESULT = "-1154715079\n" + "-1155869325\n" + "-1155484576\n" - + "431529176\n" + "1260042744\n" + "1761283695\n" + "1749940626\n" + "892128508\n" - + "155629808\n" + "1429008869\n" + "-1465154083\n" + "-723955400\n" + "-423279216\n" - + "17850135\n" + "2133836778\n" + "1033096058\n" + "-1690734402\n" + "-1557280266\n" - + "1327362106\n" + "-1930858313\n" + "502539523\n" + "-1728529858\n" + "-938301587\n" - + "-624140595\n" + "-60658084\n" + "142959438\n" + "-613647601\n" + "-330177159\n" - + "-54027108\n" + "1945002173\n" + "979930868"; - - private static final String topologyId = "Multiple Input Streams Test"; - private static final String spoutId1 = "spout1"; - private static final String spoutId2 = "spout2"; - private static final String spoutId3 = "spout3"; - private static final String boltId = "merger"; - private static final String sinkId = "sink"; - - @Test - public void testProgram() throws Exception { - String resultPath = this.getTempDirPath("result"); - - final TopologyBuilder builder = new TopologyBuilder(); - - // get input data - builder.setSpout(spoutId1, new FiniteRandomSpout(0, 10)); - builder.setSpout(spoutId2, new FiniteRandomSpout(1, 8)); - builder.setSpout(spoutId3, new FiniteRandomSpout(2, 13)); - - builder.setBolt(boltId, new MergerBolt()) - .shuffleGrouping(spoutId1, FiniteRandomSpout.STREAM_PREFIX + 0) - .shuffleGrouping(spoutId2, FiniteRandomSpout.STREAM_PREFIX + 0) - .shuffleGrouping(spoutId3, FiniteRandomSpout.STREAM_PREFIX + 0); - - final String[] tokens = resultPath.split(":"); - final String outputFile = tokens[tokens.length - 1]; - builder.setBolt(sinkId, new BoltFileSink(outputFile)).shuffleGrouping(boltId); - - // execute program locally - final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster(); - Config conf = new Config(); - conf.put(FlinkLocalCluster.SUBMIT_BLOCKING, true); // only required to stabilize integration test - cluster.submitTopology(topologyId, conf, FlinkTopology.createTopology(builder)); - cluster.shutdown(); - - compareResultsByLinesInMemory(RESULT, resultPath); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/FiniteRandomSpout.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/FiniteRandomSpout.java deleted file mode 100644 index 923f9dff21043c..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/FiniteRandomSpout.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.tests.operators; - -import org.apache.flink.storm.util.FiniteSpout; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichSpout; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Values; -import org.apache.storm.utils.Utils; - -import java.util.Map; -import java.util.Random; - -/** - * A Spout implementation that broadcasts random numbers across a specified number of output streams, until a specified - * count is reached. - */ -public class FiniteRandomSpout extends BaseRichSpout implements FiniteSpout { - private static final long serialVersionUID = 6592885571932363239L; - - public static final String STREAM_PREFIX = "stream_"; - - private final Random r; - private SpoutOutputCollector collector; - private int counter; - private final String[] outputStreams; - - public FiniteRandomSpout(long seed, int counter, int numberOfOutputStreams) { - this.r = new Random(seed); - this.counter = counter; - if (numberOfOutputStreams < 1) { - this.outputStreams = new String[] { Utils.DEFAULT_STREAM_ID }; - } else { - this.outputStreams = new String[numberOfOutputStreams]; - for (int i = 0; i < this.outputStreams.length; ++i) { - this.outputStreams[i] = STREAM_PREFIX + i; - } - } - } - - public FiniteRandomSpout(long seed, int counter) { - this(seed, counter, 1); - } - - @SuppressWarnings("rawtypes") - @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { - this.collector = collector; - } - - @Override - public void nextTuple() { - for (String s : this.outputStreams) { - this.collector.emit(s, new Values(this.r.nextInt())); - } - --this.counter; - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - for (String s : this.outputStreams) { - declarer.declareStream(s, new Fields("number")); - } - } - - @Override - public boolean reachedEnd() { - return this.counter <= 0; - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MergerBolt.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MergerBolt.java deleted file mode 100644 index c04bce308a1a59..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MergerBolt.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.tests.operators; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichBolt; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; - -import java.util.Map; - -/** - * A Bolt implementation that forwards all incoming tuples to a single output stream. - */ -public class MergerBolt extends BaseRichBolt { - private static final long serialVersionUID = -7966475984592762720L; - - private OutputCollector collector; - - @SuppressWarnings("rawtypes") - @Override - public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { - this.collector = collector; - } - - @Override - public void execute(Tuple input) { - this.collector.emit(input.getValues()); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("number")); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MetaDataSpout.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MetaDataSpout.java deleted file mode 100644 index 4061a711628881..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MetaDataSpout.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.tests.operators; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichSpout; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Values; - -import java.util.Map; - -/** - * A Spout implementation emitting meta data. - */ -public class MetaDataSpout extends BaseRichSpout { - private static final long serialVersionUID = 5305870218033256376L; - - public static final String STREAM_ID = "spoutMeta"; - - private SpoutOutputCollector collector; - private TopologyContext context; - - @SuppressWarnings("rawtypes") - @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { - this.collector = collector; - this.context = context; - } - - @Override - public void nextTuple() { - this.collector.emit(STREAM_ID, new Values(this.context.getThisComponentId(), STREAM_ID, - this.context.getThisTaskId())); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declareStream(STREAM_ID, new Fields("id", "sid", "tid")); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/TaskIdBolt.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/TaskIdBolt.java deleted file mode 100644 index 1a8f218dbf0445..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/TaskIdBolt.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.tests.operators; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichBolt; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -import java.util.Map; - -/** - * Bolt to prepend all incoming tuple values with the task id. - */ -public class TaskIdBolt extends BaseRichBolt { - private static final long serialVersionUID = -7966475984592762720L; - - private OutputCollector collector; - private int thisTaskId; - - @SuppressWarnings("rawtypes") - @Override - public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { - this.collector = collector; - this.thisTaskId = context.getThisTaskId(); - } - - @Override - public void execute(Tuple input) { - this.collector.emit(new Values(this.thisTaskId + "> " + input.getValue(0))); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("number")); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/VerifyMetaDataBolt.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/VerifyMetaDataBolt.java deleted file mode 100644 index 059882bdd09ce9..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/VerifyMetaDataBolt.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.tests.operators; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichBolt; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.MessageId; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -import java.util.Map; - -/** - * A Bolt implementation that verifies metadata emitted by a {@link MetaDataSpout}. - */ -public class VerifyMetaDataBolt extends BaseRichBolt { - private static final long serialVersionUID = 1353222852073800478L; - - public static final String STREAM_ID = "boltMeta"; - - private OutputCollector collector; - private TopologyContext context; - - public static boolean errorOccured = false; - - @SuppressWarnings("rawtypes") - @Override - public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { - this.collector = collector; - this.context = context; - } - - @Override - public void execute(Tuple input) { - if (!input.getSourceComponent().equals(input.getString(0)) - || !input.getSourceStreamId().equals(input.getString(1)) - || !input.getSourceGlobalStreamid().get_componentId().equals(input.getString(0)) - || !input.getSourceGlobalStreamid().get_streamId().equals(input.getString(1)) - || input.getSourceTask() != input.getInteger(2).intValue() - || !input.getMessageId().equals(MessageId.makeUnanchored()) - || input.getMessageId().getAnchors().size() != 0 - || input.getMessageId().getAnchorsToIds().size() != 0) { - errorOccured = true; - } - this.collector.emit(STREAM_ID, new Values(this.context.getThisComponentId(), STREAM_ID, - this.context.getThisTaskId())); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declareStream(STREAM_ID, new Fields("id", "sid", "tid")); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalITCase.java deleted file mode 100644 index 16844e54785226..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalITCase.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.test.testdata.WordCountData; -import org.apache.flink.test.util.AbstractTestBase; - -import org.junit.Test; - -/** - * Test for the WordCountLocal example. - */ -public class WordCountLocalITCase extends AbstractTestBase { - - @Test - public void testProgram() throws Exception { - String textPath = createTempFile("text.txt", WordCountData.TEXT); - String resultPath = getTempDirPath("result"); - - WordCountLocal.main(new String[]{textPath, resultPath}); - - compareResultsByLinesInMemory(WordCountData.STREAMING_COUNTS_AS_TUPLES, resultPath); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalNamedITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalNamedITCase.java deleted file mode 100644 index 0353c2c56b005d..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalNamedITCase.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.test.testdata.WordCountData; -import org.apache.flink.test.util.AbstractTestBase; - -import org.junit.Test; - -/** - * Test for the WordCountLocalByName example. - */ -public class WordCountLocalNamedITCase extends AbstractTestBase { - - @Test - public void testProgram() throws Exception { - String textPath = createTempFile("text.txt", WordCountData.TEXT); - String resultPath = getTempDirPath("result"); - - WordCountLocalByName.main(new String[]{textPath, resultPath}); - - compareResultsByLinesInMemory(WordCountData.STREAMING_COUNTS_AS_TUPLES, resultPath); - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java deleted file mode 100644 index f4ea659dceed65..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java +++ /dev/null @@ -1,365 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.api; - -import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.JobID; -import org.apache.flink.client.program.JobWithJars; -import org.apache.flink.client.program.ProgramInvocationException; -import org.apache.flink.client.program.StandaloneClusterClient; -import org.apache.flink.configuration.AkkaOptions; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.GlobalConfiguration; -import org.apache.flink.configuration.JobManagerOptions; -import org.apache.flink.core.fs.Path; -import org.apache.flink.runtime.akka.AkkaUtils; -import org.apache.flink.runtime.client.JobStatusMessage; -import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.AddressResolution; -import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.jobmaster.JobMaster; -import org.apache.flink.runtime.messages.JobManagerMessages; -import org.apache.flink.runtime.messages.JobManagerMessages.RunningJobsStatus; -import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils; -import org.apache.flink.storm.util.StormConfig; -import org.apache.flink.streaming.api.graph.StreamGraph; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.pattern.Patterns; -import akka.util.Timeout; -import com.esotericsoftware.kryo.Serializer; -import org.apache.storm.Config; -import org.apache.storm.generated.AlreadyAliveException; -import org.apache.storm.generated.InvalidTopologyException; -import org.apache.storm.generated.KillOptions; -import org.apache.storm.generated.NotAliveException; -import org.apache.storm.utils.NimbusClient; -import org.apache.storm.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URL; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import scala.Some; -import scala.concurrent.Await; -import scala.concurrent.Future; -import scala.concurrent.duration.FiniteDuration; - -/** - * {@link FlinkClient} mimics a Storm {@link NimbusClient} and {@link Nimbus}{@code .Client} at once, to interact with - * Flink's JobManager instead of Storm's Nimbus. - */ -public class FlinkClient { - - /** The log used by this client. */ - private static final Logger LOG = LoggerFactory.getLogger(FlinkClient.class); - - /** The client's configuration. */ - private final Map conf; - /** The jobmanager's host name. */ - private final String jobManagerHost; - /** The jobmanager's rpc port. */ - private final int jobManagerPort; - /** The user specified timeout in milliseconds. */ - private final String timeout; - - // The following methods are derived from "backtype.storm.utils.NimbusClient" - - /** - * Instantiates a new {@link FlinkClient} for the given configuration, host name, and port. If values for {@link - * Config#NIMBUS_HOST} and {@link Config#NIMBUS_THRIFT_PORT} of the given configuration are ignored. - * - * @param conf - * A configuration. - * @param host - * The jobmanager's host name. - * @param port - * The jobmanager's rpc port. - */ - @SuppressWarnings("rawtypes") - public FlinkClient(final Map conf, final String host, final int port) { - this(conf, host, port, null); - } - - /** - * Instantiates a new {@link FlinkClient} for the given configuration, host name, and port. If values for {@link - * Config#NIMBUS_HOST} and {@link Config#NIMBUS_THRIFT_PORT} of the given configuration are ignored. - * - * @param conf - * A configuration. - * @param host - * The jobmanager's host name. - * @param port - * The jobmanager's rpc port. - * @param timeout - * Timeout - */ - @SuppressWarnings("rawtypes") - public FlinkClient(final Map conf, final String host, final int port, final Integer timeout) { - this.conf = conf; - this.jobManagerHost = host; - this.jobManagerPort = port; - if (timeout != null) { - this.timeout = timeout + " ms"; - } else { - this.timeout = null; - } - } - - /** - * Returns a {@link FlinkClient} that uses the configured {@link Config#NIMBUS_HOST} and {@link - * Config#NIMBUS_THRIFT_PORT} as JobManager address. - * - * @param conf - * Configuration that contains the jobmanager's hostname and port. - * @return A configured {@link FlinkClient}. - */ - @SuppressWarnings("rawtypes") - public static FlinkClient getConfiguredClient(final Map conf) { - final String nimbusHost = (String) conf.get(Config.NIMBUS_HOST); - final int nimbusPort = Utils.getInt(conf.get(Config.NIMBUS_THRIFT_PORT)).intValue(); - return new FlinkClient(conf, nimbusHost, nimbusPort); - } - - /** - * Return a reference to itself. - * - *

{@link FlinkClient} mimics both, {@link NimbusClient} and {@link Nimbus}{@code .Client}, at once. - * - * @return A reference to itself. - */ - public FlinkClient getClient() { - return this; - } - - // The following methods are derived from "backtype.storm.generated.Nimbus.Client" - - /** - * Parameter {@code uploadedJarLocation} is actually used to point to the local jar, because Flink does not support - * uploading a jar file before hand. Jar files are always uploaded directly when a program is submitted. - */ - public void submitTopology(final String name, final String uploadedJarLocation, final FlinkTopology topology) - throws AlreadyAliveException, InvalidTopologyException { - this.submitTopologyWithOpts(name, uploadedJarLocation, topology); - } - - /** - * Parameter {@code uploadedJarLocation} is actually used to point to the local jar, because Flink does not support - * uploading a jar file before hand. Jar files are always uploaded directly when a program is submitted. - */ - public void submitTopologyWithOpts(final String name, final String uploadedJarLocation, final FlinkTopology topology) - throws AlreadyAliveException, InvalidTopologyException { - - if (this.getTopologyJobId(name) != null) { - throw new AlreadyAliveException(); - } - - final URI uploadedJarUri; - final URL uploadedJarUrl; - try { - uploadedJarUri = new File(uploadedJarLocation).getAbsoluteFile().toURI(); - uploadedJarUrl = uploadedJarUri.toURL(); - JobWithJars.checkJarFile(uploadedJarUrl); - } catch (final IOException e) { - throw new RuntimeException("Problem with jar file " + uploadedJarLocation, e); - } - - try { - FlinkClient.addStormConfigToTopology(topology, conf); - } catch (ClassNotFoundException e) { - LOG.error("Could not register class for Kryo serialization.", e); - throw new InvalidTopologyException("Could not register class for Kryo serialization."); - } - - final StreamGraph streamGraph = topology.getExecutionEnvironment().getStreamGraph(); - streamGraph.setJobName(name); - - final JobGraph jobGraph = streamGraph.getJobGraph(); - jobGraph.addJar(new Path(uploadedJarUri)); - - final Configuration configuration = jobGraph.getJobConfiguration(); - configuration.setString(JobManagerOptions.ADDRESS, jobManagerHost); - configuration.setInteger(JobManagerOptions.PORT, jobManagerPort); - - final StandaloneClusterClient client; - try { - client = new StandaloneClusterClient(configuration); - } catch (final Exception e) { - throw new RuntimeException("Could not establish a connection to the job manager", e); - } - - try { - ClassLoader classLoader = JobWithJars.buildUserCodeClassLoader( - Collections.singletonList(uploadedJarUrl), - Collections.emptyList(), - this.getClass().getClassLoader()); - client.runDetached(jobGraph, classLoader); - } catch (final ProgramInvocationException e) { - throw new RuntimeException("Cannot execute job due to ProgramInvocationException", e); - } - } - - public void killTopology(final String name) throws NotAliveException { - this.killTopologyWithOpts(name, null); - } - - public void killTopologyWithOpts(final String name, final KillOptions options) throws NotAliveException { - final JobID jobId = this.getTopologyJobId(name); - if (jobId == null) { - throw new NotAliveException("Storm topology with name " + name + " not found."); - } - - if (options != null) { - try { - Thread.sleep(1000 * options.get_wait_secs()); - } catch (final InterruptedException e) { - throw new RuntimeException(e); - } - } - - final Configuration configuration = GlobalConfiguration.loadConfiguration(); - configuration.setString(JobManagerOptions.ADDRESS, this.jobManagerHost); - configuration.setInteger(JobManagerOptions.PORT, this.jobManagerPort); - - final StandaloneClusterClient client; - try { - client = new StandaloneClusterClient(configuration); - } catch (final Exception e) { - throw new RuntimeException("Could not establish a connection to the job manager", e); - } - - try { - client.stop(jobId); - } catch (final Exception e) { - throw new RuntimeException("Cannot stop job.", e); - } - - } - - // Flink specific additional methods - - /** - * Package internal method to get a Flink {@link JobID} from a Storm topology name. - * - * @param id - * The Storm topology name. - * @return Flink's internally used {@link JobID}. - */ - JobID getTopologyJobId(final String id) { - final Configuration configuration = GlobalConfiguration.loadConfiguration(); - if (this.timeout != null) { - configuration.setString(AkkaOptions.ASK_TIMEOUT, this.timeout); - } - - try { - final ActorRef jobManager = this.getJobManager(); - - final FiniteDuration askTimeout = this.getTimeout(); - final Future response = Patterns.ask(jobManager, JobManagerMessages.getRequestRunningJobsStatus(), - new Timeout(askTimeout)); - - final Object result; - try { - result = Await.result(response, askTimeout); - } catch (final Exception e) { - throw new RuntimeException("Could not retrieve running jobs from the JobManager", e); - } - - if (result instanceof RunningJobsStatus) { - final List jobs = ((RunningJobsStatus) result).getStatusMessages(); - - for (final JobStatusMessage status : jobs) { - if (status.getJobName().equals(id)) { - return status.getJobId(); - } - } - } else { - throw new RuntimeException("ReqeustRunningJobs requires a response of type " - + "RunningJobs. Instead the response is of type " + result.getClass() + "."); - } - } catch (final IOException e) { - throw new RuntimeException("Could not connect to Flink JobManager with address " + this.jobManagerHost - + ":" + this.jobManagerPort, e); - } - - return null; - } - - private FiniteDuration getTimeout() { - final Configuration configuration = GlobalConfiguration.loadConfiguration(); - if (this.timeout != null) { - configuration.setString(AkkaOptions.ASK_TIMEOUT, this.timeout); - } - - return AkkaUtils.getClientTimeout(configuration); - } - - private ActorRef getJobManager() throws IOException { - final Configuration configuration = GlobalConfiguration.loadConfiguration(); - - ActorSystem actorSystem; - try { - final scala.Tuple2 systemEndpoint = new scala.Tuple2("", 0); - actorSystem = AkkaUtils.createActorSystem(configuration, new Some>( - systemEndpoint)); - } catch (final Exception e) { - throw new RuntimeException("Could not start actor system to communicate with JobManager", e); - } - - final String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl( - jobManagerHost, - jobManagerPort, - JobMaster.JOB_MANAGER_NAME, - AddressResolution.TRY_ADDRESS_RESOLUTION, - configuration); - - return AkkaUtils.getActorRef(jobManagerAkkaUrl, actorSystem, AkkaUtils.getLookupTimeout(configuration)); - } - - @SuppressWarnings({ "unchecked", "rawtypes" }) - static void addStormConfigToTopology(FlinkTopology topology, Map conf) throws ClassNotFoundException { - if (conf != null) { - ExecutionConfig flinkConfig = topology.getExecutionEnvironment().getConfig(); - - flinkConfig.setGlobalJobParameters(new StormConfig(conf)); - - // add all registered types to ExecutionConfig - List registeredClasses = (List) conf.get(Config.TOPOLOGY_KRYO_REGISTER); - if (registeredClasses != null) { - for (Object klass : registeredClasses) { - if (klass instanceof String) { - flinkConfig.registerKryoType(Class.forName((String) klass)); - } else { - for (Entry register : ((Map) klass).entrySet()) { - flinkConfig.registerTypeWithKryoSerializer(Class.forName(register.getKey()), - (Class>) Class.forName(register.getValue())); - } - } - } - } - } - } -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkLocalCluster.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkLocalCluster.java deleted file mode 100644 index 655978ffa4d798..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkLocalCluster.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.api; - -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.TaskManagerOptions; -import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.minicluster.FlinkMiniCluster; -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; -import org.apache.flink.streaming.api.graph.StreamGraph; - -import org.apache.storm.LocalCluster; -import org.apache.storm.generated.ClusterSummary; -import org.apache.storm.generated.KillOptions; -import org.apache.storm.generated.RebalanceOptions; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.generated.SubmitOptions; -import org.apache.storm.generated.TopologyInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; -import java.util.Objects; - -/** - * {@link FlinkLocalCluster} mimics a Storm {@link LocalCluster}. - */ -public class FlinkLocalCluster { - - /** The log used by this mini cluster. */ - private static final Logger LOG = LoggerFactory.getLogger(FlinkLocalCluster.class); - - /** The Flink mini cluster on which to execute the programs. */ - private FlinkMiniCluster flink; - - /** Configuration key to submit topology in blocking mode if flag is set to {@code true}. */ - public static final String SUBMIT_BLOCKING = "SUBMIT_STORM_TOPOLOGY_BLOCKING"; - - public FlinkLocalCluster() { - } - - public FlinkLocalCluster(FlinkMiniCluster flink) { - this.flink = Objects.requireNonNull(flink); - } - - @SuppressWarnings("rawtypes") - public void submitTopology(final String topologyName, final Map conf, final FlinkTopology topology) - throws Exception { - this.submitTopologyWithOpts(topologyName, conf, topology, null); - } - - @SuppressWarnings("rawtypes") - public void submitTopologyWithOpts(final String topologyName, final Map conf, final FlinkTopology topology, final SubmitOptions submitOpts) throws Exception { - LOG.info("Running Storm topology on FlinkLocalCluster"); - - boolean submitBlocking = false; - if (conf != null) { - Object blockingFlag = conf.get(SUBMIT_BLOCKING); - if (blockingFlag instanceof Boolean) { - submitBlocking = ((Boolean) blockingFlag).booleanValue(); - } - } - - FlinkClient.addStormConfigToTopology(topology, conf); - - StreamGraph streamGraph = topology.getExecutionEnvironment().getStreamGraph(); - streamGraph.setJobName(topologyName); - - JobGraph jobGraph = streamGraph.getJobGraph(); - - if (this.flink == null) { - Configuration configuration = new Configuration(); - configuration.addAll(jobGraph.getJobConfiguration()); - - configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0"); - configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); - - this.flink = new LocalFlinkMiniCluster(configuration, true); - this.flink.start(); - } - - if (submitBlocking) { - this.flink.submitJobAndWait(jobGraph, false); - } else { - this.flink.submitJobDetached(jobGraph); - } - } - - public void killTopology(final String topologyName) { - this.killTopologyWithOpts(topologyName, null); - } - - public void killTopologyWithOpts(final String name, final KillOptions options) { - } - - public void activate(final String topologyName) { - } - - public void deactivate(final String topologyName) { - } - - public void rebalance(final String name, final RebalanceOptions options) { - } - - public void shutdown() { - if (this.flink != null) { - this.flink.stop(); - this.flink = null; - } - } - - public String getTopologyConf(final String id) { - return null; - } - - public StormTopology getTopology(final String id) { - return null; - } - - public ClusterSummary getClusterInfo() { - return null; - } - - public TopologyInfo getTopologyInfo(final String id) { - return null; - } - - public Map getState() { - return null; - } - - // ------------------------------------------------------------------------ - // Access to default local cluster - // ------------------------------------------------------------------------ - - // A different {@link FlinkLocalCluster} to be used for execution of ITCases - private static LocalClusterFactory currentFactory = new DefaultLocalClusterFactory(); - - /** - * Returns a {@link FlinkLocalCluster} that should be used for execution. If no cluster was set by - * {@link #initialize(LocalClusterFactory)} in advance, a new {@link FlinkLocalCluster} is returned. - * - * @return a {@link FlinkLocalCluster} to be used for execution - */ - public static FlinkLocalCluster getLocalCluster() { - return currentFactory.createLocalCluster(); - } - - /** - * Sets a different factory for FlinkLocalClusters to be used for execution. - * - * @param clusterFactory - * The LocalClusterFactory to create the local clusters for execution. - */ - public static void initialize(LocalClusterFactory clusterFactory) { - currentFactory = Objects.requireNonNull(clusterFactory); - } - - // ------------------------------------------------------------------------ - // Cluster factory - // ------------------------------------------------------------------------ - - /** - * A factory that creates local clusters. - */ - public interface LocalClusterFactory { - - /** - * Creates a local Flink cluster. - * @return A local Flink cluster. - */ - FlinkLocalCluster createLocalCluster(); - } - - /** - * A factory that instantiates a FlinkLocalCluster. - */ - public static class DefaultLocalClusterFactory implements LocalClusterFactory { - - @Override - public FlinkLocalCluster createLocalCluster() { - return new FlinkLocalCluster(); - } - } -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarer.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarer.java deleted file mode 100644 index b1e8a47d8fd7a8..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarer.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.api; - -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple; -import org.apache.flink.api.java.typeutils.TypeExtractor; - -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.utils.Utils; - -import java.util.HashMap; -import java.util.List; - -/** - * {@link FlinkOutputFieldsDeclarer} is used to get the declared output schema of a - * {@link org.apache.storm.topology.IRichSpout spout} or {@link org.apache.storm.topology.IRichBolt bolt}.
- *
- * CAUTION: Flink does not support direct emit. - */ -final class FlinkOutputFieldsDeclarer implements OutputFieldsDeclarer { - - /** The declared output streams and schemas. */ - final HashMap outputStreams = new HashMap(); - - @Override - public void declare(final Fields fields) { - this.declareStream(Utils.DEFAULT_STREAM_ID, false, fields); - } - - /** - * {@inheritDoc} - *

- * Direct emit is no supported by Flink. Parameter {@code direct} must be {@code false}. - * - * @throws UnsupportedOperationException - * if {@code direct} is {@code true} - */ - @Override - public void declare(final boolean direct, final Fields fields) { - this.declareStream(Utils.DEFAULT_STREAM_ID, direct, fields); - } - - @Override - public void declareStream(final String streamId, final Fields fields) { - this.declareStream(streamId, false, fields); - } - - /** - * {@inheritDoc} - *

- * Direct emit is no supported by Flink. Parameter {@code direct} must be {@code false}. - * - * @throws UnsupportedOperationException - * if {@code direct} is {@code true} - */ - @Override - public void declareStream(final String streamId, final boolean direct, final Fields fields) { - if (direct) { - throw new UnsupportedOperationException("Direct emit is not supported by Flink"); - } - - this.outputStreams.put(streamId, fields); - } - - /** - * Returns {@link TypeInformation} for the declared output schema for a specific stream. - * - * @param streamId - * A stream ID. - * - * @return output type information for the declared output schema of the specified stream; or {@code null} if - * {@code streamId == null} - * - * @throws IllegalArgumentException - * If no output schema was declared for the specified stream or if more then 25 attributes got declared. - */ - TypeInformation getOutputType(final String streamId) throws IllegalArgumentException { - if (streamId == null) { - return null; - } - - Fields outputSchema = this.outputStreams.get(streamId); - if (outputSchema == null) { - throw new IllegalArgumentException("Stream with ID '" + streamId - + "' was not declared."); - } - - Tuple t; - final int numberOfAttributes = outputSchema.size(); - - if (numberOfAttributes <= 24) { - try { - t = Tuple.getTupleClass(numberOfAttributes + 1).newInstance(); - } catch (final InstantiationException e) { - throw new RuntimeException(e); - } catch (final IllegalAccessException e) { - throw new RuntimeException(e); - } - } else { - throw new IllegalArgumentException("Flink supports only a maximum number of 24 attributes"); - } - - // TODO: declare only key fields as DefaultComparable - for (int i = 0; i < numberOfAttributes + 1; ++i) { - t.setField(new DefaultComparable(), i); - } - - return TypeExtractor.getForObject(t); - } - - /** - * {@link DefaultComparable} is a {@link Comparable} helper class that is used to get the correct {@link - * TypeInformation} from {@link TypeExtractor} within {@link #getOutputType()}. If key fields are not comparable, - * Flink cannot use them and will throw an exception. - */ - private static class DefaultComparable implements Comparable { - - public DefaultComparable() { - } - - @Override - public int compareTo(final DefaultComparable o) { - return 0; - } - } - - /** - * Computes the indexes within the declared output schema of the specified stream, for a list of given - * field-grouping attributes. - * - * @param streamId - * A stream ID. - * @param groupingFields - * The names of the key fields. - * - * @return array of {@code int}s that contains the index within the output schema for each attribute in the given - * list - */ - int[] getGroupingFieldIndexes(final String streamId, final List groupingFields) { - final int[] fieldIndexes = new int[groupingFields.size()]; - - for (int i = 0; i < fieldIndexes.length; ++i) { - fieldIndexes[i] = this.outputStreams.get(streamId).fieldIndex(groupingFields.get(i)); - } - - return fieldIndexes; - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkSubmitter.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkSubmitter.java deleted file mode 100644 index 5b3f609e5ab1bb..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkSubmitter.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.api; - -import org.apache.flink.api.java.ExecutionEnvironment; -import org.apache.flink.client.program.ContextEnvironment; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.GlobalConfiguration; -import org.apache.flink.configuration.JobManagerOptions; -import org.apache.flink.util.Preconditions; - -import org.apache.storm.Config; -import org.apache.storm.StormSubmitter; -import org.apache.storm.generated.AlreadyAliveException; -import org.apache.storm.generated.InvalidTopologyException; -import org.apache.storm.generated.SubmitOptions; -import org.apache.storm.utils.Utils; -import org.json.simple.JSONValue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Map; - -/** - * {@link FlinkSubmitter} mimics a {@link StormSubmitter} to submit Storm topologies to a Flink cluster. - */ -public class FlinkSubmitter { - private static final Logger LOG = LoggerFactory.getLogger(FlinkSubmitter.class); - - /** - * Submits a topology to run on the cluster. A topology runs forever or until explicitly killed. - * - * @param name - * the name of the storm. - * @param stormConf - * the topology-specific configuration. See {@link Config}. - * @param topology - * the processing to execute. - * @param opts - * to manipulate the starting of the topology. - * @throws AlreadyAliveException - * if a topology with this name is already running - * @throws InvalidTopologyException - * if an invalid topology was submitted - */ - public static void submitTopology(final String name, final Map stormConf, final FlinkTopology topology, - final SubmitOptions opts) - throws AlreadyAliveException, InvalidTopologyException { - submitTopology(name, stormConf, topology); - } - - /** - * Submits a topology to run on the cluster. A topology runs forever or until explicitly killed. The given {@link - * FlinkProgressListener} is ignored because progress bars are not supported by Flink. - * - * @param name - * the name of the storm. - * @param stormConf - * the topology-specific configuration. See {@link Config}. - * @param topology - * the processing to execute. - * @throws AlreadyAliveException - * if a topology with this name is already running - * @throws InvalidTopologyException - * if an invalid topology was submitted - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static void submitTopology(final String name, final Map stormConf, final FlinkTopology topology) - throws AlreadyAliveException, InvalidTopologyException { - if (!Utils.isValidConf(stormConf)) { - throw new IllegalArgumentException("Storm conf is not valid. Must be json-serializable"); - } - - final Configuration flinkConfig = GlobalConfiguration.loadConfiguration(); - if (!stormConf.containsKey(Config.NIMBUS_HOST)) { - stormConf.put(Config.NIMBUS_HOST, - flinkConfig.getString(JobManagerOptions.ADDRESS, "localhost")); - } - if (!stormConf.containsKey(Config.NIMBUS_THRIFT_PORT)) { - stormConf.put(Config.NIMBUS_THRIFT_PORT, - new Integer(flinkConfig.getInteger(JobManagerOptions.PORT))); - } - - final String serConf = JSONValue.toJSONString(stormConf); - - final FlinkClient client = FlinkClient.getConfiguredClient(stormConf); - - try { - if (client.getTopologyJobId(name) != null) { - throw new RuntimeException("Topology with name `" + name + "` already exists on cluster"); - } - String localJar = System.getProperty("storm.jar"); - if (localJar == null) { - try { - for (final URL url : ((ContextEnvironment) ExecutionEnvironment.getExecutionEnvironment()) - .getJars()) { - // TODO verify that there is only one jar - localJar = new File(url.toURI()).getAbsolutePath(); - } - } catch (final URISyntaxException e) { - // ignore - } catch (final ClassCastException e) { - // ignore - } - } - Preconditions.checkNotNull(localJar, "LocalJar must not be null."); - LOG.info("Submitting topology " + name + " in distributed mode with conf " + serConf); - client.submitTopologyWithOpts(name, localJar, topology); - } catch (final InvalidTopologyException e) { - LOG.warn("Topology submission exception: " + e.get_msg()); - throw e; - } catch (final AlreadyAliveException e) { - LOG.warn("Topology already alive exception", e); - throw e; - } - - LOG.info("Finished submitting topology: " + name); - } - - /** - * Same as {@link #submitTopology(String, Map, FlinkTopology, SubmitOptions)}. Progress bars are not supported by - * Flink. - * - * @param name - * the name of the storm. - * @param stormConf - * the topology-specific configuration. See {@link Config}. - * @param topology - * the processing to execute. - * @throws AlreadyAliveException - * if a topology with this name is already running - * @throws InvalidTopologyException - * if an invalid topology was submitted - */ - public static void submitTopologyWithProgressBar(final String name, final Map stormConf, - final FlinkTopology topology) - throws AlreadyAliveException, InvalidTopologyException { - submitTopology(name, stormConf, topology); - } - - /** - * In Flink, jar files are submitted directly when a program is started. Thus, this method does nothing. The - * returned value is parameter localJar, because this give the best integration of Storm behavior within a Flink - * environment. - * - * @param conf - * the topology-specific configuration. See {@link Config}. - * @param localJar - * file path of the jar file to submit - * @return the value of parameter localJar - */ - @SuppressWarnings("rawtypes") - public static String submitJar(final Map conf, final String localJar) { - return submitJar(localJar); - } - - /** - * In Flink, jar files are submitted directly when a program is started. Thus, this method does nothing. The - * returned value is parameter localJar, because this give the best integration of Storm behavior within a Flink - * environment. - * - * @param localJar - * file path of the jar file to submit - * @return the value of parameter localJar - */ - public static String submitJar(final String localJar) { - if (localJar == null) { - throw new RuntimeException( - "Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar " + - "to upload"); - } - - return localJar; - } - - /** - * Dummy interface use to track progress of file upload. Does not do anything. Kept for compatibility. - */ - public interface FlinkProgressListener { - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkTopology.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkTopology.java deleted file mode 100644 index 3b78a9089aa508..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkTopology.java +++ /dev/null @@ -1,480 +0,0 @@ -/* -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.api; - -import org.apache.flink.api.common.JobExecutionResult; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple; -import org.apache.flink.api.java.typeutils.TypeExtractor; -import org.apache.flink.storm.util.SplitStreamMapper; -import org.apache.flink.storm.util.SplitStreamType; -import org.apache.flink.storm.util.StormStreamSelector; -import org.apache.flink.storm.wrappers.BoltWrapper; -import org.apache.flink.storm.wrappers.MergedInputsBoltWrapper; -import org.apache.flink.storm.wrappers.SpoutWrapper; -import org.apache.flink.storm.wrappers.StormTuple; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamSource; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; -import org.apache.flink.streaming.api.datastream.SplitStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.util.InstantiationUtil; - -import org.apache.storm.generated.ComponentCommon; -import org.apache.storm.generated.GlobalStreamId; -import org.apache.storm.generated.Grouping; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.topology.IRichStateSpout; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.tuple.Fields; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -/** - * {@link FlinkTopology} translates a {@link TopologyBuilder} to a Flink program. - * CAUTION: {@link IRichStateSpout StateSpout}s are currently not supported. - */ -public class FlinkTopology { - - /** All declared streams and output schemas by operator ID. */ - private final HashMap> outputStreams = new HashMap>(); - /** All spouts&bolts declarers by their ID. */ - private final HashMap declarers = new HashMap(); - - private final HashMap>> unprocessdInputsPerBolt = - new HashMap>>(); - - final HashMap>> availableInputs = new HashMap<>(); - - private final TopologyBuilder builder; - - // needs to be a class member for internal testing purpose - private final StormTopology stormTopology; - - private final Map spouts; - private final Map bolts; - - private final StreamExecutionEnvironment env; - - private FlinkTopology(TopologyBuilder builder) { - this.builder = builder; - this.stormTopology = builder.createTopology(); - // extract the spouts and bolts - this.spouts = getPrivateField("_spouts"); - this.bolts = getPrivateField("_bolts"); - - this.env = StreamExecutionEnvironment.getExecutionEnvironment(); - - // Kick off the translation immediately - translateTopology(); - } - - /** - * Creates a Flink program that uses the specified spouts and bolts. - * @param stormBuilder The Storm topology builder to use for creating the Flink topology. - * @return A {@link FlinkTopology} which contains the translated Storm topology and may be executed. - */ - public static FlinkTopology createTopology(TopologyBuilder stormBuilder) { - return new FlinkTopology(stormBuilder); - } - - /** - * Returns the underlying Flink {@link StreamExecutionEnvironment} for the Storm topology. - * @return The contextual environment (local or remote). - */ - public StreamExecutionEnvironment getExecutionEnvironment() { - return this.env; - } - - /** - * Directly executes the Storm topology based on the current context (local when in IDE and - * remote when executed through ./bin/flink). - * @return The Flink {@link JobExecutionResult} after the execution of the Storm topology. - * @throws Exception which occurs during execution of the translated Storm topology. - */ - public JobExecutionResult execute() throws Exception { - return env.execute(); - } - - @SuppressWarnings("unchecked") - private Map getPrivateField(String field) { - try { - Field f = builder.getClass().getDeclaredField(field); - f.setAccessible(true); - return copyObject((Map) f.get(builder)); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException("Couldn't get " + field + " from TopologyBuilder", e); - } - } - - private T copyObject(T object) { - try { - return InstantiationUtil.deserializeObject( - InstantiationUtil.serializeObject(object), - getClass().getClassLoader() - ); - } catch (IOException | ClassNotFoundException e) { - throw new RuntimeException("Failed to copy object.", e); - } - } - - /** - * Creates a Flink program that uses the specified spouts and bolts. - */ - private void translateTopology() { - - unprocessdInputsPerBolt.clear(); - outputStreams.clear(); - declarers.clear(); - availableInputs.clear(); - - // Storm defaults to parallelism 1 - env.setParallelism(1); - - /* Translation of topology */ - - for (final Entry spout : spouts.entrySet()) { - final String spoutId = spout.getKey(); - final IRichSpout userSpout = spout.getValue(); - - final FlinkOutputFieldsDeclarer declarer = new FlinkOutputFieldsDeclarer(); - userSpout.declareOutputFields(declarer); - final HashMap sourceStreams = declarer.outputStreams; - this.outputStreams.put(spoutId, sourceStreams); - declarers.put(spoutId, declarer); - - final HashMap> outputStreams = new HashMap>(); - final DataStreamSource source; - - if (sourceStreams.size() == 1) { - final SpoutWrapper spoutWrapperSingleOutput = new SpoutWrapper(userSpout, spoutId, null, null); - spoutWrapperSingleOutput.setStormTopology(stormTopology); - - final String outputStreamId = (String) sourceStreams.keySet().toArray()[0]; - - DataStreamSource src = env.addSource(spoutWrapperSingleOutput, spoutId, - declarer.getOutputType(outputStreamId)); - - outputStreams.put(outputStreamId, src); - source = src; - } else { - final SpoutWrapper> spoutWrapperMultipleOutputs = new SpoutWrapper>( - userSpout, spoutId, null, null); - spoutWrapperMultipleOutputs.setStormTopology(stormTopology); - - @SuppressWarnings({ "unchecked", "rawtypes" }) - DataStreamSource> multiSource = env.addSource( - spoutWrapperMultipleOutputs, spoutId, - (TypeInformation) TypeExtractor.getForClass(SplitStreamType.class)); - - SplitStream> splitSource = multiSource - .split(new StormStreamSelector()); - for (String streamId : sourceStreams.keySet()) { - SingleOutputStreamOperator outStream = splitSource.select(streamId) - .map(new SplitStreamMapper()); - outStream.getTransformation().setOutputType(declarer.getOutputType(streamId)); - outputStreams.put(streamId, outStream); - } - source = multiSource; - } - availableInputs.put(spoutId, outputStreams); - - final ComponentCommon common = stormTopology.get_spouts().get(spoutId).get_common(); - if (common.is_set_parallelism_hint()) { - int dop = common.get_parallelism_hint(); - source.setParallelism(dop); - } else { - common.set_parallelism_hint(1); - } - } - - /** - * 1. Connect all spout streams with bolts streams - * 2. Then proceed with the bolts stream already connected - * - *

Because we do not know the order in which an iterator steps over a set, we might process a consumer before - * its producer - * ->thus, we might need to repeat multiple times - */ - boolean makeProgress = true; - while (bolts.size() > 0) { - if (!makeProgress) { - StringBuilder strBld = new StringBuilder(); - strBld.append("Unable to build Topology. Could not connect the following bolts:"); - for (String boltId : bolts.keySet()) { - strBld.append("\n "); - strBld.append(boltId); - strBld.append(": missing input streams ["); - for (Entry streams : unprocessdInputsPerBolt - .get(boltId)) { - strBld.append("'"); - strBld.append(streams.getKey().get_streamId()); - strBld.append("' from '"); - strBld.append(streams.getKey().get_componentId()); - strBld.append("'; "); - } - strBld.append("]"); - } - - throw new RuntimeException(strBld.toString()); - } - makeProgress = false; - - final Iterator> boltsIterator = bolts.entrySet().iterator(); - while (boltsIterator.hasNext()) { - - final Entry bolt = boltsIterator.next(); - final String boltId = bolt.getKey(); - final IRichBolt userBolt = copyObject(bolt.getValue()); - - final ComponentCommon common = stormTopology.get_bolts().get(boltId).get_common(); - - Set> unprocessedBoltInputs = unprocessdInputsPerBolt.get(boltId); - if (unprocessedBoltInputs == null) { - unprocessedBoltInputs = new HashSet<>(); - unprocessedBoltInputs.addAll(common.get_inputs().entrySet()); - unprocessdInputsPerBolt.put(boltId, unprocessedBoltInputs); - } - - // check if all inputs are available - final int numberOfInputs = unprocessedBoltInputs.size(); - int inputsAvailable = 0; - for (Entry entry : unprocessedBoltInputs) { - final String producerId = entry.getKey().get_componentId(); - final String streamId = entry.getKey().get_streamId(); - final HashMap> streams = availableInputs.get(producerId); - if (streams != null && streams.get(streamId) != null) { - inputsAvailable++; - } - } - - if (inputsAvailable != numberOfInputs) { - // traverse other bolts first until inputs are available - continue; - } else { - makeProgress = true; - boltsIterator.remove(); - } - - final Map> inputStreams = new HashMap<>(numberOfInputs); - - for (Entry input : unprocessedBoltInputs) { - final GlobalStreamId streamId = input.getKey(); - final Grouping grouping = input.getValue(); - - final String producerId = streamId.get_componentId(); - - final Map> producer = availableInputs.get(producerId); - - inputStreams.put(streamId, processInput(boltId, userBolt, streamId, grouping, producer)); - } - - final SingleOutputStreamOperator outputStream = createOutput(boltId, - userBolt, inputStreams); - - if (common.is_set_parallelism_hint()) { - int dop = common.get_parallelism_hint(); - outputStream.setParallelism(dop); - } else { - common.set_parallelism_hint(1); - } - - } - } - } - - private DataStream processInput(String boltId, IRichBolt userBolt, - GlobalStreamId streamId, Grouping grouping, - Map> producer) { - - assert (userBolt != null); - assert (boltId != null); - assert (streamId != null); - assert (grouping != null); - assert (producer != null); - - final String producerId = streamId.get_componentId(); - final String inputStreamId = streamId.get_streamId(); - - DataStream inputStream = producer.get(inputStreamId); - - final FlinkOutputFieldsDeclarer declarer = new FlinkOutputFieldsDeclarer(); - declarers.put(boltId, declarer); - userBolt.declareOutputFields(declarer); - this.outputStreams.put(boltId, declarer.outputStreams); - - // if producer was processed already - if (grouping.is_set_shuffle()) { - // Storm uses a round-robin shuffle strategy - inputStream = inputStream.rebalance(); - } else if (grouping.is_set_fields()) { - // global grouping is emulated in Storm via an empty fields grouping list - final List fields = grouping.get_fields(); - if (fields.size() > 0) { - FlinkOutputFieldsDeclarer prodDeclarer = this.declarers.get(producerId); - inputStream = inputStream.keyBy(prodDeclarer - .getGroupingFieldIndexes(inputStreamId, - grouping.get_fields())); - } else { - inputStream = inputStream.global(); - } - } else if (grouping.is_set_all()) { - inputStream = inputStream.broadcast(); - } else if (!grouping.is_set_local_or_shuffle()) { - throw new UnsupportedOperationException( - "Flink only supports (local-or-)shuffle, fields, all, and global grouping"); - } - - return inputStream; - } - - @SuppressWarnings({ "unchecked", "rawtypes" }) - private SingleOutputStreamOperator createOutput(String boltId, IRichBolt bolt, - Map> inputStreams) { - assert (boltId != null); - assert (bolt != null); - assert (inputStreams != null); - - Iterator>> iterator = inputStreams.entrySet() - .iterator(); - - Entry> input1 = iterator.next(); - GlobalStreamId streamId1 = input1.getKey(); - String inputStreamId1 = streamId1.get_streamId(); - String inputComponentId1 = streamId1.get_componentId(); - Fields inputSchema1 = this.outputStreams.get(inputComponentId1).get(inputStreamId1); - DataStream singleInputStream = input1.getValue(); - - DataStream> mergedInputStream = null; - while (iterator.hasNext()) { - Entry> input2 = iterator.next(); - GlobalStreamId streamId2 = input2.getKey(); - DataStream inputStream2 = input2.getValue(); - - if (mergedInputStream == null) { - mergedInputStream = singleInputStream - .connect(inputStream2) - .flatMap( - new TwoFlinkStreamsMerger(streamId1, inputSchema1, - streamId2, this.outputStreams.get( - streamId2.get_componentId()).get( - streamId2.get_streamId()))) - .returns(StormTuple.class); - } else { - mergedInputStream = mergedInputStream - .connect(inputStream2) - .flatMap( - new StormFlinkStreamMerger(streamId2, this.outputStreams.get( - streamId2.get_componentId()).get(streamId2.get_streamId()))) - .returns(StormTuple.class); - } - } - - final HashMap boltOutputs = this.outputStreams.get(boltId); - final FlinkOutputFieldsDeclarer declarer = this.declarers.get(boltId); - - final SingleOutputStreamOperator outputStream; - - if (boltOutputs.size() < 2) { // single output stream or sink - String outputStreamId; - if (boltOutputs.size() == 1) { - outputStreamId = (String) boltOutputs.keySet().toArray()[0]; - } else { - outputStreamId = null; - } - - final TypeInformation outType = declarer.getOutputType(outputStreamId); - - final SingleOutputStreamOperator outStream; - - // only one input - if (inputStreams.entrySet().size() == 1) { - BoltWrapper boltWrapper = new BoltWrapper<>(bolt, boltId, - inputStreamId1, inputComponentId1, inputSchema1, null); - boltWrapper.setStormTopology(stormTopology); - outStream = singleInputStream.transform(boltId, outType, boltWrapper); - } else { - MergedInputsBoltWrapper boltWrapper = new MergedInputsBoltWrapper( - bolt, boltId, null); - boltWrapper.setStormTopology(stormTopology); - outStream = mergedInputStream.transform(boltId, outType, boltWrapper); - } - - if (outType != null) { - // only for non-sink nodes - final HashMap> op = new HashMap<>(); - op.put(outputStreamId, outStream); - availableInputs.put(boltId, op); - } - outputStream = outStream; - } else { - final TypeInformation> outType = (TypeInformation) TypeExtractor - .getForClass(SplitStreamType.class); - - final SingleOutputStreamOperator> multiStream; - - // only one input - if (inputStreams.entrySet().size() == 1) { - final BoltWrapper> boltWrapperMultipleOutputs = new BoltWrapper<>( - bolt, boltId, inputStreamId1, inputComponentId1, inputSchema1, null); - boltWrapperMultipleOutputs.setStormTopology(stormTopology); - multiStream = singleInputStream.transform(boltId, outType, boltWrapperMultipleOutputs); - } else { - final MergedInputsBoltWrapper> boltWrapperMultipleOutputs = new MergedInputsBoltWrapper>( - bolt, boltId, null); - boltWrapperMultipleOutputs.setStormTopology(stormTopology); - multiStream = mergedInputStream.transform(boltId, outType, boltWrapperMultipleOutputs); - } - - final SplitStream> splitStream = multiStream - .split(new StormStreamSelector()); - - final HashMap> op = new HashMap<>(); - for (String outputStreamId : boltOutputs.keySet()) { - op.put(outputStreamId, - splitStream.select(outputStreamId).map( - new SplitStreamMapper())); - SingleOutputStreamOperator outStream = splitStream - .select(outputStreamId).map(new SplitStreamMapper()); - outStream.getTransformation().setOutputType(declarer.getOutputType(outputStreamId)); - op.put(outputStreamId, outStream); - } - availableInputs.put(boltId, op); - outputStream = multiStream; - } - - return outputStream; - } - - // for internal testing purpose only - public StormTopology getStormTopology() { - return this.stormTopology; - } -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/StormFlinkStreamMerger.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/StormFlinkStreamMerger.java deleted file mode 100644 index 00c467e3da7a2d..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/StormFlinkStreamMerger.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * /* Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package org.apache.flink.storm.api; - -import org.apache.flink.storm.wrappers.StormTuple; -import org.apache.flink.streaming.api.functions.co.CoFlatMapFunction; -import org.apache.flink.util.Collector; - -import org.apache.storm.generated.GlobalStreamId; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.MessageId; - -/** - * Merges a stream of type {@link StormTuple} with a Flink {@link DataStreams} into a stream of type {@link StormTuple}. - */ -@SuppressWarnings("rawtypes") -final class StormFlinkStreamMerger implements CoFlatMapFunction, IN2, StormTuple> { - private static final long serialVersionUID = -914164633830563631L; - - private final String inputStreamId; - private final String inputComponentId; - private final Fields inputSchema; - - public StormFlinkStreamMerger(GlobalStreamId streamId, Fields schema) { - this.inputStreamId = streamId.get_streamId(); - this.inputComponentId = streamId.get_componentId(); - this.inputSchema = schema; - } - - @Override - public void flatMap1(StormTuple value, Collector out) throws Exception { - out.collect(value); - } - - @Override - public void flatMap2(IN2 value, Collector out) throws Exception { - out.collect(new StormTuple(value, this.inputSchema, 0, this.inputStreamId, - this.inputComponentId, MessageId.makeUnanchored())); - } -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/TwoFlinkStreamsMerger.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/TwoFlinkStreamsMerger.java deleted file mode 100644 index f7bcb12c8110ba..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/TwoFlinkStreamsMerger.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * /* Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package org.apache.flink.storm.api; - -import org.apache.flink.storm.wrappers.StormTuple; -import org.apache.flink.streaming.api.functions.co.CoFlatMapFunction; -import org.apache.flink.util.Collector; - -import org.apache.storm.generated.GlobalStreamId; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.MessageId; - -/** - * Merges two Flink {@link DataStreams} into a stream of type {@link StormTuple}. - */ -@SuppressWarnings("rawtypes") -final class TwoFlinkStreamsMerger implements CoFlatMapFunction { - private static final long serialVersionUID = -495174165824062256L; - - private final String inputStreamId1; - private final String inputComponentId1; - private final Fields inputSchema1; - private final String inputStreamId2; - private final String inputComponentId2; - private final Fields inputSchema2; - - public TwoFlinkStreamsMerger(GlobalStreamId streamId1, Fields schema1, GlobalStreamId streamId2, - Fields schema2) { - this.inputStreamId1 = streamId1.get_streamId(); - this.inputComponentId1 = streamId1.get_componentId(); - this.inputSchema1 = schema1; - this.inputStreamId2 = streamId2.get_streamId(); - this.inputComponentId2 = streamId2.get_componentId(); - this.inputSchema2 = schema2; - } - - @Override - public void flatMap1(IN1 value, Collector out) throws Exception { - out.collect(new StormTuple(value, this.inputSchema1, 0, - this.inputStreamId1, this.inputComponentId1, MessageId.makeUnanchored())); - } - - @Override - public void flatMap2(IN2 value, Collector out) throws Exception { - out.collect(new StormTuple(value, this.inputSchema2, 0, - this.inputStreamId2, this.inputComponentId2, MessageId.makeUnanchored())); - } -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormStreamSelector.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormStreamSelector.java index 33ba3748a7aeaf..e8007266e53c56 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormStreamSelector.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormStreamSelector.java @@ -18,7 +18,6 @@ package org.apache.flink.storm.util; -import org.apache.flink.storm.api.FlinkTopology; import org.apache.flink.streaming.api.collector.selector.OutputSelector; import java.util.ArrayList; @@ -26,7 +25,7 @@ import java.util.List; /** - * Used by {@link FlinkTopology} to split multiple declared output streams within Flink. + * Used to split multiple declared output streams within Flink. */ public final class StormStreamSelector implements OutputSelector> { private static final long serialVersionUID = 2553423379715401023L; diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltWrapper.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltWrapper.java index ba2435e94d3326..1c12290eb26e31 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltWrapper.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltWrapper.java @@ -18,7 +18,6 @@ package org.apache.flink.storm.wrappers; import org.apache.flink.api.common.ExecutionConfig.GlobalJobParameters; -import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.api.java.tuple.Tuple0; import org.apache.flink.api.java.tuple.Tuple1; import org.apache.flink.api.java.tuple.Tuple25; @@ -28,9 +27,6 @@ import org.apache.flink.streaming.api.operators.TimestampedCollector; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.storm.generated.GlobalStreamId; -import org.apache.storm.generated.Grouping; -import org.apache.storm.generated.StormTopology; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.IRichBolt; @@ -40,7 +36,6 @@ import java.util.Collection; import java.util.HashMap; -import java.util.Map; import static java.util.Arrays.asList; @@ -68,15 +63,9 @@ public class BoltWrapper extends AbstractStreamOperator implements /** Number of attributes of the bolt's output tuples per stream. */ private final HashMap numberOfAttributes; - /** The original Storm topology. */ - private StormTopology stormTopology; /** The topology context of the bolt. */ private transient TopologyContext topologyContext; - /** The IDs of the input streams for this bolt per producer task ID. */ - private final HashMap inputStreamIds = new HashMap(); - /** The IDs of the producers for this bolt per producer task ID.. */ - private final HashMap inputComponentIds = new HashMap(); /** The schema (ie, ordered field names) of the input streams per producer taskID. */ private final HashMap inputSchemas = new HashMap(); @@ -242,16 +231,6 @@ public BoltWrapper(final IRichBolt bolt, final String name, final String inputSt this.numberOfAttributes = WrapperSetupHelper.getNumberOfAttributes(bolt, rawOutputs); } - /** - * Sets the original Storm topology. - * - * @param stormTopology - * The original Storm topology. - */ - public void setStormTopology(StormTopology stormTopology) { - this.stormTopology = stormTopology; - } - @Override public void open() throws Exception { super.open(); @@ -270,25 +249,11 @@ public void open() throws Exception { } this.topologyContext = WrapperSetupHelper.createTopologyContext( - getRuntimeContext(), this.bolt, this.name, this.stormTopology, stormConfig); + getRuntimeContext(), this.bolt, this.name, stormConfig); final OutputCollector stormCollector = new OutputCollector(new BoltCollector( this.numberOfAttributes, this.topologyContext.getThisTaskId(), this.flinkCollector)); - if (this.stormTopology != null) { - Map inputs = this.topologyContext.getThisSources(); - - for (GlobalStreamId inputStream : inputs.keySet()) { - for (Integer tid : this.topologyContext.getComponentTasks(inputStream - .get_componentId())) { - this.inputComponentIds.put(tid, inputStream.get_componentId()); - this.inputStreamIds.put(tid, inputStream.get_streamId()); - this.inputSchemas.put(tid, - this.topologyContext.getComponentOutputFields(inputStream)); - } - } - } - this.bolt.prepare(stormConfig, this.topologyContext, stormCollector); } @@ -304,17 +269,7 @@ public void processElement(final StreamRecord element) throws Exception { IN value = element.getValue(); - if (this.stormTopology != null) { - Tuple tuple = (Tuple) value; - Integer producerTaskId = tuple.getField(tuple.getArity() - 1); - - this.bolt.execute(new StormTuple<>(value, this.inputSchemas.get(producerTaskId), - producerTaskId, this.inputStreamIds.get(producerTaskId), this.inputComponentIds - .get(producerTaskId), MessageId.makeUnanchored())); - - } else { - this.bolt.execute(new StormTuple<>(value, this.inputSchemas.get(null), -1, null, null, - MessageId.makeUnanchored())); - } + this.bolt.execute(new StormTuple<>(value, this.inputSchemas.get(null), -1, null, null, + MessageId.makeUnanchored())); } } diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutWrapper.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutWrapper.java index 882ba273fef6af..eb103840f8789b 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutWrapper.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutWrapper.java @@ -27,7 +27,6 @@ import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; -import org.apache.storm.generated.StormTopology; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.IRichSpout; @@ -66,8 +65,6 @@ public final class SpoutWrapper extends RichParallelSourceFunction imp private volatile boolean isRunning = true; /** The number of {@link IRichSpout#nextTuple()} calls. */ private Integer numberOfInvocations; // do not use int -> null indicates an infinite loop - /** The original Storm topology. */ - private StormTopology stormTopology; /** * Instantiates a new {@link SpoutWrapper} that calls the {@link IRichSpout#nextTuple() nextTuple()} method of @@ -229,16 +226,6 @@ public SpoutWrapper(final IRichSpout spout, final String name, final Collection< this.numberOfInvocations = numberOfInvocations; } - /** - * Sets the original Storm topology. - * - * @param stormTopology - * The original Storm topology. - */ - public void setStormTopology(StormTopology stormTopology) { - this.stormTopology = stormTopology; - } - @Override public final void run(final SourceContext ctx) throws Exception { final GlobalJobParameters config = super.getRuntimeContext().getExecutionConfig() @@ -255,7 +242,7 @@ public final void run(final SourceContext ctx) throws Exception { final TopologyContext stormTopologyContext = WrapperSetupHelper.createTopologyContext( (StreamingRuntimeContext) super.getRuntimeContext(), this.spout, this.name, - this.stormTopology, stormConfig); + stormConfig); SpoutCollector collector = new SpoutCollector(this.numberOfAttributes, stormTopologyContext.getThisTaskId(), ctx); diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/WrapperSetupHelper.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/WrapperSetupHelper.java index 1d3a544a830caf..03af852ac53e4f 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/WrapperSetupHelper.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/WrapperSetupHelper.java @@ -26,7 +26,6 @@ import org.apache.storm.generated.SpoutSpec; import org.apache.storm.generated.StateSpoutSpec; import org.apache.storm.generated.StormTopology; -import org.apache.storm.generated.StreamInfo; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.IComponent; import org.apache.storm.topology.IRichBolt; @@ -93,9 +92,6 @@ static HashMap getNumberOfAttributes(final IComponent spoutOrBo return declarer.outputSchemas; } - /** Used to compute unique task IDs for a Storm topology. */ - private static int tid; - /** * Creates a {@link TopologyContext} for a Spout or Bolt instance (ie, Flink task / Storm executor). * @@ -103,8 +99,6 @@ static HashMap getNumberOfAttributes(final IComponent spoutOrBo * The Flink runtime context. * @param spoutOrBolt * The Spout or Bolt this context is created for. - * @param stormTopology - * The original Storm topology. * @param stormConfig * The user provided configuration. * @return The created {@link TopologyContext}. @@ -112,7 +106,7 @@ static HashMap getNumberOfAttributes(final IComponent spoutOrBo @SuppressWarnings({ "rawtypes", "unchecked" }) static synchronized TopologyContext createTopologyContext( final StreamingRuntimeContext context, final IComponent spoutOrBolt, - final String operatorName, StormTopology stormTopology, final Map stormConfig) { + final String operatorName, final Map stormConfig) { final int dop = context.getNumberOfParallelSubtasks(); @@ -131,66 +125,29 @@ static synchronized TopologyContext createTopologyContext( final Map registeredMetrics = new HashMap(); Atom openOrPrepareWasCalled = null; - if (stormTopology == null) { - // embedded mode - ComponentCommon common = new ComponentCommon(); - common.set_parallelism_hint(dop); - - HashMap spouts = new HashMap(); - HashMap bolts = new HashMap(); - if (spoutOrBolt instanceof IRichSpout) { - spouts.put(operatorName, new SpoutSpec(null, common)); - } else { - assert (spoutOrBolt instanceof IRichBolt); - bolts.put(operatorName, new Bolt(null, common)); - } - stormTopology = new StormTopology(spouts, bolts, new HashMap()); - - List sortedTasks = new ArrayList(dop); - for (int i = 1; i <= dop; ++i) { - taskToComponents.put(i, operatorName); - sortedTasks.add(i); - } - componentToSortedTasks.put(operatorName, sortedTasks); + ComponentCommon common = new ComponentCommon(); + common.set_parallelism_hint(dop); - SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - spoutOrBolt.declareOutputFields(declarer); - componentToStreamToFields.put(operatorName, declarer.outputStreams); + HashMap spouts = new HashMap(); + HashMap bolts = new HashMap(); + if (spoutOrBolt instanceof IRichSpout) { + spouts.put(operatorName, new SpoutSpec(null, common)); } else { - // whole topology is built (i.e. FlinkTopology is used) - Map spouts = stormTopology.get_spouts(); - Map bolts = stormTopology.get_bolts(); - Map stateSpouts = stormTopology.get_state_spouts(); - - tid = 1; - - for (Entry spout : spouts.entrySet()) { - Integer rc = processSingleOperator(spout.getKey(), spout.getValue().get_common(), - operatorName, context.getIndexOfThisSubtask(), dop, taskToComponents, - componentToSortedTasks, componentToStreamToFields); - if (rc != null) { - taskId = rc; - } - } - for (Entry bolt : bolts.entrySet()) { - Integer rc = processSingleOperator(bolt.getKey(), bolt.getValue().get_common(), - operatorName, context.getIndexOfThisSubtask(), dop, taskToComponents, - componentToSortedTasks, componentToStreamToFields); - if (rc != null) { - taskId = rc; - } - } - for (Entry stateSpout : stateSpouts.entrySet()) { - Integer rc = processSingleOperator(stateSpout.getKey(), stateSpout - .getValue().get_common(), operatorName, context.getIndexOfThisSubtask(), - dop, taskToComponents, componentToSortedTasks, componentToStreamToFields); - if (rc != null) { - taskId = rc; - } - } + assert (spoutOrBolt instanceof IRichBolt); + bolts.put(operatorName, new Bolt(null, common)); + } + StormTopology stormTopology = new StormTopology(spouts, bolts, new HashMap()); - assert(taskId != null); + List sortedTasks = new ArrayList(dop); + for (int i = 1; i <= dop; ++i) { + taskToComponents.put(i, operatorName); + sortedTasks.add(i); } + componentToSortedTasks.put(operatorName, sortedTasks); + + SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); + spoutOrBolt.declareOutputFields(declarer); + componentToStreamToFields.put(operatorName, declarer.outputStreams); if (!stormConfig.containsKey(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)) { stormConfig.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 30); // Storm default value @@ -201,58 +158,4 @@ static synchronized TopologyContext createTopologyContext( taskId, workerPort, workerTasks, defaultResources, userResources, executorData, registeredMetrics, openOrPrepareWasCalled); } - - /** - * Sets up {@code taskToComponents}, {@code componentToSortedTasks}, and {@code componentToStreamToFields} for a - * single instance of a Spout or Bolt (ie, task or executor). Furthermore, is computes the unique task-id. - * - * @param componentId - * The ID of the Spout/Bolt in the topology. - * @param common - * The common operator object (that is all Spouts and Bolts have). - * @param operatorName - * The Flink operator name. - * @param index - * The index of the currently processed tasks with its operator. - * @param dop - * The parallelism of the operator. - * @param taskToComponents - * OUTPUT: A map from all task IDs of the topology to their component IDs. - * @param componentToSortedTasks - * OUTPUT: A map from all component IDs to their sorted list of corresponding task IDs. - * @param componentToStreamToFields - * OUTPUT: A map from all component IDs to there output streams and output fields. - * - * @return A unique task ID if the currently processed Spout or Bolt ({@code componentId}) is equal to the current - * Flink operator {@code operatorName} -- {@code null} otherwise. - */ - private static Integer processSingleOperator(final String componentId, - final ComponentCommon common, final String operatorName, final int index, - final int dop, final Map taskToComponents, - final Map> componentToSortedTasks, - final Map> componentToStreamToFields) { - final int parallelismHint = common.get_parallelism_hint(); - Integer taskId = null; - - if (componentId.equals(operatorName)) { - taskId = tid + index; - } - - List sortedTasks = new ArrayList(dop); - for (int i = 0; i < parallelismHint; ++i) { - taskToComponents.put(tid, componentId); - sortedTasks.add(tid); - ++tid; - } - componentToSortedTasks.put(componentId, sortedTasks); - - Map outputStreams = new HashMap(); - for (Entry outStream : common.get_streams().entrySet()) { - outputStreams.put(outStream.getKey(), new Fields(outStream.getValue().get_output_fields())); - } - componentToStreamToFields.put(componentId, outputStreams); - - return taskId; - } - } diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarerTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarerTest.java deleted file mode 100644 index d035bb2784d7b5..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarerTest.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.api; - -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.storm.util.AbstractTest; - -import org.apache.storm.tuple.Fields; -import org.apache.storm.utils.Utils; -import org.junit.Assert; -import org.junit.Test; - -import java.util.LinkedList; - -/** - * Tests for the FlinkOutputFieldsDeclarer. - */ -public class FlinkOutputFieldsDeclarerTest extends AbstractTest { - - @Test - public void testNull() { - Assert.assertNull(new FlinkOutputFieldsDeclarer().getOutputType(null)); - } - - @Test - public void testDeclare() { - for (int i = 0; i < 2; ++i) { // test case: simple / non-direct - for (int j = 1; j < 2; ++j) { // number of streams - for (int k = 0; k <= 24; ++k) { // number of attributes - this.runDeclareTest(i, j, k); - } - } - } - } - - @Test(expected = IllegalArgumentException.class) - public void testDeclareSimpleToManyAttributes() { - this.runDeclareTest(0, this.r.nextBoolean() ? 1 : 2, 25); - } - - @Test(expected = IllegalArgumentException.class) - public void testDeclareNonDirectToManyAttributes() { - this.runDeclareTest(1, this.r.nextBoolean() ? 1 : 2, 25); - } - - @Test(expected = IllegalArgumentException.class) - public void testDeclareDefaultStreamToManyAttributes() { - this.runDeclareTest(2, this.r.nextBoolean() ? 1 : 2, 25); - } - - @Test(expected = IllegalArgumentException.class) - public void testDeclareFullToManyAttributes() { - this.runDeclareTest(3, this.r.nextBoolean() ? 1 : 2, 25); - } - - private void runDeclareTest(final int testCase, final int numberOfStreams, - final int numberOfAttributes) { - final FlinkOutputFieldsDeclarer declarer = new FlinkOutputFieldsDeclarer(); - - String[] streams = null; - if (numberOfStreams > 1 || r.nextBoolean()) { - streams = new String[numberOfStreams]; - for (int i = 0; i < numberOfStreams; ++i) { - streams[i] = "stream" + i; - } - } - - final String[] attributes = new String[numberOfAttributes]; - for (int i = 0; i < attributes.length; ++i) { - attributes[i] = "a" + i; - } - - switch (testCase) { - case 0: - this.declareSimple(declarer, streams, attributes); - break; - default: - this.declareNonDirect(declarer, streams, attributes); - } - - if (streams == null) { - streams = new String[] { Utils.DEFAULT_STREAM_ID }; - } - - for (String stream : streams) { - final TypeInformation type = declarer.getOutputType(stream); - - Assert.assertEquals(numberOfAttributes + 1, type.getArity()); - Assert.assertTrue(type.isTupleType()); - } - } - - private void declareSimple(final FlinkOutputFieldsDeclarer declarer, final String[] streams, - final String[] attributes) { - - if (streams != null) { - for (String stream : streams) { - declarer.declareStream(stream, new Fields(attributes)); - } - } else { - declarer.declare(new Fields(attributes)); - } - } - - private void declareNonDirect(final FlinkOutputFieldsDeclarer declarer, final String[] streams, - final String[] attributes) { - - if (streams != null) { - for (String stream : streams) { - declarer.declareStream(stream, false, new Fields(attributes)); - } - } else { - declarer.declare(false, new Fields(attributes)); - } - } - - @Test(expected = IllegalArgumentException.class) - public void testUndeclared() { - final FlinkOutputFieldsDeclarer declarer = new FlinkOutputFieldsDeclarer(); - declarer.getOutputType("unknownStreamId"); - } - - @Test(expected = UnsupportedOperationException.class) - public void testDeclareDirect() { - new FlinkOutputFieldsDeclarer().declare(true, null); - } - - @Test(expected = UnsupportedOperationException.class) - public void testDeclareDirect2() { - new FlinkOutputFieldsDeclarer().declareStream(Utils.DEFAULT_STREAM_ID, true, null); - } - - @Test - public void testGetGroupingFieldIndexes() { - final int numberOfAttributes = 5 + this.r.nextInt(20); - final String[] attributes = new String[numberOfAttributes]; - for (int i = 0; i < numberOfAttributes; ++i) { - attributes[i] = "a" + i; - } - - final FlinkOutputFieldsDeclarer declarer = new FlinkOutputFieldsDeclarer(); - declarer.declare(new Fields(attributes)); - - final int numberOfKeys = 1 + this.r.nextInt(24); - final LinkedList groupingFields = new LinkedList(); - final boolean[] indexes = new boolean[numberOfAttributes]; - - for (int i = 0; i < numberOfAttributes; ++i) { - if (this.r.nextInt(25) < numberOfKeys) { - groupingFields.add(attributes[i]); - indexes[i] = true; - } else { - indexes[i] = false; - } - } - - final int[] expectedResult = new int[groupingFields.size()]; - int j = 0; - for (int i = 0; i < numberOfAttributes; ++i) { - if (indexes[i]) { - expectedResult[j++] = i; - } - } - - final int[] result = declarer.getGroupingFieldIndexes(Utils.DEFAULT_STREAM_ID, - groupingFields); - - Assert.assertEquals(expectedResult.length, result.length); - for (int i = 0; i < expectedResult.length; ++i) { - Assert.assertEquals(expectedResult[i], result[i]); - } - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkTopologyTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkTopologyTest.java deleted file mode 100644 index aaecc061b18d18..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkTopologyTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.api; - -import org.apache.flink.storm.util.TestDummyBolt; -import org.apache.flink.storm.util.TestDummySpout; -import org.apache.flink.storm.util.TestSink; - -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.tuple.Fields; -import org.junit.Assert; -import org.junit.Test; - -/** - * Tests for the FlinkTopology. - */ -public class FlinkTopologyTest { - - @Test - public void testDefaultParallelism() { - final TopologyBuilder builder = new TopologyBuilder(); - final FlinkTopology flinkTopology = FlinkTopology.createTopology(builder); - Assert.assertEquals(1, flinkTopology.getExecutionEnvironment().getParallelism()); - } - - @Test(expected = RuntimeException.class) - public void testUnknowSpout() { - TopologyBuilder builder = new TopologyBuilder(); - builder.setSpout("spout", new TestSpout()); - builder.setBolt("bolt", new TestBolt()).shuffleGrouping("unknown"); - - FlinkTopology.createTopology(builder); - } - - @Test(expected = RuntimeException.class) - public void testUnknowBolt() { - TopologyBuilder builder = new TopologyBuilder(); - builder.setSpout("spout", new TestSpout()); - builder.setBolt("bolt1", new TestBolt()).shuffleGrouping("spout"); - builder.setBolt("bolt2", new TestBolt()).shuffleGrouping("unknown"); - - FlinkTopology.createTopology(builder); - } - - @Test(expected = RuntimeException.class) - public void testUndeclaredStream() { - TopologyBuilder builder = new TopologyBuilder(); - builder.setSpout("spout", new TestSpout()); - builder.setBolt("bolt", new TestBolt()).shuffleGrouping("spout"); - - FlinkTopology.createTopology(builder); - } - - @Test - public void testFieldsGroupingOnMultipleSpoutOutputStreams() { - TopologyBuilder builder = new TopologyBuilder(); - - builder.setSpout("spout", new TestDummySpout()); - builder.setBolt("sink", new TestSink()).fieldsGrouping("spout", - TestDummySpout.SPOUT_STREAM_ID, new Fields("id")); - - FlinkTopology.createTopology(builder); - } - - @Test - public void testFieldsGroupingOnMultipleBoltOutputStreams() { - TopologyBuilder builder = new TopologyBuilder(); - - builder.setSpout("spout", new TestDummySpout()); - builder.setBolt("bolt", new TestDummyBolt()).shuffleGrouping("spout"); - builder.setBolt("sink", new TestSink()).fieldsGrouping("bolt", - TestDummyBolt.GROUPING_STREAM_ID, new Fields("id")); - - FlinkTopology.createTopology(builder); - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestBolt.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestBolt.java deleted file mode 100644 index 001e9c4a542e50..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestBolt.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.api; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Tuple; - -import java.util.Map; - -/** - * A no-op test implementation of a {@link IRichBolt}. - */ -public class TestBolt implements IRichBolt { - private static final long serialVersionUID = -667148827441397683L; - - @SuppressWarnings("rawtypes") - @Override - public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {} - - @Override - public void execute(Tuple input) {} - - @Override - public void cleanup() {} - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) {} - - @Override - public Map getComponentConfiguration() { - return null; - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestSpout.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestSpout.java deleted file mode 100644 index 3466ff40971e00..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestSpout.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.api; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.topology.OutputFieldsDeclarer; - -import java.util.Map; - -/** - * A no-op test implementation of a {@link IRichSpout}. - */ -public class TestSpout implements IRichSpout { - private static final long serialVersionUID = -4884029383198924007L; - - @SuppressWarnings("rawtypes") - @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {} - - @Override - public void close() {} - - @Override - public void activate() {} - - @Override - public void deactivate() {} - - @Override - public void nextTuple() {} - - @Override - public void ack(Object msgId) {} - - @Override - public void fail(Object msgId) {} - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) {} - - @Override - public Map getComponentConfiguration() { - return null; - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupInLocalClusterTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupInLocalClusterTest.java deleted file mode 100644 index 2b0b2753f2c2df..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupInLocalClusterTest.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.storm.api.FlinkTopology; -import org.apache.flink.storm.util.AbstractTest; -import org.apache.flink.storm.util.TestDummyBolt; -import org.apache.flink.storm.util.TestDummySpout; -import org.apache.flink.storm.util.TestSink; -import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; - -import org.apache.storm.Config; -import org.apache.storm.LocalCluster; -import org.apache.storm.generated.ComponentCommon; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IComponent; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.utils.Utils; -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Tests for the setup of wrappers in a local cluster. - */ -public class WrapperSetupInLocalClusterTest extends AbstractTest { - - @Test - public void testCreateTopologyContext() { - HashMap dops = new HashMap(); - dops.put("spout1", 1); - dops.put("spout2", 3); - dops.put("bolt1", 1); - dops.put("bolt2", 2); - dops.put("sink", 1); - - HashMap taskCounter = new HashMap(); - taskCounter.put("spout1", 0); - taskCounter.put("spout2", 0); - taskCounter.put("bolt1", 0); - taskCounter.put("bolt2", 0); - taskCounter.put("sink", 0); - - HashMap operators = new HashMap(); - operators.put("spout1", new TestDummySpout()); - operators.put("spout2", new TestDummySpout()); - operators.put("bolt1", new TestDummyBolt()); - operators.put("bolt2", new TestDummyBolt()); - operators.put("sink", new TestSink()); - - TopologyBuilder builder = new TopologyBuilder(); - - builder.setSpout("spout1", (IRichSpout) operators.get("spout1"), dops.get("spout1")); - builder.setSpout("spout2", (IRichSpout) operators.get("spout2"), dops.get("spout2")); - builder.setBolt("bolt1", (IRichBolt) operators.get("bolt1"), dops.get("bolt1")).shuffleGrouping("spout1"); - builder.setBolt("bolt2", (IRichBolt) operators.get("bolt2"), dops.get("bolt2")).allGrouping("spout2"); - builder.setBolt("sink", (IRichBolt) operators.get("sink"), dops.get("sink")) - .shuffleGrouping("bolt1", TestDummyBolt.GROUPING_STREAM_ID) - .shuffleGrouping("bolt1", TestDummyBolt.SHUFFLE_STREAM_ID) - .shuffleGrouping("bolt2", TestDummyBolt.GROUPING_STREAM_ID) - .shuffleGrouping("bolt2", TestDummyBolt.SHUFFLE_STREAM_ID); - - LocalCluster cluster = new LocalCluster(); - Config c = new Config(); - c.setNumAckers(0); - cluster.submitTopology("test", c, builder.createTopology()); - - while (TestSink.RESULT.size() != 8) { - Utils.sleep(100); - } - cluster.shutdown(); - final FlinkTopology flinkBuilder = FlinkTopology.createTopology(builder); - StormTopology stormTopology = flinkBuilder.getStormTopology(); - - Set taskIds = new HashSet(); - - for (TopologyContext expectedContext : TestSink.RESULT) { - final String thisComponentId = expectedContext.getThisComponentId(); - int index = taskCounter.get(thisComponentId); - - StreamingRuntimeContext context = mock(StreamingRuntimeContext.class); - when(context.getTaskName()).thenReturn(thisComponentId); - when(context.getNumberOfParallelSubtasks()).thenReturn(dops.get(thisComponentId)); - when(context.getIndexOfThisSubtask()).thenReturn(index); - taskCounter.put(thisComponentId, ++index); - - Config stormConfig = new Config(); - stormConfig.put(WrapperSetupHelper.TOPOLOGY_NAME, "test"); - - TopologyContext topologyContext = WrapperSetupHelper.createTopologyContext(context, - operators.get(thisComponentId), thisComponentId, stormTopology, stormConfig); - - ComponentCommon expcetedCommon = expectedContext.getComponentCommon(thisComponentId); - ComponentCommon common = topologyContext.getComponentCommon(thisComponentId); - - Assert.assertNull(topologyContext.getCodeDir()); - Assert.assertNull(common.get_json_conf()); - Assert.assertNull(topologyContext.getExecutorData(null)); - Assert.assertNull(topologyContext.getPIDDir()); - Assert.assertNull(topologyContext.getResource(null)); - Assert.assertNull(topologyContext.getSharedExecutor()); - Assert.assertNull(expectedContext.getTaskData(null)); - Assert.assertNull(topologyContext.getThisWorkerPort()); - - Assert.assertTrue(expectedContext.getStormId().startsWith(topologyContext.getStormId())); - - Assert.assertEquals(expcetedCommon.get_inputs(), common.get_inputs()); - Assert.assertEquals(expcetedCommon.get_parallelism_hint(), common.get_parallelism_hint()); - Assert.assertEquals(expcetedCommon.get_streams(), common.get_streams()); - Assert.assertEquals(expectedContext.getComponentIds(), topologyContext.getComponentIds()); - Assert.assertEquals(expectedContext.getComponentStreams(thisComponentId), - topologyContext.getComponentStreams(thisComponentId)); - Assert.assertEquals(thisComponentId, topologyContext.getThisComponentId()); - Assert.assertEquals(expectedContext.getThisSources(), topologyContext.getThisSources()); - Assert.assertEquals(expectedContext.getThisStreams(), topologyContext.getThisStreams()); - Assert.assertEquals(expectedContext.getThisTargets(), topologyContext.getThisTargets()); - Assert.assertEquals(0, topologyContext.getThisWorkerTasks().size()); - - for (int taskId : topologyContext.getComponentTasks(thisComponentId)) { - Assert.assertEquals(thisComponentId, topologyContext.getComponentId(taskId)); - } - - for (String componentId : expectedContext.getComponentIds()) { - Assert.assertEquals(expectedContext.getSources(componentId), - topologyContext.getSources(componentId)); - Assert.assertEquals(expectedContext.getTargets(componentId), - topologyContext.getTargets(componentId)); - - for (String streamId : expectedContext.getComponentStreams(componentId)) { - Assert.assertEquals( - expectedContext.getComponentOutputFields(componentId, streamId).toList(), - topologyContext.getComponentOutputFields(componentId, streamId).toList()); - } - } - - for (String streamId : expectedContext.getThisStreams()) { - Assert.assertEquals(expectedContext.getThisOutputFields(streamId).toList(), - topologyContext.getThisOutputFields(streamId).toList()); - } - - HashMap taskToComponents = new HashMap(); - Set allTaskIds = new HashSet(); - for (String componentId : expectedContext.getComponentIds()) { - List possibleTasks = expectedContext.getComponentTasks(componentId); - List tasks = topologyContext.getComponentTasks(componentId); - - Iterator pIt = possibleTasks.iterator(); - Iterator tIt = tasks.iterator(); - while (pIt.hasNext()) { - Assert.assertTrue(tIt.hasNext()); - Assert.assertNull(taskToComponents.put(pIt.next(), componentId)); - Assert.assertTrue(allTaskIds.add(tIt.next())); - } - Assert.assertFalse(tIt.hasNext()); - } - - Assert.assertEquals(taskToComponents, expectedContext.getTaskToComponent()); - Assert.assertTrue(taskIds.add(topologyContext.getThisTaskId())); - - try { - topologyContext.getHooks(); - Assert.fail(); - } catch (UnsupportedOperationException e) { /* expected */ } - - try { - topologyContext.getRegisteredMetricByName(null); - Assert.fail(); - } catch (UnsupportedOperationException e) { /* expected */ } - } - } - -} From ca9460bc8e91e206deb83e20b7fde5d7cb0d9e8b Mon Sep 17 00:00:00 2001 From: zentol Date: Fri, 4 Jan 2019 13:33:11 +0100 Subject: [PATCH 095/147] [FLINK-11267][release] Only create hadoop-free binary by default --- tools/releasing/create_binary_release.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh index 444e01b28cee9d..137555356a9f8e 100755 --- a/tools/releasing/create_binary_release.sh +++ b/tools/releasing/create_binary_release.sh @@ -95,27 +95,15 @@ make_binary_release() { cd ${FLINK_DIR} } -HADOOP_CLASSIFIERS=("24" "26" "27" "28") -HADOOP_VERSIONS=("2.4.1" "2.6.5" "2.7.5" "2.8.3") - if [ "$SCALA_VERSION" == "none" ] && [ "$HADOOP_VERSION" == "none" ]; then make_binary_release "" "-DwithoutHadoop" "2.12" - for i in "${!HADOOP_CLASSIFIERS[@]}"; do - make_binary_release "hadoop${HADOOP_CLASSIFIERS[$i]}" "-Dhadoop.version=${HADOOP_VERSIONS[$i]}" "2.12" - done make_binary_release "" "-DwithoutHadoop" "2.11" - for i in "${!HADOOP_CLASSIFIERS[@]}"; do - make_binary_release "hadoop${HADOOP_CLASSIFIERS[$i]}" "-Dhadoop.version=${HADOOP_VERSIONS[$i]}" "2.11" - done elif [ "$SCALA_VERSION" == none ] && [ "$HADOOP_VERSION" != "none" ] then make_binary_release "hadoop2" "-Dhadoop.version=$HADOOP_VERSION" "2.11" elif [ "$SCALA_VERSION" != none ] && [ "$HADOOP_VERSION" == "none" ] then make_binary_release "" "-DwithoutHadoop" "$SCALA_VERSION" - for i in "${!HADOOP_CLASSIFIERS[@]}"; do - make_binary_release "hadoop${HADOOP_CLASSIFIERS[$i]}" "-Dhadoop.version=${HADOOP_VERSIONS[$i]}" "$SCALA_VERSION" - done else make_binary_release "hadoop2x" "-Dhadoop.version=$HADOOP_VERSION" "$SCALA_VERSION" fi From 4ec77d605895d202537b6c2d829568defec2bc5f Mon Sep 17 00:00:00 2001 From: kkloudas Date: Mon, 15 Oct 2018 14:28:50 +0200 Subject: [PATCH 096/147] [FLINK-10583][table] Add base TwoInputStreamOperator with TTL operator. --- ...nputStreamOperatorWithStateRetention.scala | 156 ++++++++++++++++ ...actTwoInputStreamOperatorWithTTLTest.scala | 174 ++++++++++++++++++ 2 files changed, 330 insertions(+) create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/BaseTwoInputStreamOperatorWithStateRetention.scala create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/AbstractTwoInputStreamOperatorWithTTLTest.scala diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/BaseTwoInputStreamOperatorWithStateRetention.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/BaseTwoInputStreamOperatorWithStateRetention.scala new file mode 100644 index 00000000000000..fda5c2e1686504 --- /dev/null +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/BaseTwoInputStreamOperatorWithStateRetention.scala @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.join + +import java.lang.{Long => JLong} +import java.util.Optional + +import org.apache.flink.annotation.Internal +import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} +import org.apache.flink.runtime.state.{VoidNamespace, VoidNamespaceSerializer} +import org.apache.flink.streaming.api.SimpleTimerService +import org.apache.flink.streaming.api.operators.{AbstractStreamOperator, InternalTimer, Triggerable, TwoInputStreamOperator} +import org.apache.flink.table.api.{StreamQueryConfig, Types} +import org.apache.flink.table.runtime.types.CRow + +/** + * An abstract [[TwoInputStreamOperator]] that allows its subclasses to clean + * up their state based on a TTL. This TTL should be specified in the provided + * [[StreamQueryConfig]]. + * + * For each known key, this operator registers a timer (in processing time) to + * fire after the TTL expires. When the timer fires, the subclass can decide which + * state to cleanup and what further action to take. + * + * This class takes care of maintaining at most one timer per key. + * + *

IMPORTANT NOTE TO USERS: When extending this class, do not use processing time + * timers in your business logic. The reason is that: + * + * 1) if your timers collide with clean up timers and you delete them, then state + * clean-up will not be performed, and + * + * 2) (this one is the reason why this class does not allow to override the onProcessingTime()) + * the onProcessingTime with your logic would be also executed on each clean up timer. + */ +@Internal +abstract class BaseTwoInputStreamOperatorWithStateRetention( + queryConfig: StreamQueryConfig) + extends AbstractStreamOperator[CRow] + with TwoInputStreamOperator[CRow, CRow, CRow] + with Triggerable[Any, VoidNamespace] { + + private val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime + private val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime + private val stateCleaningEnabled: Boolean = minRetentionTime > 1 + + private val CLEANUP_TIMESTAMP = "cleanup-timestamp" + private val TIMERS_STATE_NAME = "timers" + + private var latestRegisteredCleanUpTimer: ValueState[JLong] = _ + + protected var timerService: SimpleTimerService = _ + + override def open(): Unit = { + + initializeTimerService() + + if (stateCleaningEnabled) { + val cleanUpStateDescriptor: ValueStateDescriptor[JLong] = + new ValueStateDescriptor[JLong](CLEANUP_TIMESTAMP, Types.LONG) + latestRegisteredCleanUpTimer = getRuntimeContext.getState(cleanUpStateDescriptor) + } + } + + private def initializeTimerService(): Unit = { + val internalTimerService = getInternalTimerService( + TIMERS_STATE_NAME, + VoidNamespaceSerializer.INSTANCE, + this) + + timerService = new SimpleTimerService(internalTimerService) + } + + /** + * If the user has specified a `minRetentionTime` and `maxRetentionTime`, this + * method registers a cleanup timer for `currentProcessingTime + minRetentionTime`. + * + *

When this timer fires, the [[BaseTwoInputStreamOperatorWithStateRetention.cleanUpState()]] + * method is called. + */ + protected def registerProcessingCleanUpTimer(): Unit = { + if (stateCleaningEnabled) { + val currentProcessingTime = timerService.currentProcessingTime() + val currentCleanUpTime = Optional.ofNullable[JLong](latestRegisteredCleanUpTimer.value()) + + if (!currentCleanUpTime.isPresent + || (currentProcessingTime + minRetentionTime) > currentCleanUpTime.get) { + + updateCleanUpTimer(currentProcessingTime, currentCleanUpTime) + } + } + } + + private def updateCleanUpTimer( + currentProcessingTime: JLong, + currentCleanUpTime: Optional[JLong]): Unit = { + + if (currentCleanUpTime.isPresent) { + timerService.deleteProcessingTimeTimer(currentCleanUpTime.get()) + } + + val newCleanUpTime: JLong = currentProcessingTime + maxRetentionTime + timerService.registerProcessingTimeTimer(newCleanUpTime) + latestRegisteredCleanUpTimer.update(newCleanUpTime) + } + + protected def cleanUpLastTimer(): Unit = { + if (stateCleaningEnabled) { + val currentCleanUpTime = Optional.ofNullable[JLong](latestRegisteredCleanUpTimer.value()) + if (currentCleanUpTime.isPresent) { + latestRegisteredCleanUpTimer.clear() + timerService.deleteProcessingTimeTimer(currentCleanUpTime.get()) + } + } + } + + /** + * The users of this class are not allowed to use processing time timers. + * See class javadoc. + */ + override final def onProcessingTime(timer: InternalTimer[Any, VoidNamespace]): Unit = { + if (stateCleaningEnabled) { + val timerTime = timer.getTimestamp + val cleanupTime = latestRegisteredCleanUpTimer.value() + + if (cleanupTime != null && cleanupTime == timerTime) { + cleanUpState(cleanupTime) + latestRegisteredCleanUpTimer.clear() + } + } + } + + // ----------------- Abstract Methods ----------------- + + /** + * The method to be called when a cleanup timer fires. + * @param time The timestamp of the fired timer. + */ + def cleanUpState(time: Long): Unit +} diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/AbstractTwoInputStreamOperatorWithTTLTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/AbstractTwoInputStreamOperatorWithTTLTest.scala new file mode 100644 index 00000000000000..e8305b77b79ee0 --- /dev/null +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/AbstractTwoInputStreamOperatorWithTTLTest.scala @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.harness + +import java.lang.{Long => JLong} + +import org.apache.flink.api.common.time.Time +import org.apache.flink.api.common.typeinfo.BasicTypeInfo +import org.apache.flink.runtime.state.VoidNamespace +import org.apache.flink.streaming.api.operators.InternalTimer +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord +import org.apache.flink.streaming.util.KeyedTwoInputStreamOperatorTestHarness +import org.apache.flink.table.api.StreamQueryConfig +import org.apache.flink.table.runtime.harness.HarnessTestBase.{TestStreamQueryConfig, TupleRowKeySelector} +import org.apache.flink.table.runtime.join.BaseTwoInputStreamOperatorWithStateRetention +import org.apache.flink.table.runtime.types.CRow +import org.hamcrest.{Description, TypeSafeMatcher} +import org.junit.{After, Before, Test} +import org.hamcrest.MatcherAssert.assertThat + +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer + +/** + * Tests for the [[BaseTwoInputStreamOperatorWithStateRetention]]. + */ +class BaseTwoInputStreamOperatorWithStateRetentionTest extends HarnessTestBase { + + private val recordAForFirstKey = new StreamRecord(CRow(1L: JLong, "hello")) + private val recordBForFirstKey = new StreamRecord(CRow(1L: JLong, "world")) + + private val streamQueryConfig = new TestStreamQueryConfig( + Time.milliseconds(2), + Time.milliseconds(4) + ) + + private var operatorUnderTest: StubOperatorWithStateTTL = _ + + private var testHarness: KeyedTwoInputStreamOperatorTestHarness[JLong, CRow, CRow, CRow] = _ + + @Before + def createTestHarness(): Unit = { + operatorUnderTest = new StubOperatorWithStateTTL(streamQueryConfig) + testHarness = createTestHarness(operatorUnderTest) + testHarness.open() + } + + @After + def closeTestHarness(): Unit = { + testHarness.close() + } + + @Test + def normalScenarioWorks(): Unit = { + testHarness.setProcessingTime(1L) + testHarness.processElement1(recordAForFirstKey) + + testHarness.setProcessingTime(10L) + + assertThat(operatorUnderTest, hasFiredCleanUpTimersForTimestamps(5L)) + } + + @Test + def whenCurrentTimePlusMinRetentionSmallerThanCurrentCleanupTimeNoNewTimerRegistered(): Unit = { + testHarness.setProcessingTime(1L) + testHarness.processElement1(recordAForFirstKey) + + testHarness.setProcessingTime(2L) + testHarness.processElement1(recordBForFirstKey) + + testHarness.setProcessingTime(20L) + + assertThat(operatorUnderTest, hasFiredCleanUpTimersForTimestamps(5L)) + } + + @Test + def whenCurrentTimePlusMinRetentionLargerThanCurrentCleanupTimeTimerIsUpdated(): Unit = { + testHarness.setProcessingTime(1L) + testHarness.processElement1(recordAForFirstKey) + + testHarness.setProcessingTime(4L) + testHarness.processElement1(recordBForFirstKey) + + testHarness.setProcessingTime(20L) + + assertThat(operatorUnderTest, hasFiredCleanUpTimersForTimestamps(8L)) + } + + @Test + def otherSideToSameKeyStateAlsoUpdatesCleanupTimer(): Unit = { + testHarness.setProcessingTime(1L) + testHarness.processElement1(recordAForFirstKey) + + testHarness.setProcessingTime(4L) + testHarness.processElement2(recordBForFirstKey) + + testHarness.setProcessingTime(20L) + + assertThat(operatorUnderTest, hasFiredCleanUpTimersForTimestamps(8L)) + } + + // -------------------------------- Test Utilities -------------------------------- + + private def createTestHarness(operator: BaseTwoInputStreamOperatorWithStateRetention) = { + new KeyedTwoInputStreamOperatorTestHarness[JLong, CRow, CRow, CRow]( + operator, + new TupleRowKeySelector[JLong](0), + new TupleRowKeySelector[JLong](0), + BasicTypeInfo.LONG_TYPE_INFO, + 1, + 1, + 0) + } + + // -------------------------------- Matchers -------------------------------- + + private def hasFiredCleanUpTimersForTimestamps(timers: JLong*) = + new TypeSafeMatcher[StubOperatorWithStateTTL]() { + + override protected def matchesSafely(operator: StubOperatorWithStateTTL): Boolean = { + operator.firedCleanUpTimers.toArray.deep == timers.toArray.deep + } + + def describeTo(description: Description): Unit = { + description + .appendText("a list of timers with timestamps=") + .appendValue(timers.mkString(",")) + } + } + + // -------------------------------- Test Classes -------------------------------- + + /** + * A mock [[BaseTwoInputStreamOperatorWithStateRetention]] which registers + * the timestamps of the clean-up timers that fired (not the registered + * ones, which can be deleted without firing). + */ + class StubOperatorWithStateTTL( + queryConfig: StreamQueryConfig) + extends BaseTwoInputStreamOperatorWithStateRetention(queryConfig) { + + val firedCleanUpTimers: mutable.Buffer[JLong] = ArrayBuffer.empty + + override def cleanUpState(time: Long): Unit = { + firedCleanUpTimers.append(time) + } + + override def processElement1(element: StreamRecord[CRow]): Unit = { + registerProcessingCleanUpTimer() + } + + override def processElement2(element: StreamRecord[CRow]): Unit = { + registerProcessingCleanUpTimer() + } + + override def onEventTime(timer: InternalTimer[Any, VoidNamespace]): Unit = () + } +} From 81ec2d752881045921579b3d38e73238c58f18c7 Mon Sep 17 00:00:00 2001 From: Kostas Kloudas Date: Mon, 26 Nov 2018 16:36:58 +0100 Subject: [PATCH 097/147] [hotfix] Avoid redundant state access in TemporalProcessTimeJoin. --- .../flink/table/runtime/join/TemporalProcessTimeJoin.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalProcessTimeJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalProcessTimeJoin.scala index 7a8bb61b1cfce7..2d9fdf35538f2b 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalProcessTimeJoin.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalProcessTimeJoin.scala @@ -74,13 +74,13 @@ class TemporalProcessTimeJoin( override def processElement1(element: StreamRecord[CRow]): Unit = { - if (rightState.value() == null) { + val rightSideRow = rightState.value() + if (rightSideRow == null) { return } cRowWrapper.setChange(element.getValue.change) - val rightSideRow = rightState.value() joinFunction.join(element.getValue.row, rightSideRow, cRowWrapper) } From f8280313b76dda98323e8b4c0f8b23193ebe57a8 Mon Sep 17 00:00:00 2001 From: Kostas Kloudas Date: Tue, 27 Nov 2018 13:18:45 +0100 Subject: [PATCH 098/147] [FLINK-10584][table] Add State Retention support to TemporalRowtimeJoin. --- ...nputStreamOperatorWithStateRetention.scala | 3 +- .../runtime/join/TemporalRowtimeJoin.scala | 40 ++++-- .../harness/TemporalJoinHarnessTest.scala | 128 ++++++++++++++++++ .../TwoInputStreamOperatorTestHarness.java | 5 + 4 files changed, 160 insertions(+), 16 deletions(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/BaseTwoInputStreamOperatorWithStateRetention.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/BaseTwoInputStreamOperatorWithStateRetention.scala index fda5c2e1686504..f44d7997c8f6ac 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/BaseTwoInputStreamOperatorWithStateRetention.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/BaseTwoInputStreamOperatorWithStateRetention.scala @@ -58,13 +58,14 @@ abstract class BaseTwoInputStreamOperatorWithStateRetention( private val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime private val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime - private val stateCleaningEnabled: Boolean = minRetentionTime > 1 private val CLEANUP_TIMESTAMP = "cleanup-timestamp" private val TIMERS_STATE_NAME = "timers" private var latestRegisteredCleanUpTimer: ValueState[JLong] = _ + protected val stateCleaningEnabled: Boolean = minRetentionTime > 1 + protected var timerService: SimpleTimerService = _ override def open(): Unit = { diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalRowtimeJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalRowtimeJoin.scala index 66b60d44969fa0..f6911096f9371c 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalRowtimeJoin.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalRowtimeJoin.scala @@ -26,8 +26,7 @@ import org.apache.flink.api.common.functions.util.FunctionUtils import org.apache.flink.api.common.state._ import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation} import org.apache.flink.configuration.Configuration -import org.apache.flink.runtime.state.{VoidNamespace, VoidNamespaceSerializer} -import org.apache.flink.streaming.api.SimpleTimerService +import org.apache.flink.runtime.state.VoidNamespace import org.apache.flink.streaming.api.operators._ import org.apache.flink.streaming.runtime.streamrecord.StreamRecord import org.apache.flink.table.api.StreamQueryConfig @@ -69,9 +68,7 @@ class TemporalRowtimeJoin( queryConfig: StreamQueryConfig, leftTimeAttribute: Int, rightTimeAttribute: Int) - extends AbstractStreamOperator[CRow] - with TwoInputStreamOperator[CRow, CRow, CRow] - with Triggerable[Any, VoidNamespace] + extends BaseTwoInputStreamOperatorWithStateRetention(queryConfig) with Compiler[FlatJoinFunction[Row, Row, Row]] with Logging { @@ -112,7 +109,6 @@ class TemporalRowtimeJoin( private var cRowWrapper: CRowWrappingCollector = _ private var collector: TimestampedCollector[CRow] = _ - private var timerService: SimpleTimerService = _ private var joinFunction: FlatJoinFunction[Row, Row, Row] = _ @@ -142,12 +138,7 @@ class TemporalRowtimeJoin( cRowWrapper.out = collector cRowWrapper.setChange(true) - val internalTimerService = getInternalTimerService( - TIMERS_STATE_NAME, - VoidNamespaceSerializer.INSTANCE, - this) - - timerService = new SimpleTimerService(internalTimerService) + super.open() } override def processElement1(element: StreamRecord[CRow]): Unit = { @@ -155,6 +146,8 @@ class TemporalRowtimeJoin( leftState.put(getNextLeftIndex, element.getValue.row) registerSmallestTimer(getLeftTime(element.getValue.row)) // Timer to emit and clean up the state + + registerProcessingCleanUpTimer() } override def processElement2(element: StreamRecord[CRow]): Unit = { @@ -163,10 +156,8 @@ class TemporalRowtimeJoin( val rowTime = getRightTime(element.getValue.row) rightState.put(rowTime, element.getValue.row) registerSmallestTimer(rowTime) // Timer to clean up the state - } - override def onProcessingTime(timer: InternalTimer[Any, VoidNamespace]): Unit = { - throw new IllegalStateException("This should never happen") + registerProcessingCleanUpTimer() } override def onEventTime(timer: InternalTimer[Any, VoidNamespace]): Unit = { @@ -175,6 +166,15 @@ class TemporalRowtimeJoin( if (lastUnprocessedTime < Long.MaxValue) { registerTimer(lastUnprocessedTime) } + + // if we have more state at any side, then update the timer, else clean it up. + if (stateCleaningEnabled) { + if (lastUnprocessedTime < Long.MaxValue || rightState.iterator().hasNext) { + registerProcessingCleanUpTimer() + } else { + cleanUpLastTimer() + } + } } override def close(): Unit = { @@ -245,6 +245,16 @@ class TemporalRowtimeJoin( } } + /** + * The method to be called when a cleanup timer fires. + * + * @param time The timestamp of the fired timer. + */ + override def cleanUpState(time: Long): Unit = { + leftState.clear() + rightState.clear() + } + private def firstIndexToKeep(timerTimestamp: Long, rightRowsSorted: util.List[Row]): Int = { val firstIndexNewerThenTimer = indexOfFirstElementNewerThanTimer(timerTimestamp, rightRowsSorted) diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/TemporalJoinHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/TemporalJoinHarnessTest.scala index 0d46db4197079c..7b942c284ea57f 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/TemporalJoinHarnessTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/TemporalJoinHarnessTest.scala @@ -530,6 +530,134 @@ class TemporalJoinHarnessTest extends HarnessTestBase { 0) } + // ---------------------- Row time TTL tests ---------------------- + + @Test + def testRowTimeJoinCleanupTimerUpdatedFromProbeSide(): Unit = { + // min=2ms max=4ms + val testHarness = createTestHarness(new OrdersRatesRowtimeTemporalJoinInfo()) + + testHarness.open() + val expectedOutput = new ConcurrentLinkedQueue[Object]() + + testHarness.setProcessingTime(1L) + + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 1L))) + testHarness.processElement2(new StreamRecord(CRow("Euro", 114L, 0L))) + + expectedOutput.add(new StreamRecord(CRow(2L, "Euro", 1L, "Euro", 114L, 0L))) + + testHarness.processBothWatermarks(new Watermark(2L)) + + // this should update the clean-up timer to 8 + testHarness.setProcessingTime(4L) + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 4L))) + + expectedOutput.add(new StreamRecord(CRow(2L, "Euro", 4L, "Euro", 114L, 0L))) + + // this should now do nothing (also it does not update the timer as 5 + 2ms (min) < 8) + testHarness.setProcessingTime(5L) + + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 5L))) + expectedOutput.add(new StreamRecord(CRow(2L, "Euro", 5L, "Euro", 114L, 0L))) + + testHarness.processBothWatermarks(new Watermark(5L)) + + // this should now clean up the state + testHarness.setProcessingTime(8L) + + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 7L))) // this should find no match + + testHarness.processBothWatermarks(new Watermark(10L)) + + verify(expectedOutput, testHarness.getOutput) + + testHarness.close() + } + + @Test + def testRowTimeJoinCleanupTimerUpdatedFromBuildSide(): Unit = { + // min=2ms max=4ms + val testHarness = createTestHarness(new OrdersRatesRowtimeTemporalJoinInfo()) + + testHarness.open() + val expectedOutput = new ConcurrentLinkedQueue[Object]() + + testHarness.setProcessingTime(1L) + + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 1L))) + testHarness.processElement2(new StreamRecord(CRow("Euro", 114L, 0L))) + + expectedOutput.add(new StreamRecord(CRow(2L, "Euro", 1L, "Euro", 114L, 0L))) + + testHarness.processBothWatermarks(new Watermark(2L)) + + // this should update the clean-up timer to 8 + testHarness.setProcessingTime(4L) + testHarness.processElement2(new StreamRecord(CRow("Euro", 117L, 4L))) + + // this should now do nothing + testHarness.setProcessingTime(5L) + + // so this should be joined with the "old" value + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 3L))) + expectedOutput.add(new StreamRecord(CRow(2L, "Euro", 3L, "Euro", 114L, 0L))) + + testHarness.processBothWatermarks(new Watermark(5L)) + + // this should now clean up the state + testHarness.setProcessingTime(8L) + + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 7L))) // this should find no match + + testHarness.processBothWatermarks(new Watermark(10L)) + + verify(expectedOutput, testHarness.getOutput) + + testHarness.close() + } + + @Test + def testRowTimeJoinCleanupTimerUpdatedAfterEvaluation(): Unit = { + // min=2ms max=4ms + val testHarness = createTestHarness(new OrdersRatesRowtimeTemporalJoinInfo()) + + testHarness.open() + val expectedOutput = new ConcurrentLinkedQueue[Object]() + + testHarness.setProcessingTime(1L) + + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 1L))) + testHarness.processElement2(new StreamRecord(CRow("Euro", 114L, 0L))) + + expectedOutput.add(new StreamRecord(CRow(2L, "Euro", 1L, "Euro", 114L, 0L))) + + testHarness.setProcessingTime(4L) + + // this should trigger an evaluation, which should also update the clean-up timer to 8 + testHarness.processBothWatermarks(new Watermark(2L)) + + // this should now do nothing (also it does not update the timer as 5 + 2ms (min) < 8) + testHarness.setProcessingTime(5L) + + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 3L))) + expectedOutput.add(new StreamRecord(CRow(2L, "Euro", 3L, "Euro", 114L, 0L))) + + testHarness.processBothWatermarks(new Watermark(5L)) + + // this should now clean up the state + testHarness.setProcessingTime(8L) + + // so this should not find any match + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 7L))) + + testHarness.processBothWatermarks(new Watermark(10L)) + + verify(expectedOutput, testHarness.getOutput) + + testHarness.close() + } + def translateJoin(joinInfo: TemporalJoinInfo, joinRelType: JoinRelType = JoinRelType.INNER) : (CRowKeySelector, CRowKeySelector, TwoInputStreamOperator[CRow, CRow, CRow]) = { diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java index 7bb697331da6c6..9289252479a9bd 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/TwoInputStreamOperatorTestHarness.java @@ -64,4 +64,9 @@ public void processWatermark1(Watermark mark) throws Exception { public void processWatermark2(Watermark mark) throws Exception { twoInputOperator.processWatermark2(mark); } + + public void processBothWatermarks(Watermark mark) throws Exception { + twoInputOperator.processWatermark1(mark); + twoInputOperator.processWatermark2(mark); + } } From 120c4fc21abc9138d5a2c1c90fe9f3b5e3b9e437 Mon Sep 17 00:00:00 2001 From: Kostas Kloudas Date: Mon, 26 Nov 2018 17:27:24 +0100 Subject: [PATCH 099/147] [FLINK-10583][table] Add State Retention support to TemporalProcessTimeJoin. --- .../join/TemporalProcessTimeJoin.scala | 26 ++++++- .../harness/TemporalJoinHarnessTest.scala | 78 ++++++++++++++++++- 2 files changed, 100 insertions(+), 4 deletions(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalProcessTimeJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalProcessTimeJoin.scala index 2d9fdf35538f2b..ffda1e200af81e 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalProcessTimeJoin.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/TemporalProcessTimeJoin.scala @@ -22,7 +22,8 @@ import org.apache.flink.api.common.functions.util.FunctionUtils import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.operators.{AbstractStreamOperator, TimestampedCollector, TwoInputStreamOperator} +import org.apache.flink.runtime.state.VoidNamespace +import org.apache.flink.streaming.api.operators.{InternalTimer, TimestampedCollector} import org.apache.flink.streaming.runtime.streamrecord.StreamRecord import org.apache.flink.table.api.StreamQueryConfig import org.apache.flink.table.codegen.Compiler @@ -38,8 +39,7 @@ class TemporalProcessTimeJoin( genJoinFuncName: String, genJoinFuncCode: String, queryConfig: StreamQueryConfig) - extends AbstractStreamOperator[CRow] - with TwoInputStreamOperator[CRow, CRow, CRow] + extends BaseTwoInputStreamOperatorWithStateRetention(queryConfig) with Compiler[FlatJoinFunction[Row, Row, Row]] with Logging { @@ -70,6 +70,8 @@ class TemporalProcessTimeJoin( collector = new TimestampedCollector[CRow](output) cRowWrapper = new CRowWrappingCollector() cRowWrapper.out = collector + + super.open() } override def processElement1(element: StreamRecord[CRow]): Unit = { @@ -82,18 +84,36 @@ class TemporalProcessTimeJoin( cRowWrapper.setChange(element.getValue.change) joinFunction.join(element.getValue.row, rightSideRow, cRowWrapper) + + registerProcessingCleanUpTimer() } override def processElement2(element: StreamRecord[CRow]): Unit = { if (element.getValue.change) { rightState.update(element.getValue.row) + registerProcessingCleanUpTimer() } else { rightState.clear() + cleanUpLastTimer() } } override def close(): Unit = { FunctionUtils.closeFunction(joinFunction) } + + /** + * The method to be called when a cleanup timer fires. + * + * @param time The timestamp of the fired timer. + */ + override def cleanUpState(time: Long): Unit = { + rightState.clear() + } + + /** + * Invoked when an event-time timer fires. + */ + override def onEventTime(timer: InternalTimer[Any, VoidNamespace]): Unit = ??? } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/TemporalJoinHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/TemporalJoinHarnessTest.scala index 7b942c284ea57f..94c22e675f0b38 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/TemporalJoinHarnessTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/TemporalJoinHarnessTest.scala @@ -42,7 +42,7 @@ import org.apache.flink.table.runtime.CRowKeySelector import org.apache.flink.table.runtime.harness.HarnessTestBase.{RowResultSortComparator, TestStreamQueryConfig} import org.apache.flink.table.runtime.types.CRow import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo -import org.hamcrest.{CoreMatchers, Matcher} +import org.hamcrest.CoreMatchers import org.hamcrest.Matchers.{endsWith, startsWith} import org.junit.Assert.assertTrue import org.junit.Test @@ -658,6 +658,82 @@ class TemporalJoinHarnessTest extends HarnessTestBase { testHarness.close() } + // ---------------------- Processing time TTL tests ---------------------- + + @Test + def testProcessingTimeJoinCleanupTimerUpdatedFromProbeSide(): Unit = { + // min=2ms max=4ms + val testHarness = createTestHarness(new OrdersRatesProctimeTemporalJoinInfo()) + + testHarness.open() + val expectedOutput = new ConcurrentLinkedQueue[Object]() + + testHarness.setProcessingTime(1L) + + testHarness.processElement2(new StreamRecord(CRow("Euro", 114L, 0L))) + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 1L))) + + expectedOutput.add(new StreamRecord(CRow(2L, "Euro", 1L, "Euro", 114L, 0L))) + + // this should push further the clean-up the state + testHarness.setProcessingTime(4L) + + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 6L))) + expectedOutput.add(new StreamRecord(CRow(2L, "Euro", 6L, "Euro", 114L, 0L))) + + // this should do nothing + testHarness.setProcessingTime(5L) + + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 8L))) + expectedOutput.add(new StreamRecord(CRow(2L, "Euro", 8L, "Euro", 114L, 0L))) + + // this should clean up the state + testHarness.setProcessingTime(8L) + + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 10L))) + + verify(expectedOutput, testHarness.getOutput) + + testHarness.close() + } + + @Test + def testProcessingTimeJoinCleanupTimerUpdatedFromBuildSide(): Unit = { + // min=2ms max=4ms + val testHarness = createTestHarness(new OrdersRatesProctimeTemporalJoinInfo()) + + testHarness.open() + val expectedOutput = new ConcurrentLinkedQueue[Object]() + + testHarness.setProcessingTime(1L) + + testHarness.processElement2(new StreamRecord(CRow("Euro", 114L, 0L))) + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 1L))) + + expectedOutput.add(new StreamRecord(CRow(2L, "Euro", 1L, "Euro", 114L, 0L))) + + // this should push further the clean-up the state + testHarness.setProcessingTime(4L) + + testHarness.processElement2(new StreamRecord(CRow("Euro", 116L, 1L))) + + // this should do nothing + testHarness.setProcessingTime(5L) + + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 6L))) + expectedOutput.add(new StreamRecord(CRow(2L, "Euro", 6L, "Euro", 116L, 1L))) + + // this should clean up the state + testHarness.setProcessingTime(8L) + + // so this should find no match + testHarness.processElement1(new StreamRecord(CRow(2L, "Euro", 10L))) + + verify(expectedOutput, testHarness.getOutput) + + testHarness.close() + } + def translateJoin(joinInfo: TemporalJoinInfo, joinRelType: JoinRelType = JoinRelType.INNER) : (CRowKeySelector, CRowKeySelector, TwoInputStreamOperator[CRow, CRow, CRow]) = { From 6826c67028171b401df8bd32dad327051133d31a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=B2=E7=82=9C?= Date: Wed, 9 Jan 2019 11:13:38 +0800 Subject: [PATCH 100/147] [FLINK-11278] [docs] Add documentation for TableAPI&SQL in scala-shell. This closes #7437 --- docs/ops/scala_shell.md | 114 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 3 deletions(-) diff --git a/docs/ops/scala_shell.md b/docs/ops/scala_shell.md index e7df8647d22a3a..50987af2ae9e5a 100644 --- a/docs/ops/scala_shell.md +++ b/docs/ops/scala_shell.md @@ -36,9 +36,10 @@ cluster, please see the Setup section below. ## Usage -The shell supports Batch and Streaming. -Two different ExecutionEnvironments are automatically prebound after startup. -Use "benv" and "senv" to access the Batch and Streaming environment respectively. +The shell supports DataSet, DataStream, Table API and SQL. +Four different Environments are automatically prebound after startup. +Use "benv" and "senv" to access the Batch and Streaming ExecutionEnvironment respectively. +Use "btenv" and "stenv" to access BatchTableEnvironment and StreamTableEnvironment respectively. ### DataSet API @@ -85,6 +86,113 @@ Note, that in the Streaming case, the print operation does not trigger execution The Flink Shell comes with command history and auto-completion. +### Table API + +The example below is a wordcount program using Table API: +

+
+{% highlight scala %} +Scala-Flink> import org.apache.flink.table.functions.TableFunction +Scala-Flink> val textSource = stenv.fromDataStream( + senv.fromElements( + "To be, or not to be,--that is the question:--", + "Whether 'tis nobler in the mind to suffer", + "The slings and arrows of outrageous fortune", + "Or to take arms against a sea of troubles,"), + 'text) +Scala-Flink> class $Split extends TableFunction[String] { + def eval(s: String): Unit = { + s.toLowerCase.split("\\W+").foreach(collect) + } + } +Scala-Flink> val split = new $Split +Scala-Flink> textSource.join(split('text) as 'word). + groupBy('word).select('word, 'word.count as 'count). + toRetractStream[(String, Long)].print +Scala-Flink> senv.execute("Table Wordcount") +{% endhighlight %} +
+
+{% highlight scala %} +Scala-Flink> import org.apache.flink.table.functions.TableFunction +Scala-Flink> val textSource = btenv.fromDataSet( + benv.fromElements( + "To be, or not to be,--that is the question:--", + "Whether 'tis nobler in the mind to suffer", + "The slings and arrows of outrageous fortune", + "Or to take arms against a sea of troubles,"), + 'text) +Scala-Flink> class $Split extends TableFunction[String] { + def eval(s: String): Unit = { + s.toLowerCase.split("\\W+").foreach(collect) + } + } +Scala-Flink> val split = new $Split +Scala-Flink> textSource.join(split('text) as 'word). + groupBy('word).select('word, 'word.count as 'count). + toDataSet[(String, Long)].print +{% endhighlight %} +
+
+ +Note, that using $ as a prefix for the class name of TableFunction is a workaround of the issue that scala incorrectly generated inner class name. + +### SQL + +The following example is a wordcount program written in SQL: +
+
+{% highlight scala %} +Scala-Flink> import org.apache.flink.table.functions.TableFunction +Scala-Flink> val textSource = stenv.fromDataStream( + senv.fromElements( + "To be, or not to be,--that is the question:--", + "Whether 'tis nobler in the mind to suffer", + "The slings and arrows of outrageous fortune", + "Or to take arms against a sea of troubles,"), + 'text) +Scala-Flink> stenv.registerTable("text_source", textSource) +Scala-Flink> class $Split extends TableFunction[String] { + def eval(s: String): Unit = { + s.toLowerCase.split("\\W+").foreach(collect) + } + } +Scala-Flink> stenv.registerFunction("split", new $Split) +Scala-Flink> val result = stenv.sqlQuery("""SELECT T.word, count(T.word) AS `count` + FROM text_source + JOIN LATERAL table(split(text)) AS T(word) + ON TRUE + GROUP BY T.word""") +Scala-Flink> result.toRetractStream[(String, Long)].print +Scala-Flink> senv.execute("SQL Wordcount") +{% endhighlight %} +
+
+{% highlight scala %} +Scala-Flink> import org.apache.flink.table.functions.TableFunction +Scala-Flink> val textSource = btenv.fromDataSet( + benv.fromElements( + "To be, or not to be,--that is the question:--", + "Whether 'tis nobler in the mind to suffer", + "The slings and arrows of outrageous fortune", + "Or to take arms against a sea of troubles,"), + 'text) +Scala-Flink> btenv.registerTable("text_source", textSource) +Scala-Flink> class $Split extends TableFunction[String] { + def eval(s: String): Unit = { + s.toLowerCase.split("\\W+").foreach(collect) + } + } +Scala-Flink> btenv.registerFunction("split", new $Split) +Scala-Flink> val result = btenv.sqlQuery("""SELECT T.word, count(T.word) AS `count` + FROM text_source + JOIN LATERAL table(split(text)) AS T(word) + ON TRUE + GROUP BY T.word""") +Scala-Flink> result.toDataSet[(String, Long)].print +{% endhighlight %} +
+
## Adding external dependencies From 85dff904adb2b9fbe0588b3b68731af4970c28be Mon Sep 17 00:00:00 2001 From: Yangze Guo Date: Thu, 10 Jan 2019 16:46:42 +0800 Subject: [PATCH 101/147] [hotfix][docs] Reverse paragraphs in cli.md to cli enumeration (#7443) The markdown treats the TOC as the last item in CLI use list, thus this list could not be shown properly. This PR revert the two paragraph to fix this issue. --- docs/ops/cli.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/ops/cli.md b/docs/ops/cli.md index 0e658c10f27215..7da2a091b8bb23 100644 --- a/docs/ops/cli.md +++ b/docs/ops/cli.md @@ -30,11 +30,6 @@ distributed setups. It is located under `/bin/flink` and connects by default to the running Flink master (JobManager) that was started from the same installation directory. -A prerequisite to using the command line interface is that the Flink -master (JobManager) has been started (via -`/bin/start-cluster.sh`) or that a YARN environment is -available. - The command line can be used to - submit jobs for execution, @@ -44,6 +39,11 @@ The command line can be used to - trigger and dispose savepoints, and - modify a running job +A prerequisite to using the command line interface is that the Flink +master (JobManager) has been started (via +`/bin/start-cluster.sh`) or that a YARN environment is +available. + * This will be replaced by the TOC {:toc} From e8182dcd0987a7326a8c73df902c156e44a7cebf Mon Sep 17 00:00:00 2001 From: eaglewatcherwb Date: Fri, 4 Jan 2019 16:35:41 +0800 Subject: [PATCH 102/147] [FLINK-10866] Only load queryable state when explicitly configured Change-Id: Id79c8bf97002a387a80be563a43dce3210143dc2 --- .../queryable_state_configuration.html | 5 +++++ .../configuration/QueryableStateOptions.java | 10 ++++++++++ .../HAQueryableStateFsBackendITCase.java | 1 + .../HAQueryableStateRocksDBBackendITCase.java | 1 + .../NonHAQueryableStateFsBackendITCase.java | 1 + ...onHAQueryableStateRocksDBBackendITCase.java | 1 + .../runtime/io/network/NetworkEnvironment.java | 4 ++-- .../taskexecutor/TaskManagerServices.java | 18 ++++++++++++------ .../TaskManagerServicesConfiguration.java | 9 +++++++-- 9 files changed, 40 insertions(+), 10 deletions(-) diff --git a/docs/_includes/generated/queryable_state_configuration.html b/docs/_includes/generated/queryable_state_configuration.html index c457c40e147154..5940fdc822991d 100644 --- a/docs/_includes/generated/queryable_state_configuration.html +++ b/docs/_includes/generated/queryable_state_configuration.html @@ -42,5 +42,10 @@ 0 Number of query Threads for queryable state server. Uses the number of slots if set to 0. + +
queryable-state.enable
+ false + Option whether the queryable state proxy and server should be enabled where possible and configurable. + diff --git a/flink-core/src/main/java/org/apache/flink/configuration/QueryableStateOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/QueryableStateOptions.java index 6a8041926b7dc5..20c6b53a2090e0 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/QueryableStateOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/QueryableStateOptions.java @@ -106,6 +106,16 @@ public class QueryableStateOptions { .defaultValue(0) .withDescription("Number of query Threads for queryable state server. Uses the number of slots if set to 0."); + /** Option whether the queryable state proxy and server should be enabled where possible and configurable. + * + *

Queryable state proxy and server are still more experimental features, hence disabled unless they are enable + * in user configuration. */ + public static final ConfigOption ENABLE_QUERYABLE_STATE_PROXY_SERVER = + key("queryable-state.enable") + .defaultValue(false) + .withDescription("Option whether the queryable state proxy and server should be enabled where possible" + + " and configurable."); + // ------------------------------------------------------------------------ // Client Options // ------------------------------------------------------------------------ diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java index ef6a267df0fbdd..43fdf13a4bb214 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java @@ -95,6 +95,7 @@ public static void tearDown() throws Exception { private static Configuration getConfig() throws Exception { Configuration config = new Configuration(); + config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true); config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS); config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java index 46202c322cb5c6..8a622436977fb1 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java @@ -95,6 +95,7 @@ public static void tearDown() throws Exception { private static Configuration getConfig() throws Exception { Configuration config = new Configuration(); + config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true); config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS); config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java index f1382556eb7c7d..183ad9e59b5c15 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java @@ -79,6 +79,7 @@ public static void tearDown() { private static Configuration getConfig() { Configuration config = new Configuration(); + config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true); config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java index 3ad0dda52e2be9..34713f38f0209b 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java @@ -78,6 +78,7 @@ public static void tearDown() { private static Configuration getConfig() { Configuration config = new Configuration(); + config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true); config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java index d124456c213a15..89e23dac1b23e9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java @@ -345,7 +345,7 @@ public void start() throws IOException { } catch (Throwable ie) { kvStateServer.shutdown(); kvStateServer = null; - throw new IOException("Failed to start the Queryable State Data Server.", ie); + LOG.error("Failed to start the Queryable State Data Server.", ie); } } @@ -355,7 +355,7 @@ public void start() throws IOException { } catch (Throwable ie) { kvStateProxy.shutdown(); kvStateProxy = null; - throw new IOException("Failed to start the Queryable State Client Proxy.", ie); + LOG.error("Failed to start the Queryable State Client Proxy.", ie); } } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java index c46d800bc95349..f3feb2dab71992 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java @@ -419,32 +419,38 @@ private static NetworkEnvironment createNetworkEnvironment( QueryableStateConfiguration qsConfig = taskManagerServicesConfiguration.getQueryableStateConfig(); - int numProxyServerNetworkThreads = qsConfig.numProxyServerThreads() == 0 ? + KvStateClientProxy kvClientProxy = null; + KvStateServer kvStateServer = null; + + if (qsConfig != null) { + int numProxyServerNetworkThreads = qsConfig.numProxyServerThreads() == 0 ? taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numProxyServerThreads(); - int numProxyServerQueryThreads = qsConfig.numProxyQueryThreads() == 0 ? + int numProxyServerQueryThreads = qsConfig.numProxyQueryThreads() == 0 ? taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numProxyQueryThreads(); - final KvStateClientProxy kvClientProxy = QueryableStateUtils.createKvStateClientProxy( + + kvClientProxy = QueryableStateUtils.createKvStateClientProxy( taskManagerServicesConfiguration.getTaskManagerAddress(), qsConfig.getProxyPortRange(), numProxyServerNetworkThreads, numProxyServerQueryThreads, new DisabledKvStateRequestStats()); - int numStateServerNetworkThreads = qsConfig.numStateServerThreads() == 0 ? + int numStateServerNetworkThreads = qsConfig.numStateServerThreads() == 0 ? taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numStateServerThreads(); - int numStateServerQueryThreads = qsConfig.numStateQueryThreads() == 0 ? + int numStateServerQueryThreads = qsConfig.numStateQueryThreads() == 0 ? taskManagerServicesConfiguration.getNumberOfSlots() : qsConfig.numStateQueryThreads(); - final KvStateServer kvStateServer = QueryableStateUtils.createKvStateServer( + kvStateServer = QueryableStateUtils.createKvStateServer( taskManagerServicesConfiguration.getTaskManagerAddress(), qsConfig.getStateServerPortRange(), numStateServerNetworkThreads, numStateServerQueryThreads, kvStateRegistry, new DisabledKvStateRequestStats()); + } // we start the network first, to make sure it can allocate its buffers first return new NetworkEnvironment( diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java index c27e54d7649727..5acbddebfeda77 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java @@ -39,6 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.Iterator; @@ -66,6 +67,7 @@ public class TaskManagerServicesConfiguration { private final NetworkEnvironmentConfiguration networkConfig; + @Nullable private final QueryableStateConfiguration queryableStateConfig; /** @@ -93,7 +95,7 @@ public TaskManagerServicesConfiguration( String[] localRecoveryStateRootDirectories, boolean localRecoveryEnabled, NetworkEnvironmentConfiguration networkConfig, - QueryableStateConfiguration queryableStateConfig, + @Nullable QueryableStateConfiguration queryableStateConfig, int numberOfSlots, long configuredMemory, MemoryType memoryType, @@ -107,7 +109,7 @@ public TaskManagerServicesConfiguration( this.localRecoveryStateRootDirectories = checkNotNull(localRecoveryStateRootDirectories); this.localRecoveryEnabled = checkNotNull(localRecoveryEnabled); this.networkConfig = checkNotNull(networkConfig); - this.queryableStateConfig = checkNotNull(queryableStateConfig); + this.queryableStateConfig = queryableStateConfig; this.numberOfSlots = checkNotNull(numberOfSlots); this.configuredMemory = configuredMemory; @@ -466,6 +468,9 @@ public static boolean hasNewNetworkBufConf(final Configuration config) { * Creates the {@link QueryableStateConfiguration} from the given Configuration. */ private static QueryableStateConfiguration parseQueryableStateConfiguration(Configuration config) { + if (!config.getBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER)) { + return null; + } final Iterator proxyPorts = NetUtils.getPortRangeFromString( config.getString(QueryableStateOptions.PROXY_PORT_RANGE)); From f692d372f7cb1ecd04ad82d9d546fe68c702ce05 Mon Sep 17 00:00:00 2001 From: leesf <490081539@qq.com> Date: Mon, 7 Jan 2019 20:17:22 +0800 Subject: [PATCH 103/147] [FLINK-11272][flink-yarn] Support for parsing multiple --yarnship parameters --- docs/ops/cli.md | 3 ++- .../flink/yarn/cli/FlinkYarnSessionCli.java | 16 +++++++++------- .../flink/yarn/FlinkYarnSessionCliTest.java | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/docs/ops/cli.md b/docs/ops/cli.md index 7da2a091b8bb23..ec3596fa55d2b5 100644 --- a/docs/ops/cli.md +++ b/docs/ops/cli.md @@ -295,7 +295,8 @@ Action "run" compiles and runs a program. -ys,--yarnslots Number of slots per TaskManager -yst,--yarnstreaming Start Flink in streaming mode -yt,--yarnship Ship files in the specified directory - (t for transfer) + (t for transfer), multiple options are + supported. -ytm,--yarntaskManagerMemory Memory per TaskManager Container with optional unit (default: MB) -yz,--yarnzookeeperNamespace Namespace to create the Zookeeper diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java index 931b22c0373259..9a05c16b2af661 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java @@ -311,14 +311,16 @@ private AbstractYarnClusterDescriptor createDescriptor( } List shipFiles = new ArrayList<>(); - // path to directory to ship + // path to directories to ship if (cmd.hasOption(shipPath.getOpt())) { - String shipPath = cmd.getOptionValue(this.shipPath.getOpt()); - File shipDir = new File(shipPath); - if (shipDir.isDirectory()) { - shipFiles.add(shipDir); - } else { - LOG.warn("Ship directory is not a directory. Ignoring it."); + String[] shipPaths = cmd.getOptionValues(this.shipPath.getOpt()); + for (String shipPath : shipPaths) { + File shipDir = new File(shipPath); + if (shipDir.isDirectory()) { + shipFiles.add(shipDir); + } else { + LOG.warn("Ship directory {} is not a directory. Ignoring it.", shipDir.getAbsolutePath()); + } } } diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java index aa958e25141382..3ee3654d15ae41 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java @@ -450,6 +450,23 @@ public void testHeapMemoryPropertyWithConfigDefaultValue() throws Exception { assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(1024)); } + @Test + public void testMultipleYarnShipOptions() throws Exception { + final String[] args = new String[]{"run", "--yarnship", tmp.newFolder().getAbsolutePath(), "--yarnship", tmp.newFolder().getAbsolutePath()}; + final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( + new Configuration(), + tmp.getRoot().getAbsolutePath(), + "y", + "yarn"); + + final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false); + + AbstractYarnClusterDescriptor flinkYarnDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine); + + assertEquals(2, flinkYarnDescriptor.shipFiles.size()); + + } + /////////// // Utils // From c06e7a5427675efb63aff86debd03581343cfdb7 Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 20 Dec 2018 14:18:46 +0100 Subject: [PATCH 104/147] [FLINK-11134][rest] Do not log stacktrace for handled exceptions --- .../flink/runtime/rest/handler/AbstractHandler.java | 9 ++++++--- .../flink/runtime/rest/handler/AbstractRestHandler.java | 6 +++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractHandler.java index 865bdb1c9248ac..a87d3ada5eb8be 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractHandler.java @@ -116,15 +116,13 @@ protected void respondAsLeader(ChannelHandlerContext ctx, RoutedRequest routedRe try { request = MAPPER.readValue("{}", untypedResponseMessageHeaders.getRequestClass()); } catch (JsonParseException | JsonMappingException je) { - log.error("Request did not conform to expected format.", je); - throw new RestHandlerException("Bad request received.", HttpResponseStatus.BAD_REQUEST, je); + throw new RestHandlerException("Bad request received. Request did not conform to expected format.", HttpResponseStatus.BAD_REQUEST, je); } } else { try { ByteBufInputStream in = new ByteBufInputStream(msgContent); request = MAPPER.readValue(in, untypedResponseMessageHeaders.getRequestClass()); } catch (JsonParseException | JsonMappingException je) { - log.error("Failed to read request.", je); throw new RestHandlerException( String.format("Request did not match expected format %s.", untypedResponseMessageHeaders.getRequestClass().getSimpleName()), HttpResponseStatus.BAD_REQUEST, @@ -164,6 +162,11 @@ protected void respondAsLeader(ChannelHandlerContext ctx, RoutedRequest routedRe }); } catch (RestHandlerException rhe) { inFlightRequestTracker.deregisterRequest(); + if (log.isDebugEnabled()) { + log.error("Exception occurred in REST handler.", rhe); + } else { + log.error("Exception occurred in REST handler: {}", rhe.getMessage()); + } HandlerUtils.sendErrorResponse( ctx, httpRequest, diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractRestHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractRestHandler.java index a0cff90e9d5cd9..d88abaabc66aa3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractRestHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractRestHandler.java @@ -88,7 +88,11 @@ private Tuple2 errorResponse(Throwable throwab Throwable error = ExceptionUtils.stripCompletionException(throwable); if (error instanceof RestHandlerException) { final RestHandlerException rhe = (RestHandlerException) error; - log.error("Exception occurred in REST handler.", rhe); + if (log.isDebugEnabled()) { + log.error("Exception occurred in REST handler.", rhe); + } else { + log.error("Exception occurred in REST handler: {}", rhe.getMessage()); + } return Tuple2.of(new ErrorResponseBody(rhe.getMessage()), rhe.getHttpResponseStatus()); } else { log.error("Implementation error: Unhandled exception.", error); From 1f47c0ac27ad42d3c4b066914a351e23a6cacbef Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 3 Jan 2019 17:29:56 +0100 Subject: [PATCH 105/147] [hotfix][rest] Centralize REST error logging --- .../runtime/rest/handler/AbstractHandler.java | 42 +++++++++++++------ .../rest/handler/AbstractRestHandler.java | 28 +------------ .../AbstractTaskManagerFileHandler.java | 25 ++++------- 3 files changed, 40 insertions(+), 55 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractHandler.java index a87d3ada5eb8be..cc9f355471085a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractHandler.java @@ -31,6 +31,7 @@ import org.apache.flink.runtime.webmonitor.RestfulGateway; import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; import org.apache.flink.util.AutoCloseableAsync; +import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.Preconditions; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonParseException; @@ -50,6 +51,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.util.Arrays; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -157,33 +159,49 @@ protected void respondAsLeader(ChannelHandlerContext ctx, RoutedRequest routedRe final FileUploads finalUploadedFiles = uploadedFiles; requestProcessingFuture .whenComplete((Void ignored, Throwable throwable) -> { - inFlightRequestTracker.deregisterRequest(); - cleanupFileUploads(finalUploadedFiles); + if (throwable != null) { + handleException(ExceptionUtils.stripCompletionException(throwable), ctx, httpRequest) + .whenComplete((Void ignored2, Throwable throwable2) -> finalizeRequestProcessing(finalUploadedFiles)); + } else { + finalizeRequestProcessing(finalUploadedFiles); + } }); - } catch (RestHandlerException rhe) { - inFlightRequestTracker.deregisterRequest(); + } catch (Throwable e) { + final FileUploads finalUploadedFiles = uploadedFiles; + handleException(e, ctx, httpRequest) + .whenComplete((Void ignored, Throwable throwable) -> finalizeRequestProcessing(finalUploadedFiles)); + } + } + + private void finalizeRequestProcessing(FileUploads uploadedFiles) { + inFlightRequestTracker.deregisterRequest(); + cleanupFileUploads(uploadedFiles); + } + + private CompletableFuture handleException(Throwable throwable, ChannelHandlerContext ctx, HttpRequest httpRequest) { + if (throwable instanceof RestHandlerException) { + RestHandlerException rhe = (RestHandlerException) throwable; if (log.isDebugEnabled()) { log.error("Exception occurred in REST handler.", rhe); } else { log.error("Exception occurred in REST handler: {}", rhe.getMessage()); } - HandlerUtils.sendErrorResponse( + return HandlerUtils.sendErrorResponse( ctx, httpRequest, new ErrorResponseBody(rhe.getMessage()), rhe.getHttpResponseStatus(), responseHeaders); - cleanupFileUploads(uploadedFiles); - } catch (Throwable e) { - inFlightRequestTracker.deregisterRequest(); - log.error("Request processing failed.", e); - HandlerUtils.sendErrorResponse( + } else { + log.error("Implementation error: Unhandled exception.", throwable); + String stackTrace = String.format("", + ExceptionUtils.stringifyException(throwable)); + return HandlerUtils.sendErrorResponse( ctx, httpRequest, - new ErrorResponseBody("Internal server error."), + new ErrorResponseBody(Arrays.asList("Internal server error.", stackTrace)), HttpResponseStatus.INTERNAL_SERVER_ERROR, responseHeaders); - cleanupFileUploads(uploadedFiles); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractRestHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractRestHandler.java index d88abaabc66aa3..88a0c5e4cef0dc 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractRestHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/AbstractRestHandler.java @@ -19,17 +19,14 @@ package org.apache.flink.runtime.rest.handler; import org.apache.flink.api.common.time.Time; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.rest.handler.util.HandlerUtils; -import org.apache.flink.runtime.rest.messages.ErrorResponseBody; import org.apache.flink.runtime.rest.messages.MessageHeaders; import org.apache.flink.runtime.rest.messages.MessageParameters; import org.apache.flink.runtime.rest.messages.RequestBody; import org.apache.flink.runtime.rest.messages.ResponseBody; import org.apache.flink.runtime.webmonitor.RestfulGateway; import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; -import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.Preconditions; import org.apache.flink.shaded.netty4.io.netty.channel.ChannelHandler; @@ -39,7 +36,6 @@ import javax.annotation.Nonnull; -import java.util.Arrays; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -79,29 +75,7 @@ protected CompletableFuture respondToRequest(ChannelHandlerContext ctx, Ht response = FutureUtils.completedExceptionally(e); } - return response.handle((resp, throwable) -> throwable != null ? - errorResponse(throwable) : Tuple2.of(resp, messageHeaders.getResponseStatusCode())) - .thenCompose(r -> HandlerUtils.sendResponse(ctx, httpRequest, r.f0, r.f1, responseHeaders)); - } - - private Tuple2 errorResponse(Throwable throwable) { - Throwable error = ExceptionUtils.stripCompletionException(throwable); - if (error instanceof RestHandlerException) { - final RestHandlerException rhe = (RestHandlerException) error; - if (log.isDebugEnabled()) { - log.error("Exception occurred in REST handler.", rhe); - } else { - log.error("Exception occurred in REST handler: {}", rhe.getMessage()); - } - return Tuple2.of(new ErrorResponseBody(rhe.getMessage()), rhe.getHttpResponseStatus()); - } else { - log.error("Implementation error: Unhandled exception.", error); - String stackTrace = String.format("", - ExceptionUtils.stringifyException(throwable)); - return Tuple2.of( - new ErrorResponseBody(Arrays.asList("Internal server error.", stackTrace)), - HttpResponseStatus.INTERNAL_SERVER_ERROR); - } + return response.thenAccept(resp -> HandlerUtils.sendResponse(ctx, httpRequest, resp, messageHeaders.getResponseStatusCode(), responseHeaders)); } /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/taskmanager/AbstractTaskManagerFileHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/taskmanager/AbstractTaskManagerFileHandler.java index 82ca82cdfec9b6..c2d7a284d01bd5 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/taskmanager/AbstractTaskManagerFileHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/taskmanager/AbstractTaskManagerFileHandler.java @@ -24,12 +24,11 @@ import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; import org.apache.flink.runtime.resourcemanager.exceptions.UnknownTaskExecutorException; +import org.apache.flink.runtime.rest.NotFoundException; import org.apache.flink.runtime.rest.handler.AbstractHandler; import org.apache.flink.runtime.rest.handler.HandlerRequest; import org.apache.flink.runtime.rest.handler.RestHandlerException; -import org.apache.flink.runtime.rest.handler.util.HandlerUtils; import org.apache.flink.runtime.rest.messages.EmptyRequestBody; -import org.apache.flink.runtime.rest.messages.ErrorResponseBody; import org.apache.flink.runtime.rest.messages.UntypedResponseMessageHeaders; import org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerIdPathParameter; import org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerMessageParameters; @@ -75,7 +74,6 @@ import static org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION; import static org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE; -import static org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR; import static org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus.OK; import static org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpVersion.HTTP_1_1; @@ -159,23 +157,18 @@ protected CompletableFuture respondToRequest(ChannelHandlerContext ctx, Ht fileBlobKeys.invalidate(taskManagerId); final Throwable strippedThrowable = ExceptionUtils.stripCompletionException(throwable); - final ErrorResponseBody errorResponseBody; - final HttpResponseStatus httpResponseStatus; if (strippedThrowable instanceof UnknownTaskExecutorException) { - errorResponseBody = new ErrorResponseBody("Unknown TaskExecutor " + taskManagerId + '.'); - httpResponseStatus = HttpResponseStatus.NOT_FOUND; + throw new CompletionException( + new NotFoundException( + String.format("Failed to transfer file from TaskExecutor %s because it was unknown.", taskManagerId), + strippedThrowable)); } else { - errorResponseBody = new ErrorResponseBody("Internal server error: " + throwable.getMessage() + '.'); - httpResponseStatus = INTERNAL_SERVER_ERROR; + throw new CompletionException( + new FlinkException( + String.format("Failed to transfer file from TaskExecutor %s.", taskManagerId), + strippedThrowable)); } - - HandlerUtils.sendErrorResponse( - ctx, - httpRequest, - errorResponseBody, - httpResponseStatus, - responseHeaders); } }); } From 4adf23d3b93c301b53f11f1c05a1aa6d37112d46 Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 3 Jan 2019 17:30:53 +0100 Subject: [PATCH 106/147] [hotfix][rest] Remove unnecessary instanceof check --- .../handler/taskmanager/AbstractTaskManagerFileHandler.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/taskmanager/AbstractTaskManagerFileHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/taskmanager/AbstractTaskManagerFileHandler.java index c2d7a284d01bd5..17d1b178baced9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/taskmanager/AbstractTaskManagerFileHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/taskmanager/AbstractTaskManagerFileHandler.java @@ -123,11 +123,7 @@ protected CompletableFuture respondToRequest(ChannelHandlerContext ctx, Ht blobKeyFuture = fileBlobKeys.get(taskManagerId); } catch (ExecutionException e) { final Throwable cause = ExceptionUtils.stripExecutionException(e); - if (cause instanceof RestHandlerException) { - throw (RestHandlerException) cause; - } else { - throw new RestHandlerException("Could not retrieve file blob key future.", HttpResponseStatus.INTERNAL_SERVER_ERROR, e); - } + throw new RestHandlerException("Could not retrieve file blob key future.", HttpResponseStatus.INTERNAL_SERVER_ERROR, cause); } final CompletableFuture resultFuture = blobKeyFuture.thenAcceptAsync( From 5fe918c7946d7e585bc18346751fabfc3439b046 Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 3 Jan 2019 12:01:33 +0100 Subject: [PATCH 107/147] [FLINK-11268][release] Deploy multiple flink-shaded-hadoop2 artifacts --- .../flink-connector-filesystem/pom.xml | 2 +- .../flink-hadoop-compatibility/pom.xml | 2 +- flink-connectors/flink-hbase/pom.xml | 2 +- flink-connectors/flink-hcatalog/pom.xml | 2 +- flink-connectors/flink-orc/pom.xml | 2 +- flink-dist/pom.xml | 2 +- flink-dist/src/main/assemblies/hadoop.xml | 4 ++-- .../flink-bucketing-sink-test/pom.xml | 2 +- flink-filesystems/flink-hadoop-fs/pom.xml | 2 +- flink-formats/flink-parquet/pom.xml | 2 +- flink-formats/flink-sequence-file/pom.xml | 2 +- flink-fs-tests/pom.xml | 2 +- flink-runtime/pom.xml | 2 +- .../flink-shaded-hadoop2-uber/pom.xml | 14 +------------- flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml | 1 + flink-tests/pom.xml | 2 +- flink-yarn-tests/pom.xml | 2 +- flink-yarn/pom.xml | 2 +- tools/releasing/deploy_staging_jars.sh | 8 ++++++++ 19 files changed, 27 insertions(+), 30 deletions(-) diff --git a/flink-connectors/flink-connector-filesystem/pom.xml b/flink-connectors/flink-connector-filesystem/pom.xml index c375ac467d689c..b49453034d8e69 100644 --- a/flink-connectors/flink-connector-filesystem/pom.xml +++ b/flink-connectors/flink-connector-filesystem/pom.xml @@ -53,7 +53,7 @@ under the License. org.apache.flink flink-shaded-hadoop2 - ${project.version} + ${project.version}-${hadoop.version} provided diff --git a/flink-connectors/flink-hadoop-compatibility/pom.xml b/flink-connectors/flink-hadoop-compatibility/pom.xml index 8235bcabd0973d..fb61f6ae4b15d7 100644 --- a/flink-connectors/flink-hadoop-compatibility/pom.xml +++ b/flink-connectors/flink-hadoop-compatibility/pom.xml @@ -63,7 +63,7 @@ under the License. org.apache.flink flink-shaded-hadoop2 - ${project.version} + ${project.version}-${hadoop.version} provided diff --git a/flink-connectors/flink-hbase/pom.xml b/flink-connectors/flink-hbase/pom.xml index 2ab0ec4c2aa6a4..2692dc7b94db65 100644 --- a/flink-connectors/flink-hbase/pom.xml +++ b/flink-connectors/flink-hbase/pom.xml @@ -105,7 +105,7 @@ under the License. org.apache.flink flink-shaded-hadoop2 - ${project.version} + ${project.version}-${hadoop.version} provided diff --git a/flink-connectors/flink-hcatalog/pom.xml b/flink-connectors/flink-hcatalog/pom.xml index c7bfe2eeb4de1c..4bd377a2997a36 100644 --- a/flink-connectors/flink-hcatalog/pom.xml +++ b/flink-connectors/flink-hcatalog/pom.xml @@ -69,7 +69,7 @@ under the License. org.apache.flink flink-shaded-hadoop2 - ${project.version} + ${project.version}-${hadoop.version} provided diff --git a/flink-connectors/flink-orc/pom.xml b/flink-connectors/flink-orc/pom.xml index 733049a40962a3..4fcfe6004f4dd4 100644 --- a/flink-connectors/flink-orc/pom.xml +++ b/flink-connectors/flink-orc/pom.xml @@ -71,7 +71,7 @@ under the License. org.apache.flink flink-shaded-hadoop2 - ${project.version} + ${project.version}-${hadoop.version} provided diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml index e5991e1222840d..84339ebbecac71 100644 --- a/flink-dist/pom.xml +++ b/flink-dist/pom.xml @@ -179,7 +179,7 @@ under the License. org.apache.flink flink-shaded-hadoop2-uber - ${project.version} + ${project.version}-${hadoop.version} - ../flink-shaded-hadoop/flink-shaded-hadoop2-uber/target/flink-shaded-hadoop2-uber-${project.version}.jar + ../flink-shaded-hadoop/flink-shaded-hadoop2-uber/target/flink-shaded-hadoop2-uber-${project.version}-${hadoop.version}.jar lib/ - flink-shaded-hadoop2-uber-${project.version}.jar + flink-shaded-hadoop2-uber-${project.version}-${hadoop.version}.jar 0644 diff --git a/flink-end-to-end-tests/flink-bucketing-sink-test/pom.xml b/flink-end-to-end-tests/flink-bucketing-sink-test/pom.xml index 808cef7500af3b..2b2a0acd7f4b60 100644 --- a/flink-end-to-end-tests/flink-bucketing-sink-test/pom.xml +++ b/flink-end-to-end-tests/flink-bucketing-sink-test/pom.xml @@ -49,7 +49,7 @@ org.apache.flink flink-shaded-hadoop2 - ${project.version} + ${project.version}-${hadoop.version} provided diff --git a/flink-filesystems/flink-hadoop-fs/pom.xml b/flink-filesystems/flink-hadoop-fs/pom.xml index 12c686899ec6dd..468e3550f5cfb3 100644 --- a/flink-filesystems/flink-hadoop-fs/pom.xml +++ b/flink-filesystems/flink-hadoop-fs/pom.xml @@ -45,7 +45,7 @@ under the License. org.apache.flink flink-shaded-hadoop2 - ${project.version} + ${project.version}-${hadoop.version} true diff --git a/flink-formats/flink-parquet/pom.xml b/flink-formats/flink-parquet/pom.xml index 9caa5fb975ef63..46f1014011a348 100644 --- a/flink-formats/flink-parquet/pom.xml +++ b/flink-formats/flink-parquet/pom.xml @@ -61,7 +61,7 @@ under the License. org.apache.flink flink-shaded-hadoop2 - ${project.version} + ${project.version}-${hadoop.version} provided diff --git a/flink-formats/flink-sequence-file/pom.xml b/flink-formats/flink-sequence-file/pom.xml index 1a2f84e4629878..c1b014596f1ab9 100644 --- a/flink-formats/flink-sequence-file/pom.xml +++ b/flink-formats/flink-sequence-file/pom.xml @@ -47,7 +47,7 @@ under the License. org.apache.flink flink-shaded-hadoop2 - ${project.version} + ${project.version}-${hadoop.version} provided diff --git a/flink-fs-tests/pom.xml b/flink-fs-tests/pom.xml index d9e5909cbc130c..ba675737b8b912 100644 --- a/flink-fs-tests/pom.xml +++ b/flink-fs-tests/pom.xml @@ -39,7 +39,7 @@ under the License. org.apache.flink flink-shaded-hadoop2 - ${project.version} + ${project.version}-${hadoop.version} test diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml index 415bc35c1e434f..03b99530a3a4a3 100644 --- a/flink-runtime/pom.xml +++ b/flink-runtime/pom.xml @@ -70,7 +70,7 @@ under the License. org.apache.flink flink-shaded-hadoop2 - ${project.version} + ${project.version}-${hadoop.version} true diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml index 1196c513eb34e0..b9658670667f42 100644 --- a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml +++ b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml @@ -34,6 +34,7 @@ under the License. flink-shaded-hadoop2-uber jar + 1.8-SNAPSHOT-${hadoop.version} - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - -[Flink streaming]({{ site.baseurl }}/dev/datastream_api.html) is compatible with Apache Storm interfaces and therefore allows -reusing code that was implemented for Storm. - -You can use Storm `Spout`/`Bolt` as source/operator in Flink streaming programs. - -This document shows how to use existing Storm code with Flink. - -* This will be replaced by the TOC -{:toc} - -# Project Configuration - -Support for Storm is contained in the `flink-storm` Maven module. -The code resides in the `org.apache.flink.storm` package. - -Add the following dependency to your `pom.xml` if you want to execute Storm code in Flink. - -{% highlight xml %} - - org.apache.flink - flink-storm{{ site.scala_version_suffix }} - {{site.version}} - -{% endhighlight %} - -**Please note**: Do not add `storm-core` as a dependency. It is already included via `flink-storm`. - -**Please note**: `flink-storm` is not part of the provided binary Flink distribution. -Thus, you need to include `flink-storm` classes (and their dependencies) in your program jar (also called uber-jar or fat-jar) that is submitted to Flink's JobManager. -See *WordCount Storm* within `flink-storm-examples/pom.xml` for an example how to package a jar correctly. - -If you want to avoid large uber-jars, you can manually copy `storm-core-0.9.4.jar`, `json-simple-1.1.jar` and `flink-storm-{{site.version}}.jar` into Flink's `lib/` folder of each cluster node (*before* the cluster is started). -For this case, it is sufficient to include only your own Spout and Bolt classes (and their internal dependencies) into the program jar. - -# Embed Storm Operators in Flink Streaming Programs - -Spouts and Bolts can be embedded into regular streaming programs. -The Storm compatibility layer offers a wrapper classes for each, namely `SpoutWrapper` and `BoltWrapper` (`org.apache.flink.storm.wrappers`). - -Per default, both wrappers convert Storm output tuples to Flink's [Tuple]({{site.baseurl}}/dev/api_concepts.html#tuples-and-case-classes) types (ie, `Tuple0` to `Tuple25` according to the number of fields of the Storm tuples). -For single field output tuples a conversion to the field's data type is also possible (eg, `String` instead of `Tuple1`). - -Because Flink cannot infer the output field types of Storm operators, it is required to specify the output type manually. -In order to get the correct `TypeInformation` object, Flink's `TypeExtractor` can be used. - -## Embed Spouts - -In order to use a Spout as Flink source, use `StreamExecutionEnvironment.addSource(SourceFunction, TypeInformation)`. -The Spout object is handed to the constructor of `SpoutWrapper` that serves as first argument to `addSource(...)`. -The generic type declaration `OUT` specifies the type of the source output stream. - -

-
-{% highlight java %} -StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - -// stream has `raw` type (single field output streams only) -DataStream rawInput = env.addSource( - new SpoutWrapper(new FileSpout(localFilePath), new String[] { Utils.DEFAULT_STREAM_ID }), // emit default output stream as raw type - TypeExtractor.getForClass(String.class)); // output type - -// process data stream -[...] -{% endhighlight %} -
-
- -If a Spout emits a finite number of tuples, `SpoutWrapper` can be configures to terminate automatically by setting `numberOfInvocations` parameter in its constructor. -This allows the Flink program to shut down automatically after all data is processed. -Per default the program will run until it is [canceled]({{site.baseurl}}/ops/cli.html) manually. - - -## Embed Bolts - -In order to use a Bolt as Flink operator, use `DataStream.transform(String, TypeInformation, OneInputStreamOperator)`. -The Bolt object is handed to the constructor of `BoltWrapper` that serves as last argument to `transform(...)`. -The generic type declarations `IN` and `OUT` specify the type of the operator's input and output stream, respectively. - -
-
-{% highlight java %} -StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); -DataStream text = env.readTextFile(localFilePath); - -DataStream> counts = text.transform( - "tokenizer", // operator name - TypeExtractor.getForObject(new Tuple2("", 0)), // output type - new BoltWrapper>(new BoltTokenizer())); // Bolt operator - -// do further processing -[...] -{% endhighlight %} -
-
- -### Named Attribute Access for Embedded Bolts - -Bolts can accesses input tuple fields via name (additionally to access via index). -To use this feature with embedded Bolts, you need to have either a - - 1. [POJO]({{site.baseurl}}/dev/api_concepts.html#pojos) type input stream or - 2. [Tuple]({{site.baseurl}}/dev/api_concepts.html#tuples-and-case-classes) type input stream and specify the input schema (i.e. name-to-index-mapping) - -For POJO input types, Flink accesses the fields via reflection. -For this case, Flink expects either a corresponding public member variable or public getter method. -For example, if a Bolt accesses a field via name `sentence` (eg, `String s = input.getStringByField("sentence");`), the input POJO class must have a member variable `public String sentence;` or method `public String getSentence() { ... };` (pay attention to camel-case naming). - -For `Tuple` input types, it is required to specify the input schema using Storm's `Fields` class. -For this case, the constructor of `BoltWrapper` takes an additional argument: `new BoltWrapper, ...>(..., new Fields("sentence"))`. -The input type is `Tuple1` and `Fields("sentence")` specify that `input.getStringByField("sentence")` is equivalent to `input.getString(0)`. - -See [BoltTokenizerWordCountPojo](https://github.com/apache/flink/tree/master/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojo.java) and [BoltTokenizerWordCountWithNames](https://github.com/apache/flink/tree/master/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNames.java) for examples. - -## Configuring Spouts and Bolts - -In Storm, Spouts and Bolts can be configured with a globally distributed `Map` object that is given to `submitTopology(...)` method of `LocalCluster` or `StormSubmitter`. -This `Map` is provided by the user next to the topology and gets forwarded as a parameter to the calls `Spout.open(...)` and `Bolt.prepare(...)`. - -To replicate this functionality Flink's configuration mechanism must be used. -A global configuration can be set in a `StreamExecutionEnvironment` via `.getConfig().setGlobalJobParameters(...)`. -Flink's regular `Configuration` class can be used to configure Spouts and Bolts. -However, `Configuration` does not support arbitrary key data types as Storm does (only `String` keys are allowed). -Thus, Flink additionally provides `StormConfig` class that can be used like a raw `Map` to provide full compatibility to Storm. - -
-
-{% highlight java %} -StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - -StormConfig config = new StormConfig(); -// set config values -[...] - -// set global Storm configuration -env.getConfig().setGlobalJobParameters(config); - -// assemble program with embedded Spouts and/or Bolts -[...] -{% endhighlight %} -
-
- -## Multiple Output Streams - -Flink can also handle the declaration of multiple output streams for Spouts and Bolts. - -The output stream will be of data type `SplitStreamType` and must be split by using `DataStream.split(...)` and `SplitStream.select(...)`. -Flink provides the predefined output selector `StormStreamSelector` for `.split(...)` already. -Furthermore, the wrapper type `SplitStreamTuple` can be removed using `SplitStreamMapper`. - -
-
-{% highlight java %} -[...] - -// get DataStream from Spout or Bolt which declares two output streams s1 and s2 with output type SomeType -DataStream> multiStream = ... - -SplitStream> splitStream = multiStream.split(new StormStreamSelector()); - -// remove SplitStreamType using SplitStreamMapper to get data stream of type SomeType -DataStream s1 = splitStream.select("s1").map(new SplitStreamMapper()).returns(SomeType.class); -DataStream s2 = splitStream.select("s2").map(new SplitStreamMapper()).returns(SomeType.class); - -// do further processing on s1 and s2 -[...] -{% endhighlight %} -
-
- -See [SpoutSplitExample.java](https://github.com/apache/flink/tree/master/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/SpoutSplitExample.java) for a full example. - -# Flink Extensions - -## Finite Spouts - -In Flink, streaming sources can be finite, ie, emit a finite number of records and stop after emitting the last record. However, Spouts usually emit infinite streams. -The bridge between the two approaches is the `FiniteSpout` interface which, in addition to `IRichSpout`, contains a `reachedEnd()` method, where the user can specify a stopping-condition. -The user can create a finite Spout by implementing this interface instead of (or additionally to) `IRichSpout`, and implementing the `reachedEnd()` method in addition. -In contrast to a `SpoutWrapper` that is configured to emit a finite number of tuples, `FiniteSpout` interface allows to implement more complex termination criteria. - -Although finite Spouts are not necessary to embed Spouts into a Flink streaming program or to submit a whole Storm topology to Flink, there are cases where they may come in handy: - - * to achieve that a native Spout behaves the same way as a finite Flink source with minimal modifications - * the user wants to process a stream only for some time; after that, the Spout can stop automatically - * reading a file into a stream - * for testing purposes - -An example of a finite Spout that emits records for 10 seconds only: - -
-
-{% highlight java %} -public class TimedFiniteSpout extends BaseRichSpout implements FiniteSpout { - [...] // implement open(), nextTuple(), ... - - private long starttime = System.currentTimeMillis(); - - public boolean reachedEnd() { - return System.currentTimeMillis() - starttime > 10000l; - } -} -{% endhighlight %} -
-
- -# Storm Compatibility Examples - -You can find more examples in Maven module `flink-storm-examples`. -For the different versions of WordCount, see [README.md](https://github.com/apache/flink/tree/master/flink-contrib/flink-storm-examples/README.md). -To run the examples, you need to assemble a correct jar file. -`flink-storm-examples-{{ site.version }}.jar` is **no** valid jar file for job execution (it is only a standard maven artifact). - -There are example jars for embedded Spout and Bolt, namely `WordCount-SpoutSource.jar` and `WordCount-BoltTokenizer.jar`, respectively. -Compare `pom.xml` to see how both jars are built. - -You can run each of those examples via `bin/flink run .jar`. The correct entry point class is contained in each jar's manifest file. - -{% top %} diff --git a/docs/redirects/storm_compat.md b/docs/redirects/storm_compat.md deleted file mode 100644 index f2df117402bfee..00000000000000 --- a/docs/redirects/storm_compat.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: "Storm Compatibility" -layout: redirect -redirect: /dev/libs/storm_compatibility.html -permalink: /apis/streaming/storm_compatibility.html ---- - diff --git a/flink-contrib/flink-storm-examples/README.md b/flink-contrib/flink-storm-examples/README.md deleted file mode 100644 index 00d7906f3b62bb..00000000000000 --- a/flink-contrib/flink-storm-examples/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# flink-storm-examples - -This module contains multiple versions of a simple Word-Count example to illustrate the usage of the compatibility layer: -* the usage of spouts and bolts within a regular Flink streaming program (ie, embedded mode) - 1. `SpoutSourceWordCount` uses a spout as data source within a Flink streaming program - 2. `BoltTokenizeerWordCount` uses a bolt to split sentences into words within a Flink streaming program - * `BoltTokenizeerWordCountWithNames` used `Tuple` input type and accesses attributes by field names (rather than index) - * `BoltTokenizeerWordCountPOJO` used POJO input type and accesses attributes by field names (rather than index) - -* how to submit a whole Storm topology to Flink - 3. `WordCountTopology` plugs a Storm topology together - * `StormWordCountLocal` submits the topology to a local Flink cluster (similar to a `LocalCluster` in Storm) - (`WordCountLocalByName` accesses attributes by field names rather than index) - * `WordCountRemoteByClient` submits the topology to a remote Flink cluster (similar to the usage of `NimbusClient` in Storm) - * `WordCountRemoteBySubmitter` submits the topology to a remote Flink cluster (similar to the usage of `StormSubmitter` in Storm) - -Additionally, this module package the three example Word-Count programs as jar files to be submitted to a Flink cluster via `bin/flink run example.jar`. -(Valid jars are `WordCount-SpoutSource.jar`, `WordCount-BoltTokenizer.jar`, and `WordCount-StormTopology.jar`) - -The package `org.apache.flink.storm.wordcount.operators` contains original spouts and bolts that can be used unmodified within Storm or Flink. diff --git a/flink-contrib/flink-storm-examples/pom.xml b/flink-contrib/flink-storm-examples/pom.xml deleted file mode 100644 index 6c8bcab7061cec..00000000000000 --- a/flink-contrib/flink-storm-examples/pom.xml +++ /dev/null @@ -1,318 +0,0 @@ - - - - - 4.0.0 - - - org.apache.flink - flink-contrib - 1.8-SNAPSHOT - .. - - - flink-storm-examples_${scala.binary.version} - flink-storm-examples - - jar - - - - - - clojars - https://clojars.org/repo/ - - true - - - false - - - - - - - - - - - org.apache.flink - flink-storm_${scala.binary.version} - ${project.version} - - - - org.apache.flink - flink-streaming-java_${scala.binary.version} - ${project.version} - - - - org.apache.storm - storm-starter - 1.0.0 - - - - - org.apache.storm - storm-core - - - org.apache.curator - curator-framework - - - - - - - - org.apache.flink - flink-shaded-guava - - - - org.apache.flink - flink-test-utils_${scala.binary.version} - ${project.version} - test - - - - org.apache.flink - flink-streaming-java_${scala.binary.version} - ${project.version} - test - test-jar - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 2.9 - - - unpack - prepare-package - - unpack - - - - - org.apache.flink - flink-storm_${scala.binary.version} - ${project.version} - jar - false - ${project.build.directory}/classes - - - org.apache.storm - storm-core - 1.0.0 - jar - false - ${project.build.directory}/classes - - defaults.yaml - - - com.googlecode.json-simple - json-simple - 1.1 - jar - false - ${project.build.directory}/classes - - - org.yaml - snakeyaml - 1.11 - jar - false - ${project.build.directory}/classes - - - - - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - - - - WordCount-SpoutSource - package - - jar - - - WordCount - SpoutSource - - - - org.apache.flink.storm.wordcount.SpoutSourceWordCount - - - - - - org/apache/storm/topology/*.class - org/apache/storm/spout/*.class - org/apache/storm/task/*.class - org/apache/storm/tuple/*.class - org/apache/storm/generated/*.class - org/apache/storm/metric/**/*.class - org/apache/storm/thrift/**/*.class - - org/json/simple/**/*.class - org/apache/storm/shade/**/*.class - - org/apache/flink/storm/api/*.class - org/apache/flink/storm/util/*.class - org/apache/flink/storm/wrappers/*.class - - org/apache/flink/storm/wordcount/SpoutSourceWordCount.class - org/apache/flink/storm/wordcount/SpoutSourceWordCount$*.class - org/apache/flink/storm/wordcount/operators/WordCountFileSpout.class - org/apache/flink/storm/wordcount/operators/WordCountInMemorySpout.class - - org/apache/flink/storm/util/AbstractLineSpout.class - org/apache/flink/storm/util/FileSpout.class - org/apache/flink/storm/util/InMemorySpout.class - org/apache/flink/storm/wordcount/util/WordCountData.class - - - - - - - - WordCount-BoltTokenizer - package - - jar - - - WordCount - BoltTokenizer - - - - org.apache.flink.storm.wordcount.BoltTokenizerWordCount - - - - - - - org/apache/storm/topology/*.class - org/apache/storm/spout/*.class - org/apache/storm/task/*.class - org/apache/storm/tuple/*.class - org/apache/storm/generated/*.class - org/apache/storm/metric/**/*.class - org/apache/storm/thrift/**/*.class - - org/json/simple/**/*.class - org/apache/storm/shade/**/*.class - - org/apache/flink/storm/api/*.class - org/apache/flink/storm/util/*.class - org/apache/flink/storm/wrappers/*.class - - org/apache/flink/storm/wordcount/BoltTokenizerWordCount.class - org/apache/flink/storm/wordcount/operators/BoltTokenizer.class - org/apache/flink/storm/wordcount/util/WordCountData.class - - - - - - - test-jar - - - - - - - - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - org.apache.maven.plugins - maven-dependency-plugin - [2.9,) - - unpack - - - - - - - - - - - - - - - - diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithBolt.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithBolt.java deleted file mode 100644 index b6bb4d594804db..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithBolt.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.exclamation; - -import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.api.java.typeutils.TypeExtractor; -import org.apache.flink.storm.exclamation.operators.ExclamationBolt; -import org.apache.flink.storm.util.StormConfig; -import org.apache.flink.storm.wordcount.util.WordCountData; -import org.apache.flink.storm.wrappers.BoltWrapper; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - -import org.apache.storm.utils.Utils; - -/** - * Implements the "Exclamation" program that attaches 3+x exclamation marks to every line of a text files in a streaming - * fashion. The program is constructed as a regular {@link org.apache.storm.generated.StormTopology}. - * - *

The input is a plain text file with lines separated by newline characters. - * - *

Usage: - * ExclamationWithBolt <text path> <result path> <number of exclamation marks>
- * If no parameters are provided, the program is run with default data from {@link WordCountData} with x=2. - * - *

This example shows how to: - *

    - *
  • use a Bolt within a Flink Streaming program
    • - *
  • how to configure a Bolt using StormConfig
    • - *
- */ -public class ExclamationWithBolt { - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws Exception { - - if (!parseParameters(args)) { - return; - } - - // set up the execution environment - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - - // set Storm configuration - StormConfig config = new StormConfig(); - config.put(ExclamationBolt.EXCLAMATION_COUNT, new Integer(exclamationNum)); - env.getConfig().setGlobalJobParameters(config); - - // get input data - final DataStream text = getTextDataStream(env); - - final DataStream exclaimed = text - .transform("StormBoltTokenizer", - TypeExtractor.getForObject(""), - new BoltWrapper(new ExclamationBolt(), - new String[] { Utils.DEFAULT_STREAM_ID })) - .map(new ExclamationMap()); - - // emit result - if (fileOutput) { - exclaimed.writeAsText(outputPath); - } else { - exclaimed.print(); - } - - // execute program - env.execute("Streaming WordCount with bolt tokenizer"); - } - - // ************************************************************************* - // USER FUNCTIONS - // ************************************************************************* - - private static class ExclamationMap implements MapFunction { - private static final long serialVersionUID = 4614754344067170619L; - - @Override - public String map(String value) throws Exception { - return value + "!!!"; - } - } - - // ************************************************************************* - // UTIL METHODS - // ************************************************************************* - - private static boolean fileOutput = false; - private static String textPath; - private static String outputPath; - private static int exclamationNum = 2; - - private static boolean parseParameters(final String[] args) { - - if (args.length > 0) { - // parse input arguments - fileOutput = true; - if (args.length == 3) { - textPath = args[0]; - outputPath = args[1]; - exclamationNum = Integer.parseInt(args[2]); - } else { - System.err.println("Usage: ExclamationWithBolt "); - return false; - } - } else { - System.out.println("Executing ExclamationWithBolt example with built-in default data"); - System.out.println(" Provide parameters to read input data from a file"); - System.out.println(" Usage: ExclamationWithBolt "); - } - return true; - } - - private static DataStream getTextDataStream(final StreamExecutionEnvironment env) { - if (fileOutput) { - // read the text file from given input path - return env.readTextFile(textPath); - } - - return env.fromElements(WordCountData.WORDS); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithSpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithSpout.java deleted file mode 100644 index c86ab69b317e46..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithSpout.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.exclamation; - -import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.api.java.typeutils.TypeExtractor; -import org.apache.flink.storm.util.FiniteFileSpout; -import org.apache.flink.storm.util.FiniteInMemorySpout; -import org.apache.flink.storm.util.StormConfig; -import org.apache.flink.storm.wordcount.util.WordCountData; -import org.apache.flink.storm.wrappers.SpoutWrapper; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - -import org.apache.storm.utils.Utils; - -/** - * Implements the "Exclamation" program that attaches six exclamation marks to every line of a text files in a streaming - * fashion. The program is constructed as a regular {@link org.apache.storm.generated.StormTopology}. - * - *

The input is a plain text file with lines separated by newline characters. - * - *

Usage: ExclamationWithSpout <text path> <result path>
- * If no parameters are provided, the program is run with default data from {@link WordCountData}. - * - *

This example shows how to: - *

    - *
  • use a Storm spout within a Flink Streaming program
    • - *
  • make use of the FiniteSpout interface
    • - *
  • make use of the FiniteSpout interface
    • - *
  • how to configure a Spout using StormConfig
    • - *
- */ -public class ExclamationWithSpout { - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws Exception { - - if (!parseParameters(args)) { - return; - } - - // set up the execution environment - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - - // get input data - final DataStream text = getTextDataStream(env); - - final DataStream exclaimed = text - .map(new ExclamationMap()) - .map(new ExclamationMap()); - - // emit result - if (fileOutput) { - exclaimed.writeAsText(outputPath); - } else { - exclaimed.print(); - } - - // execute program - env.execute("Streaming Exclamation with Storm spout source"); - } - - // ************************************************************************* - // USER FUNCTIONS - // ************************************************************************* - - private static class ExclamationMap implements MapFunction { - private static final long serialVersionUID = -684993133807698042L; - - @Override - public String map(String value) throws Exception { - return value + "!!!"; - } - } - - // ************************************************************************* - // UTIL METHODS - // ************************************************************************* - - private static boolean fileOutput = false; - private static String textPath; - private static String outputPath; - - private static boolean parseParameters(final String[] args) { - - if (args.length > 0) { - // parse input arguments - fileOutput = true; - if (args.length == 2) { - textPath = args[0]; - outputPath = args[1]; - } else { - System.err.println("Usage: ExclamationWithSpout "); - return false; - } - } else { - System.out.println("Executing ExclamationWithSpout example with built-in default data"); - System.out.println(" Provide parameters to read input data from a file"); - System.out.println(" Usage: ExclamationWithSpout "); - } - return true; - } - - private static DataStream getTextDataStream(final StreamExecutionEnvironment env) { - if (fileOutput) { - final String[] tokens = textPath.split(":"); - final String inputFile = tokens[tokens.length - 1]; - - // set Storm configuration - StormConfig config = new StormConfig(); - config.put(FiniteFileSpout.INPUT_FILE_PATH, inputFile); - env.getConfig().setGlobalJobParameters(config); - - return env.addSource( - new SpoutWrapper(new FiniteFileSpout(), - new String[] { Utils.DEFAULT_STREAM_ID }), - TypeExtractor.getForClass(String.class)).setParallelism(1); - } - - return env.addSource( - new SpoutWrapper(new FiniteInMemorySpout( - WordCountData.WORDS), new String[] { Utils.DEFAULT_STREAM_ID }), - TypeExtractor.getForClass(String.class)).setParallelism(1); - - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/operators/ExclamationBolt.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/operators/ExclamationBolt.java deleted file mode 100644 index 8872acdd9cb699..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/operators/ExclamationBolt.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.exclamation.operators; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -import java.util.Map; - -/** - * A Bolt implementation that appends exclamation marks to incoming tuples. The number of added exclamation marks can - * be controlled by setting exclamation.count. - */ -public class ExclamationBolt implements IRichBolt { - private static final long serialVersionUID = -6364882114201311380L; - - public static final String EXCLAMATION_COUNT = "exclamation.count"; - - private OutputCollector collector; - private String exclamation; - - @SuppressWarnings("rawtypes") - @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { - this.collector = collector; - - Object count = conf.get(EXCLAMATION_COUNT); - if (count != null) { - int exclamationNum = (Integer) count; - StringBuilder builder = new StringBuilder(); - for (int index = 0; index < exclamationNum; ++index) { - builder.append('!'); - } - this.exclamation = builder.toString(); - } else { - this.exclamation = "!"; - } - } - - @Override - public void cleanup() { - } - - @Override - public void execute(Tuple tuple) { - collector.emit(tuple, new Values(tuple.getString(0) + this.exclamation)); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("word")); - } - - @Override - public Map getComponentConfiguration() { - return null; - } -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/SpoutSplitExample.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/SpoutSplitExample.java deleted file mode 100644 index c5bb5c398c93e1..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/SpoutSplitExample.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.split; - -import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.TypeExtractor; -import org.apache.flink.storm.split.operators.RandomSpout; -import org.apache.flink.storm.split.operators.VerifyAndEnrichBolt; -import org.apache.flink.storm.util.SplitStreamMapper; -import org.apache.flink.storm.util.SplitStreamType; -import org.apache.flink.storm.util.StormStreamSelector; -import org.apache.flink.storm.wrappers.BoltWrapper; -import org.apache.flink.storm.wrappers.SpoutWrapper; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.SplitStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - -/** - * Implements a simple example with two declared output streams for the embedded spout. - * - *

This example shows how to: - *

    - *
  • handle multiple output stream of a spout
    • - *
  • accessing each stream by .split(...) and .select(...)
    • - *
  • strip wrapper data type SplitStreamType for further processing in Flink
    • - *
- * - *

This example would work the same way for multiple bolt output streams. - */ -public class SpoutSplitExample { - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws Exception { - - boolean useFile = SpoutSplitExample.parseParameters(args); - - // set up the execution environment - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - - String[] rawOutputs = new String[] { RandomSpout.EVEN_STREAM, RandomSpout.ODD_STREAM }; - - final DataStream> numbers = env.addSource( - new SpoutWrapper>(new RandomSpout(true, seed), rawOutputs, - 1000), TypeExtractor.getForObject(new SplitStreamType())); - - SplitStream> splitStream = numbers - .split(new StormStreamSelector()); - - DataStream> evenStream = splitStream.select(RandomSpout.EVEN_STREAM); - DataStream> oddStream = splitStream.select(RandomSpout.ODD_STREAM); - - DataStream> evenResult = evenStream - .map(new SplitStreamMapper()).returns(Integer.class).map(new Enrich(true)); - DataStream> oddResult = oddStream.map( - new SplitStreamMapper()).transform("oddBolt", - TypeExtractor.getForObject(new Tuple2("", 0)), - new BoltWrapper>(new VerifyAndEnrichBolt(false))); - - if (useFile) { - evenResult.writeAsText(outputPath + "/even"); - oddResult.writeAsText(outputPath + "/odd"); - } else { - evenResult.print(); - oddResult.print(); - } - - // execute program - env.execute("Spout split stream example"); - } - - // ************************************************************************* - // USER FUNCTIONS - // ************************************************************************* - - /** - * Same as {@link VerifyAndEnrichBolt}. - */ - public static final class Enrich implements MapFunction> { - private static final long serialVersionUID = 5213888269197438892L; - private final Tuple2 out; - private final boolean isEven; - - public static boolean errorOccured = false; - - public Enrich(boolean isEven) { - this.isEven = isEven; - if (isEven) { - this.out = new Tuple2("even", 0); - } else { - this.out = new Tuple2("odd", 0); - } - } - - @Override - public Tuple2 map(Integer value) throws Exception { - if ((value.intValue() % 2 == 0) != this.isEven) { - errorOccured = true; - } - this.out.setField(value, 1); - return this.out; - } - } - - // ************************************************************************* - // UTIL METHODS - // ************************************************************************* - - private static long seed = System.currentTimeMillis(); - private static String outputPath = null; - - static boolean parseParameters(final String[] args) { - - if (args.length > 0) { - // parse input arguments - if (args.length == 2) { - seed = Long.parseLong(args[0]); - outputPath = args[1]; - return true; - } else { - throw new IllegalArgumentException( - "Usage: SplitStreamBoltLocal "); - } - } else { - System.out.println("Executing SplitBoltTopology example with random data"); - System.out.println(" Usage: SplitStreamBoltLocal "); - } - - return false; - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/RandomSpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/RandomSpout.java deleted file mode 100644 index afec47f0fe187e..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/RandomSpout.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.split.operators; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichSpout; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Values; - -import java.util.Map; -import java.util.Random; - -/** - * A Spout implementation that emits random numbers, optionally splitting them into odd/even streams. - */ -public class RandomSpout extends BaseRichSpout { - private static final long serialVersionUID = -3978554318742509334L; - - public static final String EVEN_STREAM = "even"; - public static final String ODD_STREAM = "odd"; - - private final boolean split; - private Random r = new Random(); - private SpoutOutputCollector collector; - - public RandomSpout(boolean split, long seed) { - this.split = split; - this.r = new Random(seed); - } - - @SuppressWarnings("rawtypes") - @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { - this.collector = collector; - } - - @Override - public void nextTuple() { - int i = r.nextInt(); - if (split) { - if (i % 2 == 0) { - this.collector.emit(EVEN_STREAM, new Values(i)); - } else { - this.collector.emit(ODD_STREAM, new Values(i)); - } - } else { - this.collector.emit(new Values(i)); - } - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - Fields schema = new Fields("number"); - if (split) { - declarer.declareStream(EVEN_STREAM, schema); - declarer.declareStream(ODD_STREAM, schema); - } else { - declarer.declare(schema); - } - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/VerifyAndEnrichBolt.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/VerifyAndEnrichBolt.java deleted file mode 100644 index a39ec9c34f515c..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/VerifyAndEnrichBolt.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.split.operators; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichBolt; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -import java.util.Map; - -/** - * Verifies that incoming numbers are either even or odd, controlled by the constructor argument. Emitted tuples are - * enriched with a new string field containing either "even" or "odd", based on the number's parity. - */ -public class VerifyAndEnrichBolt extends BaseRichBolt { - private static final long serialVersionUID = -7277395570966328721L; - - private final boolean evenOrOdd; // true: even -- false: odd - private final String token; - private OutputCollector collector; - - public static boolean errorOccured = false; - - public VerifyAndEnrichBolt(boolean evenOrOdd) { - this.evenOrOdd = evenOrOdd; - this.token = evenOrOdd ? "even" : "odd"; - } - - @SuppressWarnings("rawtypes") - @Override - public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { - this.collector = collector; - } - - @Override - public void execute(Tuple input) { - if ((input.getInteger(0) % 2 == 0) != this.evenOrOdd) { - errorOccured = true; - } - this.collector.emit(new Values(this.token, input.getInteger(0))); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("evenOrOdd", "number")); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractBoltSink.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractBoltSink.java deleted file mode 100644 index 5ae8cfbe205d0a..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractBoltSink.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Tuple; - -import java.util.Map; - -/** - * Implements a sink that write the received data so some external output. The result is formatted like - * {@code (a1, a2, ..., an)} with {@code Object.toString()} for each attribute). - */ -public abstract class AbstractBoltSink implements IRichBolt { - private static final long serialVersionUID = -1626323806848080430L; - - private StringBuilder lineBuilder; - private String prefix = ""; - private final OutputFormatter formatter; - - public AbstractBoltSink(final OutputFormatter formatter) { - this.formatter = formatter; - } - - @SuppressWarnings("rawtypes") - @Override - public final void prepare(final Map stormConf, final TopologyContext context, - final OutputCollector collector) { - this.prepareSimple(stormConf, context); - if (context.getComponentCommon(context.getThisComponentId()).get_parallelism_hint() > 1) { - this.prefix = context.getThisTaskId() + "> "; - } - } - - protected abstract void prepareSimple(Map stormConf, TopologyContext context); - - @Override - public final void execute(final Tuple input) { - this.lineBuilder = new StringBuilder(); - this.lineBuilder.append(this.prefix); - this.lineBuilder.append(this.formatter.format(input)); - this.writeExternal(this.lineBuilder.toString()); - } - - protected abstract void writeExternal(String line); - - @Override - public void cleanup() {/* nothing to do */} - - @Override - public final void declareOutputFields(final OutputFieldsDeclarer declarer) {/* nothing to do */} - - @Override - public Map getComponentConfiguration() { - return null; - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractLineSpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractLineSpout.java deleted file mode 100644 index caefd567c703fe..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractLineSpout.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; - -import java.util.Map; - -/** - * Base class for Spouts that read data line by line from an arbitrary source. The declared output schema has a single - * attribute called {@code line} and should be of type {@link String}. - */ -public abstract class AbstractLineSpout implements IRichSpout { - private static final long serialVersionUID = 8876828403487806771L; - - public static final String ATTRIBUTE_LINE = "line"; - - protected SpoutOutputCollector collector; - - @SuppressWarnings("rawtypes") - @Override - public void open(final Map conf, final TopologyContext context, final SpoutOutputCollector collector) { - this.collector = collector; - } - - @Override - public void close() {/* noting to do */} - - @Override - public void activate() {/* noting to do */} - - @Override - public void deactivate() {/* noting to do */} - - @Override - public void ack(final Object msgId) {/* noting to do */} - - @Override - public void fail(final Object msgId) {/* noting to do */} - - @Override - public void declareOutputFields(final OutputFieldsDeclarer declarer) { - declarer.declare(new Fields(ATTRIBUTE_LINE)); - } - - @Override - public Map getComponentConfiguration() { - return null; - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/BoltFileSink.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/BoltFileSink.java deleted file mode 100644 index cbbe1917955bb0..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/BoltFileSink.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.task.TopologyContext; - -import java.io.BufferedWriter; -import java.io.FileWriter; -import java.io.IOException; -import java.util.Map; - -/** - * Implements a sink that write the received data to the given file (as a result of {@code Object.toString()} for each - * attribute). - */ -public final class BoltFileSink extends AbstractBoltSink { - private static final long serialVersionUID = 2014027288631273666L; - - private final String path; - private BufferedWriter writer; - - public BoltFileSink(final String path) { - this(path, new SimpleOutputFormatter()); - } - - public BoltFileSink(final String path, final OutputFormatter formatter) { - super(formatter); - this.path = path; - } - - @SuppressWarnings("rawtypes") - @Override - public void prepareSimple(final Map stormConf, final TopologyContext context) { - try { - this.writer = new BufferedWriter(new FileWriter(this.path)); - } catch (final IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public void writeExternal(final String line) { - try { - this.writer.write(line + "\n"); - } catch (final IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public void cleanup() { - if (this.writer != null) { - try { - this.writer.close(); - } catch (final IOException e) { - throw new RuntimeException(e); - } - } - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/BoltPrintSink.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/BoltPrintSink.java deleted file mode 100644 index a80417b01ab2c9..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/BoltPrintSink.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.task.TopologyContext; - -import java.util.Map; - -/** - * Implements a sink that prints the received data to {@code stdout}. - */ -public final class BoltPrintSink extends AbstractBoltSink { - private static final long serialVersionUID = -6650011223001009519L; - - public BoltPrintSink(OutputFormatter formatter) { - super(formatter); - } - - @SuppressWarnings("rawtypes") - @Override - public void prepareSimple(final Map stormConf, final TopologyContext context) { - /* nothing to do */ - } - - @Override - public void writeExternal(final String line) { - System.out.println(line); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FileSpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FileSpout.java deleted file mode 100644 index 0533b0908ed622..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FileSpout.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.tuple.Values; - -import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.util.Map; - -/** - * Implements a Spout that reads data from a given local file. - */ -public class FileSpout extends AbstractLineSpout { - private static final long serialVersionUID = -6996907090003590436L; - - public static final String INPUT_FILE_PATH = "input.path"; - - protected String path = null; - protected BufferedReader reader; - - public FileSpout() {} - - public FileSpout(final String path) { - this.path = path; - } - - @SuppressWarnings("rawtypes") - @Override - public void open(final Map conf, final TopologyContext context, final SpoutOutputCollector collector) { - super.open(conf, context, collector); - - Object configuredPath = conf.get(INPUT_FILE_PATH); - if (configuredPath != null) { - this.path = (String) configuredPath; - } - - try { - this.reader = new BufferedReader(new FileReader(this.path)); - } catch (final FileNotFoundException e) { - throw new RuntimeException(e); - } - } - - @Override - public void close() { - if (this.reader != null) { - try { - this.reader.close(); - } catch (final IOException e) { - throw new RuntimeException(e); - } - } - } - - @Override - public void nextTuple() { - String line; - try { - line = this.reader.readLine(); - if (line != null) { - this.collector.emit(new Values(line)); - } - } catch (final IOException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FiniteFileSpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FiniteFileSpout.java deleted file mode 100644 index e4f39ab13e15b1..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FiniteFileSpout.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.tuple.Values; - -import java.io.IOException; -import java.util.Map; - -/** - * Implements a Spout that reads data from a given local file. The spout stops automatically - * when it reached the end of the file. - */ -public class FiniteFileSpout extends FileSpout implements FiniteSpout { - private static final long serialVersionUID = -1472978008607215864L; - - private String line; - private boolean newLineRead; - - public FiniteFileSpout() {} - - public FiniteFileSpout(String path) { - super(path); - } - - @SuppressWarnings("rawtypes") - @Override - public void open(final Map conf, final TopologyContext context, final SpoutOutputCollector collector) { - super.open(conf, context, collector); - newLineRead = false; - } - - @Override - public void nextTuple() { - this.collector.emit(new Values(line)); - newLineRead = false; - } - - /** - * Can be called before nextTuple() any times including 0. - */ - @Override - public boolean reachedEnd() { - try { - readLine(); - } catch (IOException e) { - throw new RuntimeException("Exception occured while reading file " + path); - } - return line == null; - } - - private void readLine() throws IOException { - if (!newLineRead) { - line = reader.readLine(); - newLineRead = true; - } - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FiniteInMemorySpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FiniteInMemorySpout.java deleted file mode 100644 index ff89a414cb19a7..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FiniteInMemorySpout.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -/** - * Implements a Spout that reads String[] data stored in memory. The Spout stops automatically when it emitted all of - * the data. - */ -public class FiniteInMemorySpout extends InMemorySpout implements FiniteSpout { - private static final long serialVersionUID = -4008858647468647019L; - - public FiniteInMemorySpout(String[] source) { - super(source); - } - - @Override - public boolean reachedEnd() { - return counter >= source.length; - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/InMemorySpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/InMemorySpout.java deleted file mode 100644 index de1ca20ebf128f..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/InMemorySpout.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.tuple.Values; - -/** - * Implements a Spout that reads data stored in memory. - */ -public class InMemorySpout extends AbstractLineSpout { - private static final long serialVersionUID = -4008858647468647019L; - - protected T[] source; - protected int counter = 0; - - public InMemorySpout(T[] source) { - this.source = source; - } - - @Override - public void nextTuple() { - if (this.counter < source.length) { - this.collector.emit(new Values(source[this.counter++])); - } - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/OutputFormatter.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/OutputFormatter.java deleted file mode 100644 index a0f933f1aacb8a..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/OutputFormatter.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.tuple.Tuple; - -import java.io.Serializable; - -/** - * Interface that is used to convert Storm {@link Tuple Tuples} to a string before writing them out to a file or to the - * console. - */ -public interface OutputFormatter extends Serializable { - - /** - * Converts a Storm {@link Tuple} to a string. This method is used for formatting the output tuples before writing - * them out to a file or to the console. - * - * @param input - * The tuple to be formatted - * @return The string result of the formatting - */ - String format(Tuple input); - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/SimpleOutputFormatter.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/SimpleOutputFormatter.java deleted file mode 100644 index bf30cd21c0b6ea..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/SimpleOutputFormatter.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.tuple.Tuple; - -/** - * Simple {@link OutputFormatter} implementation to convert {@link Tuple Tuples} with a size of 1 by returning the - * result of {@link Object#toString()} for the first field. - */ -public class SimpleOutputFormatter implements OutputFormatter { - private static final long serialVersionUID = 6349573860144270338L; - - /** - * Converts a Storm {@link Tuple} with 1 field to a string by retrieving the value of that field. This method is - * used for formatting raw outputs wrapped in tuples, before writing them out to a file or to the console. - * - * @param input - * The tuple to be formatted - * @return The string result of the formatting - */ - @Override - public String format(final Tuple input) { - if (input.getValues().size() != 1) { - throw new RuntimeException("The output is not raw"); - } - return input.getValue(0).toString(); - } -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/TupleOutputFormatter.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/TupleOutputFormatter.java deleted file mode 100644 index 42189a70f200f4..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/TupleOutputFormatter.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.tuple.Tuple; - -/** - * {@link OutputFormatter} implementation that converts {@link Tuple Tuples} of arbitrary size to a string. For a given - * tuple the output is (field1,field2,...,fieldX). - */ -public class TupleOutputFormatter implements OutputFormatter { - private static final long serialVersionUID = -599665757723851761L; - - @Override - public String format(final Tuple input) { - final StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("("); - for (final Object attribute : input.getValues()) { - stringBuilder.append(attribute); - stringBuilder.append(","); - } - stringBuilder.replace(stringBuilder.length() - 1, stringBuilder.length(), ")"); - return stringBuilder.toString(); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCount.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCount.java deleted file mode 100644 index 54a608000845ce..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCount.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.TypeExtractor; -import org.apache.flink.storm.wordcount.operators.BoltTokenizer; -import org.apache.flink.storm.wordcount.util.WordCountData; -import org.apache.flink.storm.wrappers.BoltWrapper; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - -import org.apache.storm.topology.IRichBolt; - -/** - * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming - * fashion. The tokenizer step is performed by a {@link IRichBolt Bolt}. - * - *

The input is a plain text file with lines separated by newline characters. - * - *

Usage: WordCount <text path> <result path>
- * If no parameters are provided, the program is run with default data from {@link WordCountData}. - * - *

This example shows how to: - *

    - *
  • use a Bolt within a Flink Streaming program.
    • - *
- */ -public class BoltTokenizerWordCount { - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws Exception { - - if (!parseParameters(args)) { - return; - } - - // set up the execution environment - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - - // get input data - final DataStream text = getTextDataStream(env); - - final DataStream> counts = text - // split up the lines in pairs (2-tuples) containing: (word,1) - // this is done by a bolt that is wrapped accordingly - .transform("BoltTokenizer", - TypeExtractor.getForObject(new Tuple2("", 0)), - new BoltWrapper>(new BoltTokenizer())) - // group by the tuple field "0" and sum up tuple field "1" - .keyBy(0).sum(1); - - // emit result - if (fileOutput) { - counts.writeAsText(outputPath); - } else { - counts.print(); - } - - // execute program - env.execute("Streaming WordCount with bolt tokenizer"); - } - - // ************************************************************************* - // UTIL METHODS - // ************************************************************************* - - private static boolean fileOutput = false; - private static String textPath; - private static String outputPath; - - private static boolean parseParameters(final String[] args) { - - if (args.length > 0) { - // parse input arguments - fileOutput = true; - if (args.length == 2) { - textPath = args[0]; - outputPath = args[1]; - } else { - System.err.println("Usage: BoltTokenizerWordCount "); - return false; - } - } else { - System.out.println("Executing BoltTokenizerWordCount example with built-in default data"); - System.out.println(" Provide parameters to read input data from a file"); - System.out.println(" Usage: BoltTokenizerWordCount "); - } - return true; - } - - private static DataStream getTextDataStream(final StreamExecutionEnvironment env) { - if (fileOutput) { - // read the text file from given input path - return env.readTextFile(textPath); - } - - return env.fromElements(WordCountData.WORDS); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojo.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojo.java deleted file mode 100644 index 06c6a03b6587ad..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojo.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.api.java.io.CsvInputFormat; -import org.apache.flink.api.java.io.PojoCsvInputFormat; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.PojoTypeInfo; -import org.apache.flink.api.java.typeutils.TypeExtractor; -import org.apache.flink.core.fs.Path; -import org.apache.flink.storm.wordcount.operators.BoltTokenizerByName; -import org.apache.flink.storm.wordcount.operators.WordCountDataPojos; -import org.apache.flink.storm.wordcount.operators.WordCountDataPojos.Sentence; -import org.apache.flink.storm.wordcount.util.WordCountData; -import org.apache.flink.storm.wrappers.BoltWrapper; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - -import org.apache.storm.topology.IRichBolt; - -/** - * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming - * fashion. The tokenizer step is performed by a {@link IRichBolt Bolt}. In contrast to {@link BoltTokenizerWordCount} - * the tokenizer's input is a POJO type and the single field is accessed by name. - * - *

The input is a plain text file with lines separated by newline characters. - * - *

Usage: WordCount <text path> <result path>
- * If no parameters are provided, the program is run with default data from {@link WordCountData}. - * - *

This example shows how to: - *

    - *
  • how to access attributes by name within a Bolt for POJO type input streams - *
- */ -public class BoltTokenizerWordCountPojo { - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws Exception { - - if (!parseParameters(args)) { - return; - } - - // set up the execution environment - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - - // get input data - final DataStream text = getTextDataStream(env); - - final DataStream> counts = text - // split up the lines in pairs (2-tuples) containing: (word,1) - // this is done by a bolt that is wrapped accordingly - .transform("BoltTokenizerPojo", - TypeExtractor.getForObject(new Tuple2("", 0)), - new BoltWrapper>(new BoltTokenizerByName())) - // group by the tuple field "0" and sum up tuple field "1" - .keyBy(0).sum(1); - - // emit result - if (fileOutput) { - counts.writeAsText(outputPath); - } else { - counts.print(); - } - - // execute program - env.execute("Streaming WordCount with POJO bolt tokenizer"); - } - - // ************************************************************************* - // UTIL METHODS - // ************************************************************************* - - private static boolean fileOutput = false; - private static String textPath; - private static String outputPath; - - private static boolean parseParameters(final String[] args) { - - if (args.length > 0) { - // parse input arguments - fileOutput = true; - if (args.length == 2) { - textPath = args[0]; - outputPath = args[1]; - } else { - System.err.println("Usage: BoltTokenizerWordCountPojo "); - return false; - } - } else { - System.out - .println("Executing BoltTokenizerWordCountPojo example with built-in default data"); - System.out.println(" Provide parameters to read input data from a file"); - System.out.println(" Usage: BoltTokenizerWordCountPojo "); - } - return true; - } - - private static DataStream getTextDataStream(final StreamExecutionEnvironment env) { - if (fileOutput) { - // read the text file from given input path - PojoTypeInfo sourceType = (PojoTypeInfo) TypeExtractor - .getForObject(new Sentence("")); - return env.createInput(new PojoCsvInputFormat(new Path( - textPath), CsvInputFormat.DEFAULT_LINE_DELIMITER, - CsvInputFormat.DEFAULT_LINE_DELIMITER, sourceType), - sourceType); - } - - return env.fromElements(WordCountDataPojos.SENTENCES); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNames.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNames.java deleted file mode 100644 index b0bb85a8054bbb..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNames.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.api.java.io.CsvInputFormat; -import org.apache.flink.api.java.io.TupleCsvInputFormat; -import org.apache.flink.api.java.tuple.Tuple; -import org.apache.flink.api.java.tuple.Tuple1; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.api.java.typeutils.TypeExtractor; -import org.apache.flink.core.fs.Path; -import org.apache.flink.storm.wordcount.operators.BoltTokenizerByName; -import org.apache.flink.storm.wordcount.operators.WordCountDataTuple; -import org.apache.flink.storm.wordcount.util.WordCountData; -import org.apache.flink.storm.wrappers.BoltWrapper; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.tuple.Fields; - -/** - * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming - * fashion. The tokenizer step is performed by a {@link IRichBolt Bolt}. In contrast to {@link BoltTokenizerWordCount} - * the tokenizer's input is a {@link Tuple} type and the single field is accessed by name. - * - *

The input is a plain text file with lines separated by newline characters. - * - *

Usage: WordCount <text path> <result path>
- * If no parameters are provided, the program is run with default data from {@link WordCountData}. - * - *

This example shows how to: - *

    - *
  • how to access attributes by name within a Bolt for {@link Tuple} type input streams - *
- */ -public class BoltTokenizerWordCountWithNames { - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws Exception { - - if (!parseParameters(args)) { - return; - } - - // set up the execution environment - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - - // get input data - final DataStream> text = getTextDataStream(env); - - final DataStream> counts = text - // split up the lines in pairs (2-tuples) containing: (word,1) - // this is done by a Storm bolt that is wrapped accordingly - .transform( - "BoltTokenizerWithNames", - TypeExtractor.getForObject(new Tuple2("", 0)), - new BoltWrapper, Tuple2>( - new BoltTokenizerByName(), new Fields("sentence"))) - // group by the tuple field "0" and sum up tuple field "1" - .keyBy(0).sum(1); - - // emit result - if (fileOutput) { - counts.writeAsText(outputPath); - } else { - counts.print(); - } - - // execute program - env.execute("Streaming WordCount with schema bolt tokenizer"); - } - - // ************************************************************************* - // UTIL METHODS - // ************************************************************************* - - private static boolean fileOutput = false; - private static String textPath; - private static String outputPath; - - private static boolean parseParameters(final String[] args) { - - if (args.length > 0) { - // parse input arguments - fileOutput = true; - if (args.length == 2) { - textPath = args[0]; - outputPath = args[1]; - } else { - System.err.println("Usage: BoltTokenizerWordCountWithNames "); - return false; - } - } else { - System.out.println("Executing BoltTokenizerWordCountWithNames example with built-in default data"); - System.out.println(" Provide parameters to read input data from a file"); - System.out.println(" Usage: BoltTokenizerWordCountWithNames "); - } - return true; - } - - private static DataStream> getTextDataStream(final StreamExecutionEnvironment env) { - if (fileOutput) { - // read the text file from given input path - TupleTypeInfo> sourceType = (TupleTypeInfo>) TypeExtractor - .getForObject(new Tuple1("")); - return env.createInput(new TupleCsvInputFormat>(new Path( - textPath), CsvInputFormat.DEFAULT_LINE_DELIMITER, - CsvInputFormat.DEFAULT_LINE_DELIMITER, sourceType), - sourceType); - } - - return env.fromElements(WordCountDataTuple.TUPLES); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/SpoutSourceWordCount.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/SpoutSourceWordCount.java deleted file mode 100644 index 7109c4ab983275..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/SpoutSourceWordCount.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.api.common.functions.FlatMapFunction; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.TypeExtractor; -import org.apache.flink.storm.wordcount.operators.WordCountFileSpout; -import org.apache.flink.storm.wordcount.operators.WordCountInMemorySpout; -import org.apache.flink.storm.wordcount.util.WordCountData; -import org.apache.flink.storm.wrappers.SpoutWrapper; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.util.Collector; - -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.utils.Utils; - -/** - * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming - * fashion. The used data source is a {@link IRichSpout Spout}. - * - *

The input is a plain text file with lines separated by newline characters. - * - *

Usage: WordCount <text path> <result path>
- * If no parameters are provided, the program is run with default data from {@link WordCountData}. - * - *

This example shows how to: - *

    - *
  • use a Spout within a Flink Streaming program.
    • - *
- */ -public class SpoutSourceWordCount { - - // ************************************************************************* - // PROGRAM - // ************************************************************************* - - public static void main(final String[] args) throws Exception { - - if (!parseParameters(args)) { - return; - } - - // set up the execution environment - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - - // get input data - final DataStream text = getTextDataStream(env); - - final DataStream> counts = - // split up the lines in pairs (2-tuples) containing: (word,1) - text.flatMap(new Tokenizer()) - // group by the tuple field "0" and sum up tuple field "1" - .keyBy(0).sum(1); - - // emit result - if (fileOutput) { - counts.writeAsText(outputPath); - } else { - counts.print(); - } - - // execute program - env.execute("Streaming WordCount with spout source"); - } - - // ************************************************************************* - // USER FUNCTIONS - // ************************************************************************* - - /** - * Implements the string tokenizer that splits sentences into words as a user-defined FlatMapFunction. The function - * takes a line (String) and splits it into multiple pairs in the form of "(word,1)" ({@code Tuple2}). - */ - public static final class Tokenizer implements FlatMapFunction> { - private static final long serialVersionUID = 1L; - - @Override - public void flatMap(final String value, final Collector> out) throws Exception { - // normalize and split the line - final String[] tokens = value.toLowerCase().split("\\W+"); - - // emit the pairs - for (final String token : tokens) { - if (token.length() > 0) { - out.collect(new Tuple2(token, 1)); - } - } - } - } - - // ************************************************************************* - // UTIL METHODS - // ************************************************************************* - - private static boolean fileOutput = false; - private static String textPath; - private static String outputPath; - - private static boolean parseParameters(final String[] args) { - - if (args.length > 0) { - // parse input arguments - fileOutput = true; - if (args.length == 2) { - textPath = args[0]; - outputPath = args[1]; - } else { - System.err.println("Usage: SpoutSourceWordCount "); - return false; - } - } else { - System.out.println("Executing SpoutSourceWordCount example with built-in default data"); - System.out.println(" Provide parameters to read input data from a file"); - System.out.println(" Usage: SpoutSourceWordCount "); - } - return true; - } - - private static DataStream getTextDataStream(final StreamExecutionEnvironment env) { - if (fileOutput) { - // read the text file from given input path - final String[] tokens = textPath.split(":"); - final String localFile = tokens[tokens.length - 1]; - return env.addSource( - new SpoutWrapper(new WordCountFileSpout(localFile), - new String[] { Utils.DEFAULT_STREAM_ID }, -1), - TypeExtractor.getForClass(String.class)).setParallelism(1); - } - - return env.addSource( - new SpoutWrapper(new WordCountInMemorySpout(), - new String[] { Utils.DEFAULT_STREAM_ID }, -1), - TypeExtractor.getForClass(String.class)).setParallelism(1); - - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounter.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounter.java deleted file mode 100644 index 34fc703f6be555..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounter.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount.operators; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -import java.util.HashMap; -import java.util.Map; - -/** - * Implements the word counter that counts the occurrence of each unique word. The bolt takes a pair (input tuple - * schema: {@code }) and sums the given word count for each unique word (output tuple schema: - * {@code } ). - * - *

Same as {@link BoltCounterByName}, but accesses input attribute by index (instead of name). - */ -public class BoltCounter implements IRichBolt { - private static final long serialVersionUID = 399619605462625934L; - - public static final String ATTRIBUTE_WORD = "word"; - public static final String ATTRIBUTE_COUNT = "count"; - - private final HashMap counts = new HashMap(); - private OutputCollector collector; - - @SuppressWarnings("rawtypes") - @Override - public void prepare(final Map stormConf, final TopologyContext context, final OutputCollector collector) { - this.collector = collector; - } - - @Override - public void execute(final Tuple input) { - final String word = input.getString(BoltTokenizer.ATTRIBUTE_WORD_INDEX); - - Count currentCount = this.counts.get(word); - if (currentCount == null) { - currentCount = new Count(); - this.counts.put(word, currentCount); - } - currentCount.count += input.getInteger(BoltTokenizer.ATTRIBUTE_COUNT_INDEX); - - this.collector.emit(new Values(word, currentCount.count)); - } - - @Override - public void cleanup() {/* nothing to do */} - - @Override - public void declareOutputFields(final OutputFieldsDeclarer declarer) { - declarer.declare(new Fields(ATTRIBUTE_WORD, ATTRIBUTE_COUNT)); - } - - @Override - public Map getComponentConfiguration() { - return null; - } - - /** - * A counter helper to emit immutable tuples to the given stormCollector and avoid unnecessary object - * creating/deletion. - */ - private static final class Count { - public int count; - - public Count() {/* nothing to do */} - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounterByName.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounterByName.java deleted file mode 100644 index cd53d505564377..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounterByName.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount.operators; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -import java.util.HashMap; -import java.util.Map; - -/** - * Implements the word counter that counts the occurrence of each unique word. The bolt takes a pair (input tuple - * schema: {@code }) and sums the given word count for each unique word (output tuple schema: - * {@code } ). - * - *

Same as {@link BoltCounter}, but accesses input attribute by name (instead of index). - */ -public class BoltCounterByName implements IRichBolt { - private static final long serialVersionUID = 399619605462625934L; - - public static final String ATTRIBUTE_WORD = "word"; - public static final String ATTRIBUTE_COUNT = "count"; - - private final HashMap counts = new HashMap(); - private OutputCollector collector; - - @SuppressWarnings("rawtypes") - @Override - public void prepare(final Map stormConf, final TopologyContext context, final OutputCollector collector) { - this.collector = collector; - } - - @Override - public void execute(final Tuple input) { - final String word = input.getStringByField(BoltTokenizer.ATTRIBUTE_WORD); - - Count currentCount = this.counts.get(word); - if (currentCount == null) { - currentCount = new Count(); - this.counts.put(word, currentCount); - } - currentCount.count += input.getIntegerByField(BoltTokenizer.ATTRIBUTE_COUNT); - - this.collector.emit(new Values(word, currentCount.count)); - } - - @Override - public void cleanup() {/* nothing to do */} - - @Override - public void declareOutputFields(final OutputFieldsDeclarer declarer) { - declarer.declare(new Fields(ATTRIBUTE_WORD, ATTRIBUTE_COUNT)); - } - - @Override - public Map getComponentConfiguration() { - return null; - } - - /** - * A counter helper to emit immutable tuples to the given stormCollector and avoid unnecessary object - * creating/deletion. - */ - private static final class Count { - public int count; - - public Count() {/* nothing to do */} - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizer.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizer.java deleted file mode 100644 index 41e8a8d5a5b98b..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizer.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount.operators; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -import java.util.Map; - -/** - * Implements the string tokenizer that splits sentences into words as a bolt. The bolt takes a line (input tuple - * schema: {@code }) and splits it into multiple pairs in the form of "(word,1)" (output tuple schema: - * {@code }). - * - *

Same as {@link BoltTokenizerByName}, but accesses input attribute by index (instead of name). - */ -public final class BoltTokenizer implements IRichBolt { - private static final long serialVersionUID = -8589620297208175149L; - - public static final String ATTRIBUTE_WORD = "word"; - public static final String ATTRIBUTE_COUNT = "count"; - - public static final int ATTRIBUTE_WORD_INDEX = 0; - public static final int ATTRIBUTE_COUNT_INDEX = 1; - - private OutputCollector collector; - - @SuppressWarnings("rawtypes") - @Override - public void prepare(final Map stormConf, final TopologyContext context, final OutputCollector collector) { - this.collector = collector; - } - - @Override - public void execute(final Tuple input) { - final String[] tokens = input.getString(0).toLowerCase().split("\\W+"); - - for (final String token : tokens) { - if (token.length() > 0) { - this.collector.emit(new Values(token, 1)); - } - } - } - - @Override - public void cleanup() {/* nothing to do */} - - @Override - public void declareOutputFields(final OutputFieldsDeclarer declarer) { - declarer.declare(new Fields(ATTRIBUTE_WORD, ATTRIBUTE_COUNT)); - } - - @Override - public Map getComponentConfiguration() { - return null; - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizerByName.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizerByName.java deleted file mode 100644 index dff39eba1373f4..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizerByName.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount.operators; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -import java.util.Map; - -/** - * Implements the string tokenizer that splits sentences into words as a bolt. The bolt takes a line (input tuple - * schema: {@code }) and splits it into multiple pairs in the form of "(word,1)" (output tuple schema: - * {@code }). - * - *

Same as {@link BoltTokenizer}, but accesses input attribute by name (instead of index). - */ -public final class BoltTokenizerByName implements IRichBolt { - private static final long serialVersionUID = -8589620297208175149L; - - public static final String ATTRIBUTE_WORD = "word"; - public static final String ATTRIBUTE_COUNT = "count"; - - public static final int ATTRIBUTE_WORD_INDEX = 0; - public static final int ATTRIBUTE_COUNT_INDEX = 1; - - private OutputCollector collector; - - @SuppressWarnings("rawtypes") - @Override - public void prepare(final Map stormConf, final TopologyContext context, final OutputCollector collector) { - this.collector = collector; - } - - @Override - public void execute(final Tuple input) { - final String[] tokens = input.getStringByField("sentence").toLowerCase().split("\\W+"); - - for (final String token : tokens) { - if (token.length() > 0) { - this.collector.emit(new Values(token, 1)); - } - } - } - - @Override - public void cleanup() {/* nothing to do */} - - @Override - public void declareOutputFields(final OutputFieldsDeclarer declarer) { - declarer.declare(new Fields(ATTRIBUTE_WORD, ATTRIBUTE_COUNT)); - } - - @Override - public Map getComponentConfiguration() { - return null; - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataPojos.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataPojos.java deleted file mode 100644 index 7273eca7e401c2..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataPojos.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount.operators; - -import org.apache.flink.storm.wordcount.util.WordCountData; - -import java.io.Serializable; - -/** - * Input POJOs for WordCount programs. - */ -public class WordCountDataPojos { - public static final Sentence[] SENTENCES; - - static { - SENTENCES = new Sentence[WordCountData.WORDS.length]; - for (int i = 0; i < SENTENCES.length; ++i) { - SENTENCES[i] = new Sentence(WordCountData.WORDS[i]); - } - } - - /** - * Simple POJO containing a string. - */ - public static class Sentence implements Serializable { - private static final long serialVersionUID = -7336372859203407522L; - - private String sentence; - - public Sentence() { - } - - public Sentence(String sentence) { - this.sentence = sentence; - } - - public String getSentence() { - return sentence; - } - - public void setSentence(String sentence) { - this.sentence = sentence; - } - - @Override - public String toString() { - return "(" + this.sentence + ")"; - } - } -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataTuple.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataTuple.java deleted file mode 100644 index 6adfdb4be4d470..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataTuple.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount.operators; - -import org.apache.flink.api.java.tuple.Tuple1; -import org.apache.flink.storm.wordcount.util.WordCountData; - -/** - * Input tuples for WordCount programs. - */ -@SuppressWarnings("unchecked") -public class WordCountDataTuple { - public static final Tuple1[] TUPLES; - - static { - TUPLES = new Tuple1[WordCountData.WORDS.length]; - for (int i = 0; i < TUPLES.length; ++i) { - TUPLES[i] = new Tuple1(WordCountData.WORDS[i]); - } - } - -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountFileSpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountFileSpout.java deleted file mode 100644 index 1298422ec6d45a..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountFileSpout.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount.operators; - -import org.apache.flink.storm.util.FileSpout; - -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; - -/** - * Implements a Spout that reads data from a given local file. - */ -public final class WordCountFileSpout extends FileSpout { - private static final long serialVersionUID = 2372251989250954503L; - - public WordCountFileSpout(String path) { - super(path); - } - - @Override - public void declareOutputFields(final OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("sentence")); - } -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountInMemorySpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountInMemorySpout.java deleted file mode 100644 index 6bb8ac188f8b7b..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountInMemorySpout.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount.operators; - -import org.apache.flink.storm.util.FiniteInMemorySpout; -import org.apache.flink.storm.wordcount.util.WordCountData; - -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; - -/** - * Implements a Spout that reads data from {@link WordCountData#WORDS}. - */ -public final class WordCountInMemorySpout extends FiniteInMemorySpout { - private static final long serialVersionUID = 8832143302409465843L; - - public WordCountInMemorySpout() { - super(WordCountData.WORDS); - } - - @Override - public void declareOutputFields(final OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("sentence")); - } -} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/util/WordCountData.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/util/WordCountData.java deleted file mode 100644 index 968207227478d7..00000000000000 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/util/WordCountData.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount.util; - -/** - * Provides the default data sets used for the WordCount example program. - * The default data sets are used, if no parameters are given to the program. - */ -public class WordCountData { - - public static final String[] WORDS = new String[]{ - "To be, or not to be,--that is the question:--", - "Whether 'tis nobler in the mind to suffer", - "The slings and arrows of outrageous fortune", - "Or to take arms against a sea of troubles,", - "And by opposing end them?--To die,--to sleep,--", - "No more; and by a sleep to say we end", - "The heartache, and the thousand natural shocks", - "That flesh is heir to,--'tis a consummation", - "Devoutly to be wish'd. To die,--to sleep;--", - "To sleep! perchance to dream:--ay, there's the rub;", - "For in that sleep of death what dreams may come,", - "When we have shuffled off this mortal coil,", - "Must give us pause: there's the respect", - "That makes calamity of so long life;", - "For who would bear the whips and scorns of time,", - "The oppressor's wrong, the proud man's contumely,", - "The pangs of despis'd love, the law's delay,", - "The insolence of office, and the spurns", - "That patient merit of the unworthy takes,", - "When he himself might his quietus make", - "With a bare bodkin? who would these fardels bear,", - "To grunt and sweat under a weary life,", - "But that the dread of something after death,--", - "The undiscover'd country, from whose bourn", - "No traveller returns,--puzzles the will,", - "And makes us rather bear those ills we have", - "Than fly to others that we know not of?", - "Thus conscience does make cowards of us all;", - "And thus the native hue of resolution", - "Is sicklied o'er with the pale cast of thought;", - "And enterprises of great pith and moment,", - "With this regard, their currents turn awry,", - "And lose the name of action.--Soft you now!", - "The fair Ophelia!--Nymph, in thy orisons", - "Be all my sins remember'd." - }; -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithBoltITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithBoltITCase.java deleted file mode 100644 index f0725e91baac9a..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithBoltITCase.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.exclamation; - -import org.apache.flink.storm.exclamation.util.ExclamationData; -import org.apache.flink.test.testdata.WordCountData; -import org.apache.flink.test.util.AbstractTestBase; - -import org.junit.Test; - -/** - * Test for the ExclamationWithBolt example. - */ -public class ExclamationWithBoltITCase extends AbstractTestBase { - - @Test - public void testProgram() throws Exception { - String textPath = createTempFile("text.txt", WordCountData.TEXT); - String resultPath = getTempDirPath("result"); - String exclamationNum = "3"; - - ExclamationWithBolt.main(new String[]{textPath, resultPath, exclamationNum}); - - compareResultsByLinesInMemory(ExclamationData.TEXT_WITH_EXCLAMATIONS, resultPath); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithSpoutITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithSpoutITCase.java deleted file mode 100644 index 4d16c4a8a707d7..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithSpoutITCase.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.exclamation; - -import org.apache.flink.storm.exclamation.util.ExclamationData; -import org.apache.flink.test.testdata.WordCountData; -import org.apache.flink.test.util.AbstractTestBase; - -import org.junit.Test; - -/** - * Test for the ExclamationWithSpout example. - */ -public class ExclamationWithSpoutITCase extends AbstractTestBase { - - @Test - public void testProgram() throws Exception { - String textPath = createTempFile("text.txt", WordCountData.TEXT); - String resultPath = getTempDirPath("result"); - - ExclamationWithSpout.main(new String[]{textPath, resultPath}); - - compareResultsByLinesInMemory(ExclamationData.TEXT_WITH_EXCLAMATIONS, resultPath); - } -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/util/ExclamationData.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/util/ExclamationData.java deleted file mode 100644 index f7000097921ddc..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/util/ExclamationData.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.exclamation.util; - -/** - * Expected output of Exclamation programs. - */ -public class ExclamationData { - - public static final String TEXT_WITH_EXCLAMATIONS = - "Goethe - Faust: Der Tragoedie erster Teil!!!!!!\n" - + "Prolog im Himmel.!!!!!!\n" - + "Der Herr. Die himmlischen Heerscharen. Nachher Mephistopheles. Die drei!!!!!!\n" - + "Erzengel treten vor.!!!!!!\n" - + "RAPHAEL: Die Sonne toent, nach alter Weise, In Brudersphaeren Wettgesang,!!!!!!\n" - + "Und ihre vorgeschriebne Reise Vollendet sie mit Donnergang. Ihr Anblick!!!!!!\n" - + "gibt den Engeln Staerke, Wenn keiner Sie ergruenden mag; die unbegreiflich!!!!!!\n" - + "hohen Werke Sind herrlich wie am ersten Tag.!!!!!!\n" - + "GABRIEL: Und schnell und unbegreiflich schnelle Dreht sich umher der Erde!!!!!!\n" - + "Pracht; Es wechselt Paradieseshelle Mit tiefer, schauervoller Nacht. Es!!!!!!\n" - + "schaeumt das Meer in breiten Fluessen Am tiefen Grund der Felsen auf, Und!!!!!!\n" - + "Fels und Meer wird fortgerissen Im ewig schnellem Sphaerenlauf.!!!!!!\n" - + "MICHAEL: Und Stuerme brausen um die Wette Vom Meer aufs Land, vom Land!!!!!!\n" - + "aufs Meer, und bilden wuetend eine Kette Der tiefsten Wirkung rings umher.!!!!!!\n" - + "Da flammt ein blitzendes Verheeren Dem Pfade vor des Donnerschlags. Doch!!!!!!\n" - + "deine Boten, Herr, verehren Das sanfte Wandeln deines Tags.!!!!!!\n" - + "ZU DREI: Der Anblick gibt den Engeln Staerke, Da keiner dich ergruenden!!!!!!\n" - + "mag, Und alle deine hohen Werke Sind herrlich wie am ersten Tag.!!!!!!\n" - + "MEPHISTOPHELES: Da du, o Herr, dich einmal wieder nahst Und fragst, wie!!!!!!\n" - + "alles sich bei uns befinde, Und du mich sonst gewoehnlich gerne sahst, So!!!!!!\n" - + "siehst du mich auch unter dem Gesinde. Verzeih, ich kann nicht hohe Worte!!!!!!\n" - + "machen, Und wenn mich auch der ganze Kreis verhoehnt; Mein Pathos braechte!!!!!!\n" - + "dich gewiss zum Lachen, Haettst du dir nicht das Lachen abgewoehnt. Von!!!!!!\n" - + "Sonn' und Welten weiss ich nichts zu sagen, Ich sehe nur, wie sich die!!!!!!\n" - + "Menschen plagen. Der kleine Gott der Welt bleibt stets von gleichem!!!!!!\n" - + "Schlag, Und ist so wunderlich als wie am ersten Tag. Ein wenig besser!!!!!!\n" - + "wuerd er leben, Haettst du ihm nicht den Schein des Himmelslichts gegeben;!!!!!!\n" - + "Er nennt's Vernunft und braucht's allein, Nur tierischer als jedes Tier!!!!!!\n" - + "zu sein. Er scheint mir, mit Verlaub von euer Gnaden, Wie eine der!!!!!!\n" - + "langbeinigen Zikaden, Die immer fliegt und fliegend springt Und gleich im!!!!!!\n" - + "Gras ihr altes Liedchen singt; Und laeg er nur noch immer in dem Grase! In!!!!!!\n" - + "jeden Quark begraebt er seine Nase.!!!!!!\n" - + "DER HERR: Hast du mir weiter nichts zu sagen? Kommst du nur immer!!!!!!\n" - + "anzuklagen? Ist auf der Erde ewig dir nichts recht?!!!!!!\n" - + "MEPHISTOPHELES: Nein Herr! ich find es dort, wie immer, herzlich!!!!!!\n" - + "schlecht. Die Menschen dauern mich in ihren Jammertagen, Ich mag sogar!!!!!!\n" - + "die armen selbst nicht plagen.!!!!!!\n" + "DER HERR: Kennst du den Faust?!!!!!!\n" - + "MEPHISTOPHELES: Den Doktor?!!!!!!\n" - + "DER HERR: Meinen Knecht!!!!!!!\n" - + "MEPHISTOPHELES: Fuerwahr! er dient Euch auf besondre Weise. Nicht irdisch!!!!!!\n" - + "ist des Toren Trank noch Speise. Ihn treibt die Gaerung in die Ferne, Er!!!!!!\n" - + "ist sich seiner Tollheit halb bewusst; Vom Himmel fordert er die schoensten!!!!!!\n" - + "Sterne Und von der Erde jede hoechste Lust, Und alle Naeh und alle Ferne!!!!!!\n" - + "Befriedigt nicht die tiefbewegte Brust.!!!!!!\n" - + "DER HERR: Wenn er mir auch nur verworren dient, So werd ich ihn bald in!!!!!!\n" - + "die Klarheit fuehren. Weiss doch der Gaertner, wenn das Baeumchen gruent, Das!!!!!!\n" - + "Bluet und Frucht die kuenft'gen Jahre zieren.!!!!!!\n" - + "MEPHISTOPHELES: Was wettet Ihr? den sollt Ihr noch verlieren! Wenn Ihr!!!!!!\n" - + "mir die Erlaubnis gebt, Ihn meine Strasse sacht zu fuehren.!!!!!!\n" - + "DER HERR: Solang er auf der Erde lebt, So lange sei dir's nicht verboten,!!!!!!\n" - + "Es irrt der Mensch so lang er strebt.!!!!!!\n" - + "MEPHISTOPHELES: Da dank ich Euch; denn mit den Toten Hab ich mich niemals!!!!!!\n" - + "gern befangen. Am meisten lieb ich mir die vollen, frischen Wangen. Fuer!!!!!!\n" - + "einem Leichnam bin ich nicht zu Haus; Mir geht es wie der Katze mit der Maus.!!!!!!\n" - + "DER HERR: Nun gut, es sei dir ueberlassen! Zieh diesen Geist von seinem!!!!!!\n" - + "Urquell ab, Und fuehr ihn, kannst du ihn erfassen, Auf deinem Wege mit!!!!!!\n" - + "herab, Und steh beschaemt, wenn du bekennen musst: Ein guter Mensch, in!!!!!!\n" - + "seinem dunklen Drange, Ist sich des rechten Weges wohl bewusst.!!!!!!\n" - + "MEPHISTOPHELES: Schon gut! nur dauert es nicht lange. Mir ist fuer meine!!!!!!\n" - + "Wette gar nicht bange. Wenn ich zu meinem Zweck gelange, Erlaubt Ihr mir!!!!!!\n" - + "Triumph aus voller Brust. Staub soll er fressen, und mit Lust, Wie meine!!!!!!\n" - + "Muhme, die beruehmte Schlange.!!!!!!\n" - + "DER HERR: Du darfst auch da nur frei erscheinen; Ich habe deinesgleichen!!!!!!\n" - + "nie gehasst. Von allen Geistern, die verneinen, ist mir der Schalk am!!!!!!\n" - + "wenigsten zur Last. Des Menschen Taetigkeit kann allzu leicht erschlaffen,!!!!!!\n" - + "er liebt sich bald die unbedingte Ruh; Drum geb ich gern ihm den Gesellen!!!!!!\n" - + "zu, Der reizt und wirkt und muss als Teufel schaffen. Doch ihr, die echten!!!!!!\n" - + "Goettersoehne, Erfreut euch der lebendig reichen Schoene! Das Werdende, das!!!!!!\n" - + "ewig wirkt und lebt, Umfass euch mit der Liebe holden Schranken, Und was!!!!!!\n" - + "in schwankender Erscheinung schwebt, Befestigt mit dauernden Gedanken!!!!!!!\n" - + "(Der Himmel schliesst, die Erzengel verteilen sich.)!!!!!!\n" - + "MEPHISTOPHELES (allein): Von Zeit zu Zeit seh ich den Alten gern, Und!!!!!!\n" - + "huete mich, mit ihm zu brechen. Es ist gar huebsch von einem grossen Herrn,!!!!!!\n" - + "So menschlich mit dem Teufel selbst zu sprechen.!!!!!!"; -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitITCase.java deleted file mode 100644 index 944000ca369d33..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitITCase.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.split; - -import org.apache.flink.storm.split.SpoutSplitExample.Enrich; -import org.apache.flink.storm.split.operators.VerifyAndEnrichBolt; -import org.apache.flink.test.util.AbstractTestBase; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; - -/** - * Tests for split examples. - */ -public class SplitITCase extends AbstractTestBase { - - private String output; - - @Before - public void prepare() throws IOException { - output = getTempFilePath("dummy").split(":")[1]; - } - - @After - public void cleanUp() throws IOException { - deleteRecursively(new File(output)); - } - - @Test - public void testEmbeddedSpout() throws Exception { - SpoutSplitExample.main(new String[] { "0", output }); - Assert.assertFalse(VerifyAndEnrichBolt.errorOccured); - Assert.assertFalse(Enrich.errorOccured); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountITCase.java deleted file mode 100644 index 57e5d428ce9b37..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountITCase.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.test.testdata.WordCountData; -import org.apache.flink.test.util.AbstractTestBase; - -import org.junit.Test; - -/** - * Test for the BoltTokenizerWordCount example. - */ -public class BoltTokenizerWordCountITCase extends AbstractTestBase { - - @Test - public void testProgram() throws Exception { - String textPath = createTempFile("text.txt", WordCountData.TEXT); - String resultPath = getTempDirPath("result"); - - BoltTokenizerWordCount.main(new String[]{textPath, resultPath}); - - compareResultsByLinesInMemory(WordCountData.STREAMING_COUNTS_AS_TUPLES, resultPath); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojoITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojoITCase.java deleted file mode 100644 index 656700ee332731..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojoITCase.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.test.testdata.WordCountData; -import org.apache.flink.test.util.AbstractTestBase; - -import org.junit.Test; - -/** - * Test for the BoltTokenizerWordCountPojo example. - */ -public class BoltTokenizerWordCountPojoITCase extends AbstractTestBase { - - @Test - public void testProgram() throws Exception { - String textPath = createTempFile("text.txt", WordCountData.TEXT); - String resultPath = getTempDirPath("result"); - - BoltTokenizerWordCountPojo.main(new String[]{textPath, resultPath}); - - compareResultsByLinesInMemory(WordCountData.STREAMING_COUNTS_AS_TUPLES, resultPath); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNamesITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNamesITCase.java deleted file mode 100644 index 18e1f01e1803c8..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNamesITCase.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.test.testdata.WordCountData; -import org.apache.flink.test.util.AbstractTestBase; - -import org.junit.Test; - -/** - * Test for the BoltTokenizerWordCountWithNames example. - */ -public class BoltTokenizerWordCountWithNamesITCase extends AbstractTestBase { - - @Test - public void testProgram() throws Exception { - String textPath = createTempFile("text.txt", WordCountData.TEXT); - String resultPath = getTempDirPath("result"); - - BoltTokenizerWordCountWithNames.main(new String[]{textPath, resultPath}); - - compareResultsByLinesInMemory(WordCountData.STREAMING_COUNTS_AS_TUPLES, resultPath); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/SpoutSourceWordCountITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/SpoutSourceWordCountITCase.java deleted file mode 100644 index 594f56e281e8e0..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/SpoutSourceWordCountITCase.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wordcount; - -import org.apache.flink.test.testdata.WordCountData; -import org.apache.flink.test.util.AbstractTestBase; - -import org.junit.Test; - -/** - * Test for the SpoutSourceWordCount example. - */ -public class SpoutSourceWordCountITCase extends AbstractTestBase { - - @Test - public void testProgram() throws Exception { - String textPath = createTempFile("text.txt", WordCountData.TEXT); - String resultPath = getTempDirPath("result"); - - SpoutSourceWordCount.main(new String[]{textPath, resultPath}); - - compareResultsByLinesInMemory(WordCountData.STREAMING_COUNTS_AS_TUPLES, resultPath); - } - -} diff --git a/flink-contrib/flink-storm-examples/src/test/resources/log4j-test.properties b/flink-contrib/flink-storm-examples/src/test/resources/log4j-test.properties deleted file mode 100644 index 881dc0609b8fdb..00000000000000 --- a/flink-contrib/flink-storm-examples/src/test/resources/log4j-test.properties +++ /dev/null @@ -1,27 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -# Set root logger level to DEBUG and its only appender to A1. -log4j.rootLogger=OFF, A1 - -# A1 is set to be a ConsoleAppender. -log4j.appender.A1=org.apache.log4j.ConsoleAppender - -# A1 uses PatternLayout. -log4j.appender.A1.layout=org.apache.log4j.PatternLayout -log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/flink-contrib/flink-storm/README.md b/flink-contrib/flink-storm/README.md deleted file mode 100644 index 578d6cce962023..00000000000000 --- a/flink-contrib/flink-storm/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# flink-storm - -`flink-storm` is compatibility layer for Apache Storm and allows to embed Spouts or Bolts unmodified within a regular Flink streaming program (`SpoutWrapper` and `BoltWrapper`). -Additionally, a whole Storm topology can be submitted to Flink (see `FlinkLocalCluster`, and `FlinkSubmitter`). -Only a few minor changes to the original submitting code are required. -The code that builds the topology itself, can be reused unmodified. See `flink-storm-examples` for a simple word-count example. - -**Please note**: Do not add `storm-core` as a dependency. It is already included via `flink-storm`. - -The following Storm features are not (yet/fully) supported by the compatibility layer right now: -* no fault-tolerance guarantees (ie, calls to `ack()`/`fail()` and anchoring is ignored) -* for whole Storm topologies the following is not supported by Flink: - * direct emit connection pattern - * activating/deactivating and rebalancing of topologies - * task hooks - * metrics diff --git a/flink-contrib/flink-storm/pom.xml b/flink-contrib/flink-storm/pom.xml deleted file mode 100644 index 6b6f07ec7450ad..00000000000000 --- a/flink-contrib/flink-storm/pom.xml +++ /dev/null @@ -1,193 +0,0 @@ - - - - - 4.0.0 - - - org.apache.flink - flink-contrib - 1.8-SNAPSHOT - .. - - - flink-storm_${scala.binary.version} - flink-storm - - jar - - - - - - clojars - https://clojars.org/repo/ - - true - - - false - - - - - - - - - - - - com.esotericsoftware.kryo - kryo - provided - - - - - org.apache.flink - flink-streaming-java_${scala.binary.version} - ${project.version} - provided - - - - - org.apache.storm - storm-core - 1.0.0 - - - org.slf4j - log4j-over-slf4j - - - org.apache.logging.log4j - log4j-slf4j-impl - - - slf4j-log4j12 - org.slf4j - - - javax.servlet - servlet-api - - - junit - junit - - - org.mockito - mockito-all - - - org.mockito - mockito-all - - - org.apache.curator - curator-test - - - com.esotericsoftware - kryo - - - ring - ring-core - - - ring - ring-devel - - - ring - ring-servlet - - - ring - ring-jetty-adapter - - - org.mortbay.jetty - jetty - - - org.jgrapht - jgrapht-core - - - compojure - compojure - - - com.twitter - chill-java - - - commons-fileupload - commons-fileupload - - - javax.servlet - servlet-api - - - clout - clout - - - hiccup - hiccup - - - - - - com.googlecode.json-simple - json-simple - 1.1 - - - - - - org.apache.flink - flink-runtime_${scala.binary.version} - ${project.version} - test-jar - test - - - - org.apache.flink - flink-streaming-java_${scala.binary.version} - ${project.version} - test - test-jar - - - - - diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/FiniteSpout.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/FiniteSpout.java deleted file mode 100644 index 7615b2e606367f..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/FiniteSpout.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.topology.IRichSpout; - -/** - * This interface represents a spout that emits a finite number of records. Common spouts emit infinite streams by - * default. To change this behavior and take advantage of Flink's finite-source capabilities, the spout should implement - * this interface. - */ -public interface FiniteSpout extends IRichSpout { - - /** - * When returns true, the spout has reached the end of the stream. - * - * @return true, if the spout's stream reached its end, false otherwise - */ - boolean reachedEnd(); - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/NullTerminatingSpout.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/NullTerminatingSpout.java deleted file mode 100644 index a830b105403594..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/NullTerminatingSpout.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.topology.OutputFieldsDeclarer; - -import java.util.Map; - -/** - * {@link NullTerminatingSpout} in a finite spout (ie, implements {@link FiniteSpout} interface) that wraps an - * infinite spout, and returns {@code true} in {@link #reachedEnd()} when the wrapped spout does not emit a tuple - * in {@code nextTuple()} for the first time. - */ -public class NullTerminatingSpout implements FiniteSpout { - private static final long serialVersionUID = -6976210409932076066L; - - /** The original infinite Spout. */ - private final IRichSpout spout; - /** The observer that checks if the given spouts emit a tuple or not on nextTuple(). */ - private SpoutOutputCollectorObserver observer; - - public NullTerminatingSpout(IRichSpout spout) { - this.spout = spout; - } - - @Override - public void open(@SuppressWarnings("rawtypes") Map conf, TopologyContext context, SpoutOutputCollector collector) { - this.observer = new SpoutOutputCollectorObserver(collector); - this.observer.emitted = true; - this.spout.open(conf, context, this.observer); - } - - @Override - public void close() { - this.spout.close(); - } - - @Override - public void activate() { - this.spout.activate(); - } - - @Override - public void deactivate() { - this.spout.deactivate(); - } - - @Override - public void nextTuple() { - this.observer.emitted = false; - this.spout.nextTuple(); - } - - @Override - public void ack(Object msgId) { - this.spout.ack(msgId); - } - - @Override - public void fail(Object msgId) { - this.spout.fail(msgId); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - this.spout.declareOutputFields(declarer); - } - - @Override - public Map getComponentConfiguration() { - return this.spout.getComponentConfiguration(); - } - - @Override - public boolean reachedEnd() { - return !this.observer.emitted; - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamMapper.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamMapper.java deleted file mode 100644 index d2e84c5c2a5bd0..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamMapper.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.SplitStream; - -/** - * Strips {@link SplitStreamType}{@code } away, ie, extracts the wrapped record of type {@code T}. Can be used to get - * a "clean" stream from a Spout/Bolt that declared multiple output streams (after the streams got separated using - * {@link DataStream#split(org.apache.flink.streaming.api.collector.selector.OutputSelector) .split(...)} and - * {@link SplitStream#select(String...) .select(...)}). - * - * @param - */ -public class SplitStreamMapper implements MapFunction, T> { - private static final long serialVersionUID = 3550359150160908564L; - - @Override - public T map(SplitStreamType value) throws Exception { - return value.value; - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamType.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamType.java deleted file mode 100644 index 36894c793490c7..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamType.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.flink.streaming.api.datastream.DataStream; - -/** - * Used by org.apache.flink.storm.wrappers.AbstractStormCollector to wrap - * output tuples if multiple output streams are declared. For this case, the Flink output data stream must be split via - * {@link DataStream#split(org.apache.flink.streaming.api.collector.selector.OutputSelector) .split(...)} using - * {@link StormStreamSelector}. - * - */ -public class SplitStreamType { - - /** The stream ID this tuple belongs to. */ - public String streamId; - /** The actual data value. */ - public T value; - - @Override - public String toString() { - return ""; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SplitStreamType other = (SplitStreamType) o; - - return this.streamId.equals(other.streamId) && this.value.equals(other.value); - } -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SpoutOutputCollectorObserver.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SpoutOutputCollectorObserver.java deleted file mode 100644 index 8be466eb98b614..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SpoutOutputCollectorObserver.java +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.utils.Utils; - -import java.util.List; - -/** - * Observes if a call to any {@code emit(...)} or {@code emitDirect(...)} method is made. - * The internal flag {@link #emitted} must be reset by the user manually. - */ -class SpoutOutputCollectorObserver extends SpoutOutputCollector { - - /** The collector to be observed. */ - private final SpoutOutputCollector delegate; - /** The internal flag that it set to {@code true} if a tuple gets emitted. */ - boolean emitted; - - public SpoutOutputCollectorObserver(SpoutOutputCollector delegate) { - super(null); - this.delegate = delegate; - } - - @Override - public List emit(String streamId, List tuple, Object messageId) { - emitted = true; - return this.delegate.emit(streamId, tuple, messageId); - } - - @Override - public List emit(List tuple, Object messageId) { - return emit(Utils.DEFAULT_STREAM_ID, tuple, messageId); - } - - @Override - public List emit(List tuple) { - return emit(tuple, null); - } - - @Override - public List emit(String streamId, List tuple) { - return emit(streamId, tuple, null); - } - - @Override - public void emitDirect(int taskId, String streamId, List tuple, Object messageId) { - emitted = true; - delegate.emitDirect(taskId, streamId, tuple, messageId); - } - - @Override - public void emitDirect(int taskId, List tuple, Object messageId) { - emitDirect(taskId, Utils.DEFAULT_STREAM_ID, tuple, messageId); - } - - @Override - public void emitDirect(int taskId, String streamId, List tuple) { - emitDirect(taskId, streamId, tuple, null); - } - - @Override - public void emitDirect(int taskId, List tuple) { - emitDirect(taskId, tuple, null); - } - - @Override - public void reportError(Throwable error) { - delegate.reportError(error); - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormConfig.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormConfig.java deleted file mode 100644 index a9d7bfd7abb7f2..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormConfig.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.flink.api.common.ExecutionConfig.GlobalJobParameters; - -import org.apache.storm.Config; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * {@link StormConfig} is used to provide a user-defined Storm configuration (ie, a raw {@link Map} or {@link Config} - * object) for embedded Spouts and Bolts. - */ -@SuppressWarnings("rawtypes") -public final class StormConfig extends GlobalJobParameters implements Map { - private static final long serialVersionUID = 8019519109673698490L; - - /** Contains the actual configuration that is provided to Spouts and Bolts. */ - private final Map config = new HashMap(); - - /** - * Creates an empty configuration. - */ - public StormConfig() { - } - - /** - * Creates an configuration with initial values provided by the given {@code Map}. - * - * @param config - * Initial values for this configuration. - */ - @SuppressWarnings("unchecked") - public StormConfig(Map config) { - this.config.putAll(config); - } - - @Override - public int size() { - return this.config.size(); - } - - @Override - public boolean isEmpty() { - return this.config.isEmpty(); - } - - @Override - public boolean containsKey(Object key) { - return this.config.containsKey(key); - } - - @Override - public boolean containsValue(Object value) { - return this.config.containsValue(value); - } - - @Override - public Object get(Object key) { - return this.config.get(key); - } - - @SuppressWarnings("unchecked") - @Override - public Object put(Object key, Object value) { - return this.config.put(key, value); - } - - @Override - public Object remove(Object key) { - return this.config.remove(key); - } - - @SuppressWarnings("unchecked") - @Override - public void putAll(Map m) { - this.config.putAll(m); - } - - @Override - public void clear() { - this.config.clear(); - } - - @SuppressWarnings("unchecked") - @Override - public Set keySet() { - return this.config.keySet(); - } - - @SuppressWarnings("unchecked") - @Override - public Collection values() { - return this.config.values(); - } - - @SuppressWarnings("unchecked") - @Override - public Set> entrySet() { - return this.config.entrySet(); - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormStreamSelector.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormStreamSelector.java deleted file mode 100644 index e8007266e53c56..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormStreamSelector.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.flink.streaming.api.collector.selector.OutputSelector; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -/** - * Used to split multiple declared output streams within Flink. - */ -public final class StormStreamSelector implements OutputSelector> { - private static final long serialVersionUID = 2553423379715401023L; - - /** internal cache to avoid short living ArrayList objects. */ - private final HashMap> streams = new HashMap>(); - - @Override - public Iterable select(SplitStreamType value) { - String sid = value.streamId; - List streamId = this.streams.get(sid); - if (streamId == null) { - streamId = new ArrayList(1); - streamId.add(sid); - this.streams.put(sid, streamId); - } - return streamId; - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/AbstractStormCollector.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/AbstractStormCollector.java deleted file mode 100644 index 21ce1152387b2b..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/AbstractStormCollector.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.api.java.tuple.Tuple; -import org.apache.flink.api.java.tuple.Tuple0; -import org.apache.flink.api.java.tuple.Tuple25; -import org.apache.flink.storm.util.SplitStreamType; - -import java.util.HashMap; -import java.util.List; -import java.util.Map.Entry; - -/** - * A {@link AbstractStormCollector} transforms Storm tuples to Flink tuples. - */ -abstract class AbstractStormCollector { - - /** Flink output tuple of concrete type {@link Tuple0} to {@link Tuple25} per output stream. */ - protected final HashMap outputTuple = new HashMap(); - /** Flink split tuple. Used, if multiple output streams are declared. */ - private final SplitStreamType splitTuple = new SplitStreamType(); - /** - * The number of attributes of the output tuples per stream. (Determines the concrete type of {@link #outputTuple}). - * If {@link #numberOfAttributes} is zero, {@link #outputTuple} is not used and "raw" data type is used. - */ - protected final HashMap numberOfAttributes; - /** Indicates of multiple output stream are declared and thus {@link SplitStreamType} must be used as output. */ - private final boolean split; - /** The ID of the producer task. */ - private final int taskId; - /** Is set to {@code true} each time a tuple is emitted. */ - boolean tupleEmitted = false; - - /** - * Instantiates a new {@link AbstractStormCollector} that emits Flink tuples via {@link #doEmit(Object)}. If the - * number of attributes is negative, any output type is supported (ie, raw type). If the number of attributes is - * between 0 and 25, the output type is {@link Tuple0} to {@link Tuple25}, respectively. - * - * @param numberOfAttributes - * The number of attributes of the emitted tuples per output stream. - * @param taskId - * The ID of the producer task (negative value for unknown). - * @throws UnsupportedOperationException - * if the specified number of attributes is greater than 25 or taskId support is enabled for a raw - * stream - */ - AbstractStormCollector(final HashMap numberOfAttributes, final int taskId) - throws UnsupportedOperationException { - assert (numberOfAttributes != null); - - this.numberOfAttributes = numberOfAttributes; - this.split = this.numberOfAttributes.size() > 1; - this.taskId = taskId; - - for (Entry outputStream : numberOfAttributes.entrySet()) { - int numAtt = outputStream.getValue(); - - if (this.taskId >= 0) { - if (numAtt < 0) { - throw new UnsupportedOperationException( - "Task ID transmission not supported for raw streams: " - + outputStream.getKey()); - } - ++numAtt; - } - - if (numAtt > 25) { - if (this.taskId >= 0) { - throw new UnsupportedOperationException( - "Flink cannot handle more then 25 attributes, but 25 (24 plus 1 for produer task ID) " - + " are declared for stream '" + outputStream.getKey() + "' by the given bolt."); - } else { - throw new UnsupportedOperationException( - "Flink cannot handle more then 25 attributes, but " + numAtt - + " are declared for stream '" + outputStream.getKey() + "' by the given bolt."); - } - } else if (numAtt >= 0) { - try { - this.outputTuple.put(outputStream.getKey(), - org.apache.flink.api.java.tuple.Tuple.getTupleClass(numAtt) - .newInstance()); - } catch (final InstantiationException e) { - throw new RuntimeException(e); - } catch (final IllegalAccessException e) { - throw new RuntimeException(e); - } - - } - } - } - - /** - * Transforms a Storm tuple into a Flink tuple of type {@code OUT} and emits this tuple via {@link #doEmit(Object)} - * to the specified output stream. - * - * @param The - * The output stream id. - * @param tuple - * The Storm tuple to be emitted. - * @return the return value of {@link #doEmit(Object)} - */ - @SuppressWarnings("unchecked") - protected final List tansformAndEmit(final String streamId, final List tuple) { - List taskIds; - - int numAtt = this.numberOfAttributes.get(streamId); - int taskIdIdx = numAtt; - if (this.taskId >= 0 && numAtt < 0) { - numAtt = 1; - taskIdIdx = 0; - } - if (numAtt >= 0) { - assert (tuple.size() == numAtt); - Tuple out = this.outputTuple.get(streamId); - for (int i = 0; i < numAtt; ++i) { - out.setField(tuple.get(i), i); - } - if (this.taskId >= 0) { - out.setField(this.taskId, taskIdIdx); - } - if (this.split) { - this.splitTuple.streamId = streamId; - this.splitTuple.value = out; - - taskIds = doEmit((OUT) this.splitTuple); - } else { - taskIds = doEmit((OUT) out); - } - - } else { - assert (tuple.size() == 1); - if (this.split) { - this.splitTuple.streamId = streamId; - this.splitTuple.value = tuple.get(0); - - taskIds = doEmit((OUT) this.splitTuple); - } else { - taskIds = doEmit((OUT) tuple.get(0)); - } - } - this.tupleEmitted = true; - - return taskIds; - } - - /** - * Emits a Flink tuple. - * - * @param flinkTuple - * The tuple to be emitted. - * @return the IDs of the tasks this tuple was sent to - */ - protected abstract List doEmit(OUT flinkTuple); - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltCollector.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltCollector.java deleted file mode 100644 index 82c7be31adf2c1..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltCollector.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.api.java.tuple.Tuple0; -import org.apache.flink.api.java.tuple.Tuple25; -import org.apache.flink.streaming.api.operators.Output; -import org.apache.flink.util.Collector; - -import org.apache.storm.task.IOutputCollector; -import org.apache.storm.tuple.Tuple; - -import java.util.Collection; -import java.util.HashMap; -import java.util.List; - -/** - * A {@link BoltCollector} is used by {@link BoltWrapper} to provided an Storm compatible - * output collector to the wrapped bolt. It transforms the emitted Storm tuples into Flink tuples - * and emits them via the provide {@link Output} object. - */ -class BoltCollector extends AbstractStormCollector implements IOutputCollector { - - /** The Flink output Collector. */ - private final Collector flinkOutput; - - /** - * Instantiates a new {@link BoltCollector} that emits Flink tuples to the given Flink output object. If the - * number of attributes is negative, any output type is supported (ie, raw type). If the number of attributes is - * between 0 and 25, the output type is {@link Tuple0} to {@link Tuple25}, respectively. - * - * @param numberOfAttributes - * The number of attributes of the emitted tuples per output stream. - * @param taskId - * The ID of the producer task (negative value for unknown). - * @param flinkOutput - * The Flink output object to be used. - * @throws UnsupportedOperationException - * if the specified number of attributes is greater than 25 - */ - BoltCollector(final HashMap numberOfAttributes, final int taskId, - final Collector flinkOutput) throws UnsupportedOperationException { - super(numberOfAttributes, taskId); - assert (flinkOutput != null); - this.flinkOutput = flinkOutput; - } - - @Override - protected List doEmit(final OUT flinkTuple) { - this.flinkOutput.collect(flinkTuple); - // TODO - return null; - } - - @Override - public void reportError(final Throwable error) { - // not sure, if Flink can support this - } - - @Override - public List emit(final String streamId, final Collection anchors, final List tuple) { - return this.tansformAndEmit(streamId, tuple); - } - - @Override - public void emitDirect(final int taskId, final String streamId, final Collection anchors, final List tuple) { - throw new UnsupportedOperationException("Direct emit is not supported by Flink"); - } - - @Override - public void ack(final Tuple input) {} - - @Override - public void fail(final Tuple input) {} - - @Override - public void resetTimeout(Tuple var1) {} - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltWrapper.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltWrapper.java deleted file mode 100644 index 1c12290eb26e31..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltWrapper.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.api.common.ExecutionConfig.GlobalJobParameters; -import org.apache.flink.api.java.tuple.Tuple0; -import org.apache.flink.api.java.tuple.Tuple1; -import org.apache.flink.api.java.tuple.Tuple25; -import org.apache.flink.storm.util.StormConfig; -import org.apache.flink.streaming.api.operators.AbstractStreamOperator; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; -import org.apache.flink.streaming.api.operators.TimestampedCollector; -import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.MessageId; -import org.apache.storm.utils.Utils; - -import java.util.Collection; -import java.util.HashMap; - -import static java.util.Arrays.asList; - -/** - * A {@link BoltWrapper} wraps an {@link IRichBolt} in order to execute the Storm bolt within a Flink Streaming program. - * It takes the Flink input tuples of type {@code IN} and transforms them into {@link StormTuple}s that the bolt can - * process. Furthermore, it takes the bolt's output tuples and transforms them into Flink tuples of type {@code OUT} - * (see {@link AbstractStormCollector} for supported types).
- *
- * Works for single input streams only! See {@link MergedInputsBoltWrapper} for multi-input stream - * Bolts. - */ -public class BoltWrapper extends AbstractStreamOperator implements OneInputStreamOperator { - private static final long serialVersionUID = -4788589118464155835L; - - /** The default input component ID. */ - public static final String DEFAULT_ID = "default ID"; - /** The default bolt ID. */ - public static final String DEFUALT_BOLT_NAME = "Unnamed Bolt"; - - /** The wrapped Storm {@link IRichBolt bolt}. */ - protected final IRichBolt bolt; - /** The name of the bolt. */ - private final String name; - /** Number of attributes of the bolt's output tuples per stream. */ - private final HashMap numberOfAttributes; - - /** The topology context of the bolt. */ - private transient TopologyContext topologyContext; - - /** The schema (ie, ordered field names) of the input streams per producer taskID. */ - private final HashMap inputSchemas = new HashMap(); - - /** - * We have to use this because Operators must output {@link StreamRecord}. - */ - protected transient TimestampedCollector flinkCollector; - - /** - * Instantiates a new {@link BoltWrapper} that wraps the given Storm {@link IRichBolt bolt} such that it can be used - * within a Flink streaming program. As no input schema is defined, attribute-by-name access in only possible for - * POJO input types. The output type will be one of {@link Tuple0} to {@link Tuple25} depending on the bolt's - * declared number of attributes. - * - * @param bolt - * The Storm {@link IRichBolt bolt} to be used. - * @throws IllegalArgumentException - * If the number of declared output attributes is not with range [0;25]. - */ - public BoltWrapper(final IRichBolt bolt) throws IllegalArgumentException { - this(bolt, null, (Collection) null); - } - - /** - * Instantiates a new {@link BoltWrapper} that wraps the given Storm {@link IRichBolt bolt} such that it can be used - * within a Flink streaming program. The given input schema enable attribute-by-name access for input types - * {@link Tuple0} to {@link Tuple25}. The output type will be one of {@link Tuple0} to {@link Tuple25} depending on - * the bolt's declared number of attributes. - * - * @param bolt - * The Storm {@link IRichBolt bolt} to be used. - * @param inputSchema - * The schema (ie, ordered field names) of the input stream. @throws IllegalArgumentException - * @throws IllegalArgumentException - * If the number of declared output attributes is not with range [0;25]. - */ - public BoltWrapper(final IRichBolt bolt, final Fields inputSchema) - throws IllegalArgumentException { - this(bolt, inputSchema, (Collection) null); - } - - /** - * Instantiates a new {@link BoltWrapper} that wraps the given Storm {@link IRichBolt bolt} such that it can be used - * within a Flink streaming program. As no input schema is defined, attribute-by-name access in only possible for - * POJO input types. The output type can be any type if parameter {@code rawOutput} is {@code true} and the bolt's - * number of declared output tuples is 1. If {@code rawOutput} is {@code false} the output type will be one of - * {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of attributes. - * - * @param bolt - * The Storm {@link IRichBolt bolt} to be used. - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type {@link Tuple1} but be - * of a raw type. - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not within range - * [1;25]. - */ - public BoltWrapper(final IRichBolt bolt, final String[] rawOutputs) - throws IllegalArgumentException { - this(bolt, null, asList(rawOutputs)); - } - - /** - * Instantiates a new {@link BoltWrapper} that wraps the given Storm {@link IRichBolt bolt} such that it can be used - * within a Flink streaming program. As no input schema is defined, attribute-by-name access in only possible for - * POJO input types. The output type can be any type if parameter {@code rawOutput} is {@code true} and the bolt's - * number of declared output tuples is 1. If {@code rawOutput} is {@code false} the output type will be one of - * {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of attributes. - * - * @param bolt - * The Storm {@link IRichBolt bolt} to be used. - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type {@link Tuple1} but be - * of a raw type. - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not with range - * [1;25]. - */ - public BoltWrapper(final IRichBolt bolt, final Collection rawOutputs) throws IllegalArgumentException { - this(bolt, null, rawOutputs); - } - - /** - * Instantiates a new {@link BoltWrapper} that wraps the given Storm {@link IRichBolt bolt} such that it can be used - * within a Flink streaming program. The given input schema enable attribute-by-name access for input types - * {@link Tuple0} to {@link Tuple25}. The output type can be any type if parameter {@code rawOutput} is {@code true} - * and the bolt's number of declared output tuples is 1. If {@code rawOutput} is {@code false} the output type will - * be one of {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of attributes. - * - * @param bolt - * The Storm {@link IRichBolt bolt} to be used. - * @param inputSchema - * The schema (ie, ordered field names) of the input stream. - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type {@link Tuple1} but be - * of a raw type. - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not with range - * [0;25]. - */ - public BoltWrapper( - final IRichBolt bolt, - final Fields inputSchema, - final String[] rawOutputs) - throws IllegalArgumentException { - this(bolt, inputSchema, asList(rawOutputs)); - } - - /** - * Instantiates a new {@link BoltWrapper} that wraps the given Storm {@link IRichBolt bolt} such that it can be used - * within a Flink streaming program. The given input schema enable attribute-by-name access for input types - * {@link Tuple0} to {@link Tuple25}. The output type can be any type if parameter {@code rawOutput} is {@code true} - * and the bolt's number of declared output tuples is 1. If {@code rawOutput} is {@code false} the output type will - * be one of {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of attributes. - * - * @param bolt - * The Storm {@link IRichBolt bolt} to be used. - * @param inputSchema - * The schema (ie, ordered field names) of the input stream. @throws IllegalArgumentException If - * {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not with range - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type {@link Tuple1} but be - * of a raw type. - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not with range - * [0;25]. - */ - public BoltWrapper(final IRichBolt bolt, final Fields inputSchema, - final Collection rawOutputs) throws IllegalArgumentException { - this(bolt, DEFUALT_BOLT_NAME, Utils.DEFAULT_STREAM_ID, DEFAULT_ID, inputSchema, rawOutputs); - } - - /** - * Instantiates a new {@link BoltWrapper} that wraps the given Storm {@link IRichBolt bolt} such that it can be used - * within a Flink streaming program. The given input schema enable attribute-by-name access for input types - * {@link Tuple0} to {@link Tuple25}. The output type can be any type if parameter {@code rawOutput} is {@code true} - * and the bolt's number of declared output tuples is 1. If {@code rawOutput} is {@code false} the output type will - * be one of {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of attributes. - * - * @param bolt - * The Storm {@link IRichBolt bolt} to be used. - * @param name - * The name of the bolt. - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type {@link Tuple1} but be - * of a raw type. - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not with range - * [0;25]. - */ - public BoltWrapper(final IRichBolt bolt, final String name, final String inputStreamId, - final String inputComponentId, final Fields inputSchema, - final Collection rawOutputs) throws IllegalArgumentException { - this.bolt = bolt; - this.name = name; - this.inputSchemas.put(null, inputSchema); - this.numberOfAttributes = WrapperSetupHelper.getNumberOfAttributes(bolt, rawOutputs); - } - - @Override - public void open() throws Exception { - super.open(); - - this.flinkCollector = new TimestampedCollector<>(this.output); - - GlobalJobParameters config = getExecutionConfig().getGlobalJobParameters(); - StormConfig stormConfig = new StormConfig(); - - if (config != null) { - if (config instanceof StormConfig) { - stormConfig = (StormConfig) config; - } else { - stormConfig.putAll(config.toMap()); - } - } - - this.topologyContext = WrapperSetupHelper.createTopologyContext( - getRuntimeContext(), this.bolt, this.name, stormConfig); - - final OutputCollector stormCollector = new OutputCollector(new BoltCollector( - this.numberOfAttributes, this.topologyContext.getThisTaskId(), this.flinkCollector)); - - this.bolt.prepare(stormConfig, this.topologyContext, stormCollector); - } - - @Override - public void dispose() throws Exception { - super.dispose(); - this.bolt.cleanup(); - } - - @Override - public void processElement(final StreamRecord element) throws Exception { - this.flinkCollector.setTimestamp(element); - - IN value = element.getValue(); - - this.bolt.execute(new StormTuple<>(value, this.inputSchemas.get(null), -1, null, null, - MessageId.makeUnanchored())); - } -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/FlinkTopologyContext.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/FlinkTopologyContext.java deleted file mode 100644 index e84abccefbab7f..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/FlinkTopologyContext.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import clojure.lang.Atom; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.hooks.ITaskHook; -import org.apache.storm.metric.api.CombinedMetric; -import org.apache.storm.metric.api.ICombiner; -import org.apache.storm.metric.api.IMetric; -import org.apache.storm.metric.api.IReducer; -import org.apache.storm.metric.api.ReducedMetric; -import org.apache.storm.state.ISubscribedState; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.tuple.Fields; - -import java.util.Collection; -import java.util.List; -import java.util.Map; - -/** - * {@link FlinkTopologyContext} is a {@link TopologyContext} that overwrites certain method that are not applicable when - * a Storm topology is executed within Flink. - */ -final class FlinkTopologyContext extends TopologyContext { - - /** - * Instantiates a new {@link FlinkTopologyContext} for a given Storm topology. The context object is instantiated - * for each parallel task - */ - FlinkTopologyContext(final StormTopology topology, - @SuppressWarnings("rawtypes") final Map stormConf, - final Map taskToComponent, final Map> componentToSortedTasks, - final Map> componentToStreamToFields, final String stormId, final String codeDir, - final String pidDir, final Integer taskId, final Integer workerPort, final List workerTasks, - final Map defaultResources, final Map userResources, - final Map executorData, @SuppressWarnings("rawtypes") final Map registeredMetrics, - final Atom openOrPrepareWasCalled) { - super(topology, stormConf, taskToComponent, componentToSortedTasks, componentToStreamToFields, stormId, - codeDir, pidDir, taskId, workerPort, workerTasks, defaultResources, userResources, executorData, - registeredMetrics, openOrPrepareWasCalled); - } - - /** - * Not supported by Flink. - * - * @throws UnsupportedOperationException - * at every invocation - */ - @Override - public void addTaskHook(final ITaskHook hook) { - throw new UnsupportedOperationException("Task hooks are not supported by Flink"); - } - - /** - * Not supported by Flink. - * - * @throws UnsupportedOperationException - * at every invocation - */ - @Override - public Collection getHooks() { - throw new UnsupportedOperationException("Task hooks are not supported by Flink"); - } - - /** - * Not supported by Flink. - * - * @throws UnsupportedOperationException - * at every invocation - */ - @Override - public IMetric getRegisteredMetricByName(final String name) { - throw new UnsupportedOperationException("Metrics are not supported by Flink"); - - } - - /** - * Not supported by Flink. - * - * @throws UnsupportedOperationException - * at every invocation - */ - @SuppressWarnings("rawtypes") - @Override - public CombinedMetric registerMetric(final String name, final ICombiner combiner, final int timeBucketSizeInSecs) { - throw new UnsupportedOperationException("Metrics are not supported by Flink"); - } - - /** - * Not supported by Flink. - * - * @throws UnsupportedOperationException - * at every invocation - */ - @SuppressWarnings("rawtypes") - @Override - public ReducedMetric registerMetric(final String name, final IReducer combiner, final int timeBucketSizeInSecs) { - throw new UnsupportedOperationException("Metrics are not supported by Flink"); - } - - /** - * Not supported by Flink. - * - * @throws UnsupportedOperationException - * at every invocation - */ - @Override - public T registerMetric(final String name, final T metric, final int timeBucketSizeInSecs) { - throw new UnsupportedOperationException("Metrics are not supported by Flink"); - } - - /** - * Not supported by Flink. - * - * @throws UnsupportedOperationException - * at every invocation - */ - @Override - public T setAllSubscribedState(final T obj) { - throw new UnsupportedOperationException("Not supported by Flink"); - - } - - /** - * Not supported by Flink. - * - * @throws UnsupportedOperationException - * at every invocation - */ - @Override - public T setSubscribedState(final String componentId, final T obj) { - throw new UnsupportedOperationException("Not supported by Flink"); - } - - /** - * Not supported by Flink. - * - * @throws UnsupportedOperationException - * at every invocation - */ - @Override - public T setSubscribedState(final String componentId, final String streamId, final T - obj) { - throw new UnsupportedOperationException("Not supported by Flink"); - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/MergedInputsBoltWrapper.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/MergedInputsBoltWrapper.java deleted file mode 100644 index 88ae35521fa7d4..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/MergedInputsBoltWrapper.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.api.java.tuple.Tuple0; -import org.apache.flink.api.java.tuple.Tuple1; -import org.apache.flink.api.java.tuple.Tuple25; -import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; - -import org.apache.storm.topology.IRichBolt; - -import java.util.Collection; - -import static java.util.Arrays.asList; - -/** - * A {@link MergedInputsBoltWrapper} is a {@link BoltWrapper} that expects input tuples of type {@link StormTuple}. It - * can be used to wrap a multi-input bolt and assumes that all input stream got merged into a {@link StormTuple} stream - * already. - */ -public final class MergedInputsBoltWrapper extends BoltWrapper, OUT> { - private static final long serialVersionUID = 6399319187892878545L; - - /** - * Instantiates a new {@link MergedInputsBoltWrapper} that wraps the given Storm {@link IRichBolt bolt} such that it - * can be used within a Flink streaming program. The output type will be one of {@link Tuple0} to {@link Tuple25} - * depending on the bolt's declared number of attributes. - * - * @param bolt - * The Storm {@link IRichBolt bolt} to be used. - * @throws IllegalArgumentException - * If the number of declared output attributes is not with range [0;25]. - */ - public MergedInputsBoltWrapper(final IRichBolt bolt) throws IllegalArgumentException { - super(bolt); - } - - /** - * Instantiates a new {@link MergedInputsBoltWrapper} that wraps the given Storm {@link IRichBolt bolt} such that it - * can be used within a Flink streaming program. The output type can be any type if parameter {@code rawOutput} is - * {@code true} and the bolt's number of declared output tuples is 1. If {@code rawOutput} is {@code false} the - * output type will be one of {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of - * attributes. - * - * @param bolt - * The Storm {@link IRichBolt bolt} to be used. - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type {@link Tuple1} but be - * of a raw type. - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not within range - * [1;25]. - */ - public MergedInputsBoltWrapper(final IRichBolt bolt, final String[] rawOutputs) - throws IllegalArgumentException { - super(bolt, asList(rawOutputs)); - } - - /** - * Instantiates a new {@link MergedInputsBoltWrapper} that wraps the given Storm {@link IRichBolt bolt} such that it - * can be used within a Flink streaming program. The output type can be any type if parameter {@code rawOutput} is - * {@code true} and the bolt's number of declared output tuples is 1. If {@code rawOutput} is {@code false} the - * output type will be one of {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of - * attributes. - * - * @param bolt - * The Storm {@link IRichBolt bolt} to be used. - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type {@link Tuple1} but be - * of a raw type. - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not with range - * [1;25]. - */ - public MergedInputsBoltWrapper(final IRichBolt bolt, final Collection rawOutputs) - throws IllegalArgumentException { - super(bolt, rawOutputs); - } - - /** - * Instantiates a new {@link MergedInputsBoltWrapper} that wraps the given Storm {@link IRichBolt bolt} such that it - * can be used within a Flink streaming program. The output type can be any type if parameter {@code rawOutput} is - * {@code true} and the bolt's number of declared output tuples is 1. If {@code rawOutput} is {@code false} the - * output type will be one of {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of - * attributes. - * - * @param bolt - * The Storm {@link IRichBolt bolt} to be used. - * @param name - * The name of the bolt. - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type {@link Tuple1} but be - * of a raw type. - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not with range - * [0;25]. - */ - public MergedInputsBoltWrapper(final IRichBolt bolt, final String name, final Collection rawOutputs) - throws IllegalArgumentException { - super(bolt, name, null, null, null, rawOutputs); - } - - @Override - public void processElement(final StreamRecord> element) throws Exception { - this.flinkCollector.setTimestamp(element); - this.bolt.execute(element.getValue()); - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SetupOutputFieldsDeclarer.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SetupOutputFieldsDeclarer.java deleted file mode 100644 index d927f0ef652433..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SetupOutputFieldsDeclarer.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.utils.Utils; - -import java.util.HashMap; - -/** - * {@link SetupOutputFieldsDeclarer} is used by {@link WrapperSetupHelper} to determine the output streams and - * number of attributes declared by the wrapped spout's or bolt's {@code declare(...)}/{@code declareStream(...)} - * method. - */ -class SetupOutputFieldsDeclarer implements OutputFieldsDeclarer { - - /** The declared output streams and schemas. */ - HashMap outputStreams = new HashMap(); - /** The number of attributes for each declared stream by the wrapped operator. */ - HashMap outputSchemas = new HashMap(); - - @Override - public void declare(final Fields fields) { - this.declareStream(Utils.DEFAULT_STREAM_ID, false, fields); - } - - @Override - public void declare(final boolean direct, final Fields fields) { - this.declareStream(Utils.DEFAULT_STREAM_ID, direct, fields); - } - - @Override - public void declareStream(final String streamId, final Fields fields) { - this.declareStream(streamId, false, fields); - } - - @Override - public void declareStream(final String streamId, final boolean direct, final Fields fields) { - if (streamId == null) { - throw new IllegalArgumentException("Stream ID cannot be null."); - } - if (direct) { - throw new UnsupportedOperationException("Direct emit is not supported by Flink"); - } - - this.outputStreams.put(streamId, fields); - this.outputSchemas.put(streamId, fields.size()); - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutCollector.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutCollector.java deleted file mode 100644 index 6e3a39af6abceb..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutCollector.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.api.java.tuple.Tuple0; -import org.apache.flink.api.java.tuple.Tuple25; -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext; - -import org.apache.storm.spout.ISpoutOutputCollector; - -import java.util.HashMap; -import java.util.List; - -/** - * A {@link SpoutCollector} is used by {@link SpoutWrapper} to provided an Storm - * compatible output collector to the wrapped spout. It transforms the emitted Storm tuples into - * Flink tuples and emits them via the provide {@link SourceContext} object. - */ -class SpoutCollector extends AbstractStormCollector implements ISpoutOutputCollector { - - /** The Flink source context object. */ - private final SourceContext flinkContext; - - /** - * Instantiates a new {@link SpoutCollector} that emits Flink tuples to the given Flink source context. If the - * number of attributes is specified as zero, any output type is supported. If the number of attributes is between 0 - * to 25, the output type is {@link Tuple0} to {@link Tuple25}, respectively. - * - * @param numberOfAttributes - * The number of attributes of the emitted tuples. - * @param taskId - * The ID of the producer task (negative value for unknown). - * @param flinkContext - * The Flink source context to be used. - * @throws UnsupportedOperationException - * if the specified number of attributes is greater than 25 - */ - SpoutCollector(final HashMap numberOfAttributes, final int taskId, - final SourceContext flinkContext) throws UnsupportedOperationException { - super(numberOfAttributes, taskId); - assert (flinkContext != null); - this.flinkContext = flinkContext; - } - - @Override - protected List doEmit(final OUT flinkTuple) { - this.flinkContext.collect(flinkTuple); - // TODO - return null; - } - - @Override - public void reportError(final Throwable error) { - // not sure, if Flink can support this - } - - @Override - public List emit(final String streamId, final List tuple, final Object messageId) { - return this.tansformAndEmit(streamId, tuple); - } - - @Override - public void emitDirect(final int taskId, final String streamId, final List tuple, final Object messageId) { - throw new UnsupportedOperationException("Direct emit is not supported by Flink"); - } - - public long getPendingCount() { - return 0; - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutWrapper.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutWrapper.java deleted file mode 100644 index eb103840f8789b..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutWrapper.java +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.api.common.ExecutionConfig.GlobalJobParameters; -import org.apache.flink.api.common.functions.StoppableFunction; -import org.apache.flink.api.java.tuple.Tuple0; -import org.apache.flink.api.java.tuple.Tuple1; -import org.apache.flink.api.java.tuple.Tuple25; -import org.apache.flink.storm.util.FiniteSpout; -import org.apache.flink.storm.util.StormConfig; -import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; -import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichSpout; - -import java.util.Collection; -import java.util.HashMap; - -import static java.util.Arrays.asList; - -/** - * A {@link SpoutWrapper} wraps an {@link IRichSpout} in order to execute it within a Flink Streaming program. It - * takes the spout's output tuples and transforms them into Flink tuples of type {@code OUT} (see - * {@link SpoutCollector} for supported types).
- *
- * Per default, {@link SpoutWrapper} calls the wrapped spout's {@link IRichSpout#nextTuple() nextTuple()} method in - * an infinite loop.
- * Alternatively, {@link SpoutWrapper} can call {@link IRichSpout#nextTuple() nextTuple()} for a finite number of - * times and terminate automatically afterwards (for finite input streams). The number of {@code nextTuple()} calls can - * be specified as a certain number of invocations or can be undefined. In the undefined case, {@link SpoutWrapper} - * terminates if no record was emitted to the output collector for the first time during a call to - * {@link IRichSpout#nextTuple() nextTuple()}.
- * If the given spout implements {@link FiniteSpout} interface and {@link #numberOfInvocations} is not provided or - * is {@code null}, {@link SpoutWrapper} calls {@link IRichSpout#nextTuple() nextTuple()} method until - * {@link FiniteSpout#reachedEnd()} returns true. - */ -public final class SpoutWrapper extends RichParallelSourceFunction implements StoppableFunction { - private static final long serialVersionUID = -218340336648247605L; - - /** Number of attributes of the spouts's output tuples per stream. */ - private final HashMap numberOfAttributes; - /** The wrapped {@link IRichSpout spout}. */ - private final IRichSpout spout; - /** The name of the spout. */ - private final String name; - /** Indicates, if the source is still running or was canceled. */ - private volatile boolean isRunning = true; - /** The number of {@link IRichSpout#nextTuple()} calls. */ - private Integer numberOfInvocations; // do not use int -> null indicates an infinite loop - - /** - * Instantiates a new {@link SpoutWrapper} that calls the {@link IRichSpout#nextTuple() nextTuple()} method of - * the given {@link IRichSpout spout} in an infinite loop. The output type will be one of {@link Tuple0} to - * {@link Tuple25} depending on the spout's declared number of attributes. - * - * @param spout - * The {@link IRichSpout spout} to be used. - * @throws IllegalArgumentException - * If the number of declared output attributes is not with range [0;25]. - */ - public SpoutWrapper(final IRichSpout spout) throws IllegalArgumentException { - this(spout, (Collection) null, null); - } - - /** - * Instantiates a new {@link SpoutWrapper} that calls the {@link IRichSpout#nextTuple() nextTuple()} method of - * the given {@link IRichSpout spout} a finite number of times. The output type will be one of {@link Tuple0} to - * {@link Tuple25} depending on the spout's declared number of attributes. - * - * @param spout - * The {@link IRichSpout spout} to be used. - * @param numberOfInvocations - * The number of calls to {@link IRichSpout#nextTuple()}. If value is negative, {@link SpoutWrapper} - * terminates if no tuple was emitted for the first time. If value is {@code null}, finite invocation is - * disabled. - * @throws IllegalArgumentException - * If the number of declared output attributes is not with range [0;25]. - */ - public SpoutWrapper(final IRichSpout spout, final Integer numberOfInvocations) - throws IllegalArgumentException { - this(spout, (Collection) null, numberOfInvocations); - } - - /** - * Instantiates a new {@link SpoutWrapper} that calls the {@link IRichSpout#nextTuple() nextTuple()} method of - * the given {@link IRichSpout spout} in an infinite loop. The output type can be any type if parameter - * {@code rawOutput} is {@code true} and the spout's number of declared output tuples is 1. If {@code rawOutput} is - * {@code false} the output type will be one of {@link Tuple0} to {@link Tuple25} depending on the spout's declared - * number of attributes. - * - * @param spout - * The {@link IRichSpout spout} to be used. - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type {@link Tuple1} but be - * of a raw type. (Can be {@code null}.) - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not with range - * [0;25]. - */ - public SpoutWrapper(final IRichSpout spout, final String[] rawOutputs) - throws IllegalArgumentException { - this(spout, asList(rawOutputs), null); - } - - /** - * Instantiates a new {@link SpoutWrapper} that calls the {@link IRichSpout#nextTuple() nextTuple()} method of - * the given {@link IRichSpout spout} a finite number of times. The output type can be any type if parameter - * {@code rawOutput} is {@code true} and the spout's number of declared output tuples is 1. If {@code rawOutput} is - * {@code false} the output type will be one of {@link Tuple0} to {@link Tuple25} depending on the spout's declared - * number of attributes. - * - * @param spout - * The {@link IRichSpout spout} to be used. - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type {@link Tuple1} but be - * of a raw type. (Can be {@code null}.) - * @param numberOfInvocations - * The number of calls to {@link IRichSpout#nextTuple()}. If value is negative, {@link SpoutWrapper} - * terminates if no tuple was emitted for the first time. If value is {@code null}, finite invocation is - * disabled. - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not with range - * [0;25]. - */ - public SpoutWrapper(final IRichSpout spout, final String[] rawOutputs, - final Integer numberOfInvocations) throws IllegalArgumentException { - this(spout, asList(rawOutputs), numberOfInvocations); - } - - /** - * Instantiates a new {@link SpoutWrapper} that calls the {@link IRichSpout#nextTuple() nextTuple()} method of - * the given {@link IRichSpout spout} in an infinite loop. The output type can be any type if parameter - * {@code rawOutput} is {@code true} and the spout's number of declared output tuples is 1. If {@code rawOutput} is - * {@code false} the output type will be one of {@link Tuple0} to {@link Tuple25} depending on the spout's declared - * number of attributes. - * - * @param spout - * The {@link IRichSpout spout} to be used. - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type {@link Tuple1} but be - * of a raw type. (Can be {@code null}.) - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not with range - * [0;25]. - */ - public SpoutWrapper(final IRichSpout spout, final Collection rawOutputs) - throws IllegalArgumentException { - this(spout, rawOutputs, null); - } - - /** - * Instantiates a new {@link SpoutWrapper} that calls the {@link IRichSpout#nextTuple() nextTuple()} method of - * the given {@link IRichSpout spout} a finite number of times. The output type can be any type if parameter - * {@code rawOutput} is {@code true} and the spout's number of declared output tuples is 1. If {@code rawOutput} is - * {@code false} the output type will be one of {@link Tuple0} to {@link Tuple25} depending on the spout's declared - * number of attributes. - * - * @param spout - * The {@link IRichSpout spout} to be used. - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type {@link Tuple1} but be - * of a raw type. (Can be {@code null}.) - * @param numberOfInvocations - * The number of calls to {@link IRichSpout#nextTuple()}. If value is negative, {@link SpoutWrapper} - * terminates if no tuple was emitted for the first time. If value is {@code null}, finite invocation is - * disabled. - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not with range - * [0;25]. - */ - public SpoutWrapper(final IRichSpout spout, final Collection rawOutputs, - final Integer numberOfInvocations) throws IllegalArgumentException { - this(spout, null, rawOutputs, numberOfInvocations); - } - - /** - * Instantiates a new {@link SpoutWrapper} that calls the {@link IRichSpout#nextTuple() nextTuple()} method of - * the given {@link IRichSpout spout} a finite number of times. The output type can be any type if parameter - * {@code rawOutput} is {@code true} and the spout's number of declared output tuples is 1. If {@code rawOutput} is - * {@code false} the output type will be one of {@link Tuple0} to {@link Tuple25} depending on the spout's declared - * number of attributes. - * - * @param spout - * The {@link IRichSpout spout} to be used. - * @param name - * The name of the spout. - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type {@link Tuple1} but be - * of a raw type. (Can be {@code null}.) - * @param numberOfInvocations - * The number of calls to {@link IRichSpout#nextTuple()}. If value is negative, {@link SpoutWrapper} - * terminates if no tuple was emitted for the first time. If value is {@code null}, finite invocation is - * disabled. - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not with range - * [0;25]. - */ - public SpoutWrapper(final IRichSpout spout, final String name, final Collection rawOutputs, - final Integer numberOfInvocations) throws IllegalArgumentException { - this.spout = spout; - this.name = name; - this.numberOfAttributes = WrapperSetupHelper.getNumberOfAttributes(spout, rawOutputs); - this.numberOfInvocations = numberOfInvocations; - } - - @Override - public final void run(final SourceContext ctx) throws Exception { - final GlobalJobParameters config = super.getRuntimeContext().getExecutionConfig() - .getGlobalJobParameters(); - StormConfig stormConfig = new StormConfig(); - - if (config != null) { - if (config instanceof StormConfig) { - stormConfig = (StormConfig) config; - } else { - stormConfig.putAll(config.toMap()); - } - } - - final TopologyContext stormTopologyContext = WrapperSetupHelper.createTopologyContext( - (StreamingRuntimeContext) super.getRuntimeContext(), this.spout, this.name, - stormConfig); - - SpoutCollector collector = new SpoutCollector(this.numberOfAttributes, - stormTopologyContext.getThisTaskId(), ctx); - - this.spout.open(stormConfig, stormTopologyContext, new SpoutOutputCollector(collector)); - this.spout.activate(); - - if (numberOfInvocations == null) { - if (this.spout instanceof FiniteSpout) { - final FiniteSpout finiteSpout = (FiniteSpout) this.spout; - - while (this.isRunning && !finiteSpout.reachedEnd()) { - finiteSpout.nextTuple(); - } - } else { - while (this.isRunning) { - this.spout.nextTuple(); - } - } - } else { - int counter = this.numberOfInvocations; - if (counter >= 0) { - while ((--counter >= 0) && this.isRunning) { - this.spout.nextTuple(); - } - } else { - do { - collector.tupleEmitted = false; - this.spout.nextTuple(); - } while (collector.tupleEmitted && this.isRunning); - } - } - } - - /** - * {@inheritDoc} - * - *

Sets the {@link #isRunning} flag to {@code false}. - */ - @Override - public void cancel() { - this.isRunning = false; - } - - /** - * {@inheritDoc} - * - *

Sets the {@link #isRunning} flag to {@code false}. - */ - @Override - public void stop() { - this.isRunning = false; - } - - @Override - public void close() throws Exception { - this.spout.close(); - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/StormTuple.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/StormTuple.java deleted file mode 100644 index a1d33e80baa789..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/StormTuple.java +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -/* - * We do neither import - * org.apache.storm.tuple.Tuple; - * nor - * org.apache.flink.api.java.tuple.Tuple - * to avoid confusion - */ - -import org.apache.storm.generated.GlobalStreamId; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.MessageId; -import org.apache.storm.tuple.Values; - -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.util.List; - -/** - * {@link StormTuple} converts a Flink tuple of type {@code IN} into a Storm tuple. - */ -public class StormTuple implements org.apache.storm.tuple.Tuple { - - /** The Storm representation of the original Flink tuple. */ - private final Values stormTuple; - /** The schema (ie, ordered field names) of this tuple. */ - private final Fields schema; - /** The task ID where this tuple was produced. */ - private final int producerTaskId; - /** The input stream from which this tuple was received. */ - private final String producerStreamId; - /** The producer's component ID of this tuple. */ - private final String producerComponentId; - /** The message that is associated with this tuple. */ - private final MessageId messageId; - - /** - * Create a new Storm tuple from the given Flink tuple. - * - * @param flinkTuple - * The Flink tuple to be converted. - * @param schema - * The schema (ie, ordered field names) of the tuple. - * @param producerTaskId - * The task ID of the producer (a valid, ie, non-negative ID, implicates the truncation of the last - * attribute of {@code flinkTuple}). - * @param producerStreamId - * The input stream ID from which this tuple was received. - * @param producerComponentId - * The component ID of the producer. - * @param messageId - * The message ID of this tuple. - */ - public StormTuple(final IN flinkTuple, final Fields schema, final int producerTaskId, final String producerStreamId, final String producerComponentId, final MessageId messageId) { - if (flinkTuple instanceof org.apache.flink.api.java.tuple.Tuple) { - final org.apache.flink.api.java.tuple.Tuple t = (org.apache.flink.api.java.tuple.Tuple) flinkTuple; - - final int numberOfAttributes; - // does flinkTuple carry producerTaskId as last attribute? - if (producerTaskId < 0) { - numberOfAttributes = t.getArity(); - } else { - numberOfAttributes = t.getArity() - 1; - } - this.stormTuple = new Values(); - for (int i = 0; i < numberOfAttributes; ++i) { - this.stormTuple.add(t.getField(i)); - } - } else { - this.stormTuple = new Values(flinkTuple); - } - - this.schema = schema; - this.producerTaskId = producerTaskId; - this.producerStreamId = producerStreamId; - this.producerComponentId = producerComponentId; - this.messageId = messageId; - } - - @Override - public int size() { - return this.stormTuple.size(); - } - - @Override - public boolean contains(final String field) { - if (this.schema != null) { - return this.schema.contains(field); - } - - try { - this.getPublicMemberField(field); - return true; - } catch (NoSuchFieldException f) { - try { - this.getGetterMethod(field); - return true; - } catch (Exception g) { - return false; - } - } catch (Exception e) { - return false; - } - } - - @Override - public Fields getFields() { - return this.schema; - } - - @Override - public int fieldIndex(final String field) { - return this.schema.fieldIndex(field); - } - - @Override - public List select(final Fields selector) { - return this.schema.select(selector, this.stormTuple); - } - - @Override - public Object getValue(final int i) { - return this.stormTuple.get(i); - } - - @Override - public String getString(final int i) { - return (String) this.stormTuple.get(i); - } - - @Override - public Integer getInteger(final int i) { - return (Integer) this.stormTuple.get(i); - } - - @Override - public Long getLong(final int i) { - return (Long) this.stormTuple.get(i); - } - - @Override - public Boolean getBoolean(final int i) { - return (Boolean) this.stormTuple.get(i); - } - - @Override - public Short getShort(final int i) { - return (Short) this.stormTuple.get(i); - } - - @Override - public Byte getByte(final int i) { - return (Byte) this.stormTuple.get(i); - } - - @Override - public Double getDouble(final int i) { - return (Double) this.stormTuple.get(i); - } - - @Override - public Float getFloat(final int i) { - return (Float) this.stormTuple.get(i); - } - - @Override - public byte[] getBinary(final int i) { - return (byte[]) this.stormTuple.get(i); - } - - private Field getPublicMemberField(final String field) throws Exception { - assert (this.stormTuple.size() == 1); - return this.stormTuple.get(0).getClass().getField(field); - } - - private Method getGetterMethod(final String field) throws Exception { - assert (this.stormTuple.size() == 1); - return this.stormTuple - .get(0) - .getClass() - .getMethod("get" + Character.toUpperCase(field.charAt(0)) + field.substring(1), - (Class[]) null); - } - - private Object getValueByPublicMember(final String field) throws Exception { - assert (this.stormTuple.size() == 1); - return getPublicMemberField(field).get(this.stormTuple.get(0)); - } - - private Object getValueByGetter(final String field) throws Exception { - assert (this.stormTuple.size() == 1); - return getGetterMethod(field).invoke(this.stormTuple.get(0), (Object[]) null); - } - - @SuppressWarnings("unchecked") - public T getValueByName(final String field) { - if (this.schema != null) { - return (T) this.getValue(this.schema.fieldIndex(field)); - } - assert (this.stormTuple.size() == 1); - - Exception e; - try { - // try public member - return (T) getValueByPublicMember(field); - } catch (NoSuchFieldException f) { - try { - // try getter-method - return (T) getValueByGetter(field); - } catch (Exception g) { - e = g; - } - } catch (Exception f) { - e = f; - } - - throw new RuntimeException("Could not access field <" + field + ">", e); - } - - @Override - public Object getValueByField(final String field) { - return getValueByName(field); - } - - @Override - public String getStringByField(final String field) { - return getValueByName(field); - } - - @Override - public Integer getIntegerByField(final String field) { - return getValueByName(field); - } - - @Override - public Long getLongByField(final String field) { - return getValueByName(field); - } - - @Override - public Boolean getBooleanByField(final String field) { - return getValueByName(field); - } - - @Override - public Short getShortByField(final String field) { - return getValueByName(field); - } - - @Override - public Byte getByteByField(final String field) { - return getValueByName(field); - } - - @Override - public Double getDoubleByField(final String field) { - return getValueByName(field); - } - - @Override - public Float getFloatByField(final String field) { - return getValueByName(field); - } - - @Override - public byte[] getBinaryByField(final String field) { - return getValueByName(field); - } - - @Override - public List getValues() { - return this.stormTuple; - } - - @Override - public GlobalStreamId getSourceGlobalStreamid() { - return new GlobalStreamId(this.producerComponentId, this.producerStreamId); - } - - @Override - public String getSourceComponent() { - return this.producerComponentId; - } - - @Override - public int getSourceTask() { - return this.producerTaskId; - } - - @Override - public String getSourceStreamId() { - return this.producerStreamId; - } - - @Override - public MessageId getMessageId() { - return this.messageId; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((messageId == null) ? 0 : messageId.hashCode()); - result = prime * result - + ((producerComponentId == null) ? 0 : producerComponentId.hashCode()); - result = prime * result + ((producerStreamId == null) ? 0 : producerStreamId.hashCode()); - result = prime * result + producerTaskId; - result = prime * result + ((schema == null) ? 0 : schema.toList().hashCode()); - result = prime * result + ((stormTuple == null) ? 0 : stormTuple.hashCode()); - return result; - } - - @SuppressWarnings("rawtypes") - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - StormTuple other = (StormTuple) obj; - if (messageId == null) { - if (other.messageId != null) { - return false; - } - } else if (!messageId.equals(other.messageId)) { - return false; - } - if (producerComponentId == null) { - if (other.producerComponentId != null) { - return false; - } - } else if (!producerComponentId.equals(other.producerComponentId)) { - return false; - } - if (producerStreamId == null) { - if (other.producerStreamId != null) { - return false; - } - } else if (!producerStreamId.equals(other.producerStreamId)) { - return false; - } - if (producerTaskId != other.producerTaskId) { - return false; - } - if (schema == null) { - if (other.schema != null) { - return false; - } - } else if (!schema.toList().equals(other.schema.toList())) { - return false; - } - if (stormTuple == null) { - if (other.stormTuple != null) { - return false; - } - } else if (!stormTuple.equals(other.stormTuple)) { - return false; - } - return true; - } - - @Override - public String toString() { - return "StormTuple{ " + stormTuple.toString() + "[" + this.producerComponentId + "," - + this.producerStreamId + "," + this.producerTaskId + "," + this.messageId + "]}"; - } - - @Override - public GlobalStreamId getSourceGlobalStreamId() { - return new GlobalStreamId(this.producerComponentId, this.producerStreamId); - } - -} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/WrapperSetupHelper.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/WrapperSetupHelper.java deleted file mode 100644 index 03af852ac53e4f..00000000000000 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/WrapperSetupHelper.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; - -import clojure.lang.Atom; -import org.apache.storm.Config; -import org.apache.storm.generated.Bolt; -import org.apache.storm.generated.ComponentCommon; -import org.apache.storm.generated.SpoutSpec; -import org.apache.storm.generated.StateSpoutSpec; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IComponent; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.tuple.Fields; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -/** - * {@link WrapperSetupHelper} is an helper class used by {@link SpoutWrapper} and - * {@link BoltWrapper}. - */ -class WrapperSetupHelper { - - /** The configuration key for the topology name. */ - static final String TOPOLOGY_NAME = "storm.topology.name"; - - /** - * Computes the number of output attributes used by a {@link SpoutWrapper} or {@link BoltWrapper} - * per declared output stream. The number is {@code -1} for raw output type or a value within range [0;25] for - * output type {@link org.apache.flink.api.java.tuple.Tuple0 Tuple0} to - * {@link org.apache.flink.api.java.tuple.Tuple25 Tuple25}. - * - * @param spoutOrBolt - * The Storm {@link IRichSpout spout} or {@link IRichBolt bolt} to be used. - * @param rawOutputs - * Contains stream names if a single attribute output stream, should not be of type - * {@link org.apache.flink.api.java.tuple.Tuple1 Tuple1} but be of a raw type. (Can be {@code null}.) - * @return The number of attributes to be used for each stream. - * @throws IllegalArgumentException - * If {@code rawOutput} is {@code true} and the number of declared output attributes is not 1 or if - * {@code rawOutput} is {@code false} and the number of declared output attributes is not with range - * [0;25]. - */ - static HashMap getNumberOfAttributes(final IComponent spoutOrBolt, - final Collection rawOutputs) - throws IllegalArgumentException { - final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - spoutOrBolt.declareOutputFields(declarer); - - for (Entry schema : declarer.outputSchemas.entrySet()) { - int declaredNumberOfAttributes = schema.getValue(); - if ((declaredNumberOfAttributes < 0) || (declaredNumberOfAttributes > 25)) { - throw new IllegalArgumentException( - "Provided bolt declares non supported number of output attributes. Must be in range [0;25] but " - + "was " + declaredNumberOfAttributes); - } - - if (rawOutputs != null && rawOutputs.contains(schema.getKey())) { - if (declaredNumberOfAttributes != 1) { - throw new IllegalArgumentException( - "Ouput type is requested to be raw type, but provided bolt declares more then one output " - + "attribute."); - } - schema.setValue(-1); - } - } - - return declarer.outputSchemas; - } - - /** - * Creates a {@link TopologyContext} for a Spout or Bolt instance (ie, Flink task / Storm executor). - * - * @param context - * The Flink runtime context. - * @param spoutOrBolt - * The Spout or Bolt this context is created for. - * @param stormConfig - * The user provided configuration. - * @return The created {@link TopologyContext}. - */ - @SuppressWarnings({ "rawtypes", "unchecked" }) - static synchronized TopologyContext createTopologyContext( - final StreamingRuntimeContext context, final IComponent spoutOrBolt, - final String operatorName, final Map stormConfig) { - - final int dop = context.getNumberOfParallelSubtasks(); - - final Map taskToComponents = new HashMap(); - final Map> componentToSortedTasks = new HashMap>(); - final Map> componentToStreamToFields = new HashMap>(); - String stormId = (String) stormConfig.get(TOPOLOGY_NAME); - String codeDir = null; // not supported - String pidDir = null; // not supported - Integer taskId = -1; - Integer workerPort = null; // not supported - List workerTasks = new ArrayList(); - final Map defaultResources = new HashMap(); - final Map userResources = new HashMap(); - final Map executorData = new HashMap(); - final Map registeredMetrics = new HashMap(); - Atom openOrPrepareWasCalled = null; - - ComponentCommon common = new ComponentCommon(); - common.set_parallelism_hint(dop); - - HashMap spouts = new HashMap(); - HashMap bolts = new HashMap(); - if (spoutOrBolt instanceof IRichSpout) { - spouts.put(operatorName, new SpoutSpec(null, common)); - } else { - assert (spoutOrBolt instanceof IRichBolt); - bolts.put(operatorName, new Bolt(null, common)); - } - StormTopology stormTopology = new StormTopology(spouts, bolts, new HashMap()); - - List sortedTasks = new ArrayList(dop); - for (int i = 1; i <= dop; ++i) { - taskToComponents.put(i, operatorName); - sortedTasks.add(i); - } - componentToSortedTasks.put(operatorName, sortedTasks); - - SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - spoutOrBolt.declareOutputFields(declarer); - componentToStreamToFields.put(operatorName, declarer.outputStreams); - - if (!stormConfig.containsKey(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)) { - stormConfig.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 30); // Storm default value - } - - return new FlinkTopologyContext(stormTopology, stormConfig, taskToComponents, - componentToSortedTasks, componentToStreamToFields, stormId, codeDir, pidDir, - taskId, workerPort, workerTasks, defaultResources, userResources, executorData, - registeredMetrics, openOrPrepareWasCalled); - } -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/AbstractTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/AbstractTest.java deleted file mode 100644 index ca0e06755e6d5a..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/AbstractTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.junit.Before; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Random; - -/** - * Abstract class for all tests that require a {@link Random} to be setup before each test. - */ -public abstract class AbstractTest { - private static final Logger LOG = LoggerFactory.getLogger(AbstractTest.class); - - protected long seed; - protected Random r; - - @Before - public void prepare() { - this.seed = System.currentTimeMillis(); - this.r = new Random(this.seed); - LOG.info("Test seed: {}", new Long(this.seed)); - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/FiniteTestSpout.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/FiniteTestSpout.java deleted file mode 100644 index 8b89c95161132c..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/FiniteTestSpout.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Values; - -import java.util.Map; - -/** - * Tests for the Finite. - */ -public class FiniteTestSpout implements IRichSpout { - private static final long serialVersionUID = 7992419478267824279L; - - private int numberOfOutputTuples; - private SpoutOutputCollector collector; - - public FiniteTestSpout(final int numberOfOutputTuples) { - this.numberOfOutputTuples = numberOfOutputTuples; - } - - @SuppressWarnings("rawtypes") - @Override - public void open(final Map conf, final TopologyContext context, final SpoutOutputCollector collector) { - this.collector = collector; - } - - @Override - public void close() {/* nothing to do */} - - @Override - public void activate() {/* nothing to do */} - - @Override - public void deactivate() {/* nothing to do */} - - @Override - public void nextTuple() { - if (--this.numberOfOutputTuples >= 0) { - this.collector.emit(new Values(new Integer(this.numberOfOutputTuples))); - } - } - - @Override - public void ack(final Object msgId) {/* nothing to do */} - - @Override - public void fail(final Object msgId) {/* nothing to do */} - - @Override - public void declareOutputFields(final OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("dummy")); - } - - @Override - public Map getComponentConfiguration() { - return null; - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/NullTerminatingSpoutTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/NullTerminatingSpoutTest.java deleted file mode 100644 index 7263ce4a58ace9..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/NullTerminatingSpoutTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; - -import static org.mockito.Matchers.same; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * Tests for the NullTerminatingSpout. - */ -public class NullTerminatingSpoutTest { - - @Test - public void testMethodCalls() { - Map compConfig = new HashMap(); - - IRichSpout spoutMock = mock(IRichSpout.class); - when(spoutMock.getComponentConfiguration()).thenReturn(compConfig); - - Map conf = mock(Map.class); - TopologyContext context = mock(TopologyContext.class); - Object msgId = mock(Object.class); - OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); - - NullTerminatingSpout spout = new NullTerminatingSpout(spoutMock); - - spout.open(conf, context, null); - spout.close(); - spout.activate(); - spout.deactivate(); - spout.ack(msgId); - spout.fail(msgId); - spout.declareOutputFields(declarer); - Map c = spoutMock.getComponentConfiguration(); - - verify(spoutMock).open(same(conf), same(context), any(SpoutOutputCollector.class)); - verify(spoutMock).close(); - verify(spoutMock).activate(); - verify(spoutMock).deactivate(); - verify(spoutMock).ack(same(msgId)); - verify(spoutMock).fail(same(msgId)); - verify(spoutMock).declareOutputFields(same(declarer)); - Assert.assertSame(compConfig, c); - } - - @Test - public void testReachedEnd() { - NullTerminatingSpout finiteSpout = new NullTerminatingSpout(new TestDummySpout()); - finiteSpout.open(null, null, mock(SpoutOutputCollector.class)); - - Assert.assertFalse(finiteSpout.reachedEnd()); - - finiteSpout.nextTuple(); - Assert.assertFalse(finiteSpout.reachedEnd()); - finiteSpout.nextTuple(); - Assert.assertTrue(finiteSpout.reachedEnd()); - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/SpoutOutputCollectorObserverTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/SpoutOutputCollectorObserverTest.java deleted file mode 100644 index c150cc3c8f243b..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/SpoutOutputCollectorObserverTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.junit.Assert; -import org.junit.Test; - -import static org.mockito.Mockito.mock; - -/** - * Tests for the SpoutOutputCollectorObserver. - */ -public class SpoutOutputCollectorObserverTest { - - @Test - public void testFlag() { - SpoutOutputCollectorObserver observer = new SpoutOutputCollectorObserver(mock(SpoutOutputCollector.class)); - - observer.emitted = false; - observer.emit(null); - Assert.assertTrue(observer.emitted); - - observer.emitted = false; - observer.emit(null, (Object) null); - Assert.assertTrue(observer.emitted); - - observer.emitted = false; - observer.emit((String) null, null); - Assert.assertTrue(observer.emitted); - - observer.emitted = false; - observer.emit(null, null, null); - Assert.assertTrue(observer.emitted); - - observer.emitted = false; - observer.emitDirect(0, null); - Assert.assertTrue(observer.emitted); - - observer.emitted = false; - observer.emitDirect(0, null, (Object) null); - Assert.assertTrue(observer.emitted); - - observer.emitted = false; - observer.emitDirect(0, (String) null, null); - Assert.assertTrue(observer.emitted); - - observer.emitted = false; - observer.emitDirect(0, null, null, null); - Assert.assertTrue(observer.emitted); - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/StormStreamSelectorTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/StormStreamSelectorTest.java deleted file mode 100644 index 67d4a17e423223..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/StormStreamSelectorTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.junit.Assert; -import org.junit.Test; - -import java.util.Iterator; - -/** - * Tests for the StormStreamSelector. - */ -public class StormStreamSelectorTest { - - @Test - public void testSelector() { - StormStreamSelector selector = new StormStreamSelector(); - SplitStreamType tuple = new SplitStreamType(); - Iterator result; - - tuple.streamId = "stream1"; - result = selector.select(tuple).iterator(); - Assert.assertEquals("stream1", result.next()); - Assert.assertFalse(result.hasNext()); - - tuple.streamId = "stream2"; - result = selector.select(tuple).iterator(); - Assert.assertEquals("stream2", result.next()); - Assert.assertFalse(result.hasNext()); - - tuple.streamId = "stream1"; - result = selector.select(tuple).iterator(); - Assert.assertEquals("stream1", result.next()); - Assert.assertFalse(result.hasNext()); - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummyBolt.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummyBolt.java deleted file mode 100644 index 277369285716b7..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummyBolt.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -import java.util.Map; - -/** - * A test implementation of a {@link IRichBolt}. - */ -public class TestDummyBolt implements IRichBolt { - private static final long serialVersionUID = 6893611247443121322L; - - public static final String SHUFFLE_STREAM_ID = "shuffleStream"; - public static final String GROUPING_STREAM_ID = "groupingStream"; - - private boolean emit = true; - @SuppressWarnings("rawtypes") - public Map config; - private TopologyContext context; - private OutputCollector collector; - - @SuppressWarnings("rawtypes") - @Override - public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { - this.config = stormConf; - this.context = context; - this.collector = collector; - } - - @Override - public void execute(Tuple input) { - if (this.context.getThisTaskIndex() == 0) { - this.collector.emit(SHUFFLE_STREAM_ID, input.getValues()); - } - if (this.emit) { - this.collector.emit(GROUPING_STREAM_ID, new Values("bolt", this.context)); - this.emit = false; - } - } - - @Override - public void cleanup() {} - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declareStream(SHUFFLE_STREAM_ID, new Fields("data")); - declarer.declareStream(GROUPING_STREAM_ID, new Fields("id", "data")); - } - - @Override - public Map getComponentConfiguration() { - return null; - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummySpout.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummySpout.java deleted file mode 100644 index 5ff8289dad638b..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummySpout.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Values; -import org.apache.storm.utils.Utils; - -import java.util.Map; - -/** - * A test implementation of a {@link IRichSpout}. - */ -public class TestDummySpout implements IRichSpout { - private static final long serialVersionUID = -5190945609124603118L; - - public static final String SPOUT_STREAM_ID = "spout-stream"; - - private boolean emit = true; - @SuppressWarnings("rawtypes") - public Map config; - private TopologyContext context; - private SpoutOutputCollector collector; - - @SuppressWarnings("rawtypes") - @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { - this.config = conf; - this.context = context; - this.collector = collector; - } - - @Override - public void close() {} - - @Override - public void activate() {} - - @Override - public void deactivate() {} - - @Override - public void nextTuple() { - if (this.emit) { - this.collector.emit(new Values(this.context)); - this.emit = false; - } - } - - @Override - public void ack(Object msgId) {} - - @Override - public void fail(Object msgId) {} - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declareStream(Utils.DEFAULT_STREAM_ID, new Fields("data")); - declarer.declareStream(SPOUT_STREAM_ID, new Fields("id", "data")); - } - - @Override - public Map getComponentConfiguration() { - return null; - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestSink.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestSink.java deleted file mode 100644 index a3bb884788ae95..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestSink.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.util; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Tuple; - -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -/** - * A test implementation of a {@link IRichBolt} that stores incoming records in {@link #RESULT}. - */ -public class TestSink implements IRichBolt { - private static final long serialVersionUID = 4314871456719370877L; - - public static final List RESULT = new LinkedList(); - - @SuppressWarnings("rawtypes") - @Override - public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { - RESULT.add(context); - } - - @Override - public void execute(Tuple input) { - if (input.size() == 1) { - RESULT.add((TopologyContext) input.getValue(0)); - } else { - RESULT.add((TopologyContext) input.getValue(1)); - } - } - - @Override - public void cleanup() {} - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) {} - - @Override - public Map getComponentConfiguration() { - return null; - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltCollectorTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltCollectorTest.java deleted file mode 100644 index d48042b72449b8..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltCollectorTest.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.api.java.tuple.Tuple; -import org.apache.flink.storm.util.AbstractTest; -import org.apache.flink.streaming.api.operators.Output; - -import org.apache.storm.tuple.Values; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Collection; -import java.util.HashMap; -import java.util.List; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -/** - * Tests for the BoltCollector. - */ -public class BoltCollectorTest extends AbstractTest { - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Test - public void testBoltStormCollector() throws InstantiationException, IllegalAccessException { - for (int numberOfAttributes = -1; numberOfAttributes < 26; ++numberOfAttributes) { - final Output flinkCollector = mock(Output.class); - Tuple flinkTuple = null; - final Values tuple = new Values(); - - BoltCollector collector; - - final String streamId = "streamId"; - HashMap attributes = new HashMap(); - attributes.put(streamId, numberOfAttributes); - - if (numberOfAttributes == -1) { - collector = new BoltCollector(attributes, -1, flinkCollector); - tuple.add(new Integer(this.r.nextInt())); - - } else { - collector = new BoltCollector(attributes, -1, flinkCollector); - flinkTuple = Tuple.getTupleClass(numberOfAttributes).newInstance(); - - for (int i = 0; i < numberOfAttributes; ++i) { - tuple.add(new Integer(this.r.nextInt())); - flinkTuple.setField(tuple.get(i), i); - } - } - - final Collection anchors = mock(Collection.class); - final List taskIds; - taskIds = collector.emit(streamId, anchors, tuple); - - Assert.assertNull(taskIds); - - if (numberOfAttributes == -1) { - verify(flinkCollector).collect(tuple.get(0)); - } else { - verify(flinkCollector).collect(flinkTuple); - } - } - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Test - public void testBoltStormCollectorWithTaskId() throws InstantiationException, IllegalAccessException { - for (int numberOfAttributes = 0; numberOfAttributes < 25; ++numberOfAttributes) { - final Output flinkCollector = mock(Output.class); - final int taskId = 42; - final String streamId = "streamId"; - - HashMap attributes = new HashMap(); - attributes.put(streamId, numberOfAttributes); - - BoltCollector collector = new BoltCollector(attributes, taskId, flinkCollector); - - final Values tuple = new Values(); - final Tuple flinkTuple = Tuple.getTupleClass(numberOfAttributes + 1).newInstance(); - - for (int i = 0; i < numberOfAttributes; ++i) { - tuple.add(new Integer(this.r.nextInt())); - flinkTuple.setField(tuple.get(i), i); - } - flinkTuple.setField(taskId, numberOfAttributes); - - final Collection anchors = mock(Collection.class); - final List taskIds; - taskIds = collector.emit(streamId, anchors, tuple); - - Assert.assertNull(taskIds); - - verify(flinkCollector).collect(flinkTuple); - } - } - - @SuppressWarnings("unchecked") - @Test(expected = UnsupportedOperationException.class) - public void testToManyAttributes() { - HashMap attributes = new HashMap(); - attributes.put("", 26); - - new BoltCollector(attributes, -1, mock(Output.class)); - } - - @SuppressWarnings("unchecked") - @Test(expected = UnsupportedOperationException.class) - public void testToManyAttributesWithTaskId() { - HashMap attributes = new HashMap(); - attributes.put("", 25); - - new BoltCollector(attributes, 42, mock(Output.class)); - } - - @SuppressWarnings("unchecked") - @Test(expected = UnsupportedOperationException.class) - public void testRawStreamWithTaskId() { - HashMap attributes = new HashMap(); - attributes.put("", -1); - - new BoltCollector(attributes, 42, mock(Output.class)); - } - - @SuppressWarnings("unchecked") - @Test(expected = UnsupportedOperationException.class) - public void testEmitDirect() { - new BoltCollector(mock(HashMap.class), -1, mock(Output.class)).emitDirect(0, null, - null, null); - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java deleted file mode 100644 index d405a45f6af48a..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java +++ /dev/null @@ -1,389 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.TaskInfo; -import org.apache.flink.api.java.tuple.Tuple; -import org.apache.flink.api.java.tuple.Tuple1; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.UnmodifiableConfiguration; -import org.apache.flink.core.fs.CloseableRegistry; -import org.apache.flink.metrics.groups.UnregisteredMetricsGroup; -import org.apache.flink.runtime.execution.Environment; -import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; -import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo; -import org.apache.flink.storm.util.AbstractTest; -import org.apache.flink.storm.util.SplitStreamType; -import org.apache.flink.storm.util.StormConfig; -import org.apache.flink.storm.util.TestDummyBolt; -import org.apache.flink.streaming.api.operators.Output; -import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; -import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer; -import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.flink.streaming.runtime.tasks.StreamTask; -import org.apache.flink.streaming.util.MockStreamConfig; - -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.MessageId; -import org.apache.storm.tuple.Values; -import org.apache.storm.utils.Utils; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.util.HashSet; -import java.util.Map; -import java.util.Map.Entry; - -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; -import static org.mockito.Matchers.isNotNull; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.same; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * Tests for the BoltWrapper. - */ -@RunWith(PowerMockRunner.class) -@PrepareForTest({StreamElementSerializer.class, WrapperSetupHelper.class, StreamRecord.class}) -@PowerMockIgnore({"javax.management.*", "com.sun.jndi.*", "org.apache.log4j.*"}) -public class BoltWrapperTest extends AbstractTest { - - @Test(expected = IllegalArgumentException.class) - public void testWrapperRawType() throws Exception { - final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - declarer.declare(new Fields("dummy1", "dummy2")); - PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer); - - new BoltWrapper(mock(IRichBolt.class), new String[] { Utils.DEFAULT_STREAM_ID }); - } - - @Test(expected = IllegalArgumentException.class) - public void testWrapperToManyAttributes1() throws Exception { - final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - final String[] schema = new String[26]; - for (int i = 0; i < schema.length; ++i) { - schema[i] = "a" + i; - } - declarer.declare(new Fields(schema)); - PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer); - - new BoltWrapper(mock(IRichBolt.class)); - } - - @Test(expected = IllegalArgumentException.class) - public void testWrapperToManyAttributes2() throws Exception { - final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - final String[] schema = new String[26]; - for (int i = 0; i < schema.length; ++i) { - schema[i] = "a" + i; - } - declarer.declare(new Fields(schema)); - PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer); - - new BoltWrapper(mock(IRichBolt.class), new String[] {}); - } - - @Test - public void testWrapper() throws Exception { - for (int i = -1; i < 26; ++i) { - this.testWrapper(i); - } - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private void testWrapper(final int numberOfAttributes) throws Exception { - assert ((-1 <= numberOfAttributes) && (numberOfAttributes <= 25)); - Tuple flinkTuple = null; - String rawTuple = null; - - if (numberOfAttributes == -1) { - rawTuple = "test"; - } else { - flinkTuple = Tuple.getTupleClass(numberOfAttributes).newInstance(); - } - - final String[] schema; - if (numberOfAttributes == -1) { - schema = new String[1]; - } else { - schema = new String[numberOfAttributes]; - } - for (int i = 0; i < schema.length; ++i) { - schema[i] = "a" + i; - } - - final StreamRecord record = mock(StreamRecord.class); - if (numberOfAttributes == -1) { - when(record.getValue()).thenReturn(rawTuple); - } else { - when(record.getValue()).thenReturn(flinkTuple); - } - - final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class); - when(taskContext.getExecutionConfig()).thenReturn(mock(ExecutionConfig.class)); - when(taskContext.getTaskName()).thenReturn("name"); - when(taskContext.getMetricGroup()).thenReturn(new UnregisteredMetricsGroup()); - - final IRichBolt bolt = mock(IRichBolt.class); - - final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - declarer.declare(new Fields(schema)); - PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer); - - final BoltWrapper wrapper = new BoltWrapper(bolt, (Fields) null); - wrapper.setup(createMockStreamTask(), new MockStreamConfig(), mock(Output.class)); - wrapper.open(); - - wrapper.processElement(record); - if (numberOfAttributes == -1) { - verify(bolt).execute( - eq(new StormTuple(rawTuple, null, -1, null, null, MessageId - .makeUnanchored()))); - } else { - verify(bolt).execute( - eq(new StormTuple(flinkTuple, null, -1, null, null, MessageId - .makeUnanchored()))); - } - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Test - public void testMultipleOutputStreams() throws Exception { - final boolean rawOutType1 = super.r.nextBoolean(); - final boolean rawOutType2 = super.r.nextBoolean(); - - final StreamRecord record = mock(StreamRecord.class); - when(record.getValue()).thenReturn(2).thenReturn(3); - - final Output output = mock(Output.class); - - final TestBolt bolt = new TestBolt(); - final HashSet raw = new HashSet(); - if (rawOutType1) { - raw.add("stream1"); - } - if (rawOutType2) { - raw.add("stream2"); - } - - final BoltWrapper wrapper = new BoltWrapper(bolt, null, raw); - wrapper.setup(createMockStreamTask(), new MockStreamConfig(), output); - wrapper.open(); - - final SplitStreamType splitRecord = new SplitStreamType(); - if (rawOutType1) { - splitRecord.streamId = "stream1"; - splitRecord.value = 2; - } else { - splitRecord.streamId = "stream1"; - splitRecord.value = new Tuple1(2); - } - wrapper.processElement(record); - verify(output).collect(new StreamRecord(splitRecord)); - - if (rawOutType2) { - splitRecord.streamId = "stream2"; - splitRecord.value = 3; - } else { - splitRecord.streamId = "stream2"; - splitRecord.value = new Tuple1(3); - } - wrapper.processElement(record); - verify(output, times(2)).collect(new StreamRecord(splitRecord)); - } - - @SuppressWarnings("unchecked") - @Test - public void testOpen() throws Exception { - - // utility mocks - final StormConfig stormConfig = new StormConfig(); - final Configuration flinkConfig = new Configuration(); - - final ExecutionConfig taskConfig = mock(ExecutionConfig.class); - when(taskConfig.getGlobalJobParameters()).thenReturn(null).thenReturn(stormConfig) - .thenReturn(flinkConfig); - - final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class); - when(taskContext.getExecutionConfig()).thenReturn(taskConfig); - when(taskContext.getTaskName()).thenReturn("name"); - when(taskContext.getMetricGroup()).thenReturn(new UnregisteredMetricsGroup()); - - final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - declarer.declare(new Fields("dummy")); - PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer); - - // (1) open with no configuration - { - ExecutionConfig execConfig = mock(ExecutionConfig.class); - when(execConfig.getGlobalJobParameters()).thenReturn(null); - - final IRichBolt bolt = mock(IRichBolt.class); - BoltWrapper wrapper = new BoltWrapper(bolt); - wrapper.setup(createMockStreamTask(execConfig), new MockStreamConfig(), mock(Output.class)); - - wrapper.open(); - verify(bolt).prepare(any(Map.class), any(TopologyContext.class), any(OutputCollector.class)); - } - - // (2) open with a storm specific configuration - { - ExecutionConfig execConfig = mock(ExecutionConfig.class); - when(execConfig.getGlobalJobParameters()).thenReturn(stormConfig); - - final IRichBolt bolt = mock(IRichBolt.class); - BoltWrapper wrapper = new BoltWrapper(bolt); - wrapper.setup(createMockStreamTask(execConfig), new MockStreamConfig(), mock(Output.class)); - - wrapper.open(); - verify(bolt).prepare(same(stormConfig), any(TopologyContext.class), any(OutputCollector.class)); - } - - // (3) open with a flink config - { - final Configuration cfg = new Configuration(); - cfg.setString("foo", "bar"); - cfg.setInteger("the end (the int)", Integer.MAX_VALUE); - - ExecutionConfig execConfig = mock(ExecutionConfig.class); - when(execConfig.getGlobalJobParameters()).thenReturn(new UnmodifiableConfiguration(cfg)); - - TestDummyBolt testBolt = new TestDummyBolt(); - BoltWrapper wrapper = new BoltWrapper(testBolt); - wrapper.setup(createMockStreamTask(execConfig), new MockStreamConfig(), mock(Output.class)); - - wrapper.open(); - for (Entry entry : cfg.toMap().entrySet()) { - Assert.assertEquals(entry.getValue(), testBolt.config.get(entry.getKey())); - } - } - } - - @SuppressWarnings("unchecked") - @Test - public void testOpenSink() throws Exception { - final StormConfig stormConfig = new StormConfig(); - final Configuration flinkConfig = new Configuration(); - - final ExecutionConfig taskConfig = mock(ExecutionConfig.class); - when(taskConfig.getGlobalJobParameters()).thenReturn(null).thenReturn(stormConfig) - .thenReturn(flinkConfig); - - final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class); - when(taskContext.getExecutionConfig()).thenReturn(taskConfig); - when(taskContext.getTaskName()).thenReturn("name"); - when(taskContext.getMetricGroup()).thenReturn(new UnregisteredMetricsGroup()); - - final IRichBolt bolt = mock(IRichBolt.class); - BoltWrapper wrapper = new BoltWrapper(bolt); - - wrapper.setup(createMockStreamTask(), new MockStreamConfig(), mock(Output.class)); - wrapper.open(); - - verify(bolt).prepare(any(Map.class), any(TopologyContext.class), isNotNull(OutputCollector.class)); - } - - @SuppressWarnings("unchecked") - @Test - public void testClose() throws Exception { - final IRichBolt bolt = mock(IRichBolt.class); - - final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - declarer.declare(new Fields("dummy")); - PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer); - - final BoltWrapper wrapper = new BoltWrapper(bolt); - - wrapper.setup(createMockStreamTask(), new MockStreamConfig(), mock(Output.class)); - - wrapper.close(); - wrapper.dispose(); - - verify(bolt).cleanup(); - } - - private static final class TestBolt implements IRichBolt { - private static final long serialVersionUID = 7278692872260138758L; - private transient OutputCollector collector; - - @SuppressWarnings("rawtypes") - @Override - public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { - this.collector = collector; - } - - int counter = 0; - @Override - public void execute(org.apache.storm.tuple.Tuple input) { - if (++counter % 2 == 1) { - this.collector.emit("stream1", new Values(input.getInteger(0))); - } else { - this.collector.emit("stream2", new Values(input.getInteger(0))); - } - } - - @Override - public void cleanup() {} - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declareStream("stream1", new Fields("a1")); - declarer.declareStream("stream2", new Fields("a2")); - } - - @Override - public Map getComponentConfiguration() { - return null; - } - } - - public static StreamTask createMockStreamTask() { - return createMockStreamTask(new ExecutionConfig()); - } - - public static StreamTask createMockStreamTask(ExecutionConfig execConfig) { - Environment env = mock(Environment.class); - when(env.getTaskInfo()).thenReturn(new TaskInfo("Mock Task", 1, 0, 1, 0)); - when(env.getUserClassLoader()).thenReturn(BoltWrapperTest.class.getClassLoader()); - when(env.getMetricGroup()).thenReturn(UnregisteredMetricGroups.createUnregisteredTaskMetricGroup()); - when(env.getTaskManagerInfo()).thenReturn(new TestingTaskManagerRuntimeInfo()); - - final CloseableRegistry closeableRegistry = new CloseableRegistry(); - StreamTask mockTask = mock(StreamTask.class); - when(mockTask.getCheckpointLock()).thenReturn(new Object()); - when(mockTask.getConfiguration()).thenReturn(new MockStreamConfig()); - when(mockTask.getEnvironment()).thenReturn(env); - when(mockTask.getExecutionConfig()).thenReturn(execConfig); - when(mockTask.getCancelables()).thenReturn(closeableRegistry); - - return mockTask; - } -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/FlinkTopologyContextTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/FlinkTopologyContextTest.java deleted file mode 100644 index 0ee796b4db6ff6..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/FlinkTopologyContextTest.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.storm.util.AbstractTest; - -import org.apache.storm.generated.Bolt; -import org.apache.storm.generated.SpoutSpec; -import org.apache.storm.generated.StateSpoutSpec; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.metric.api.ICombiner; -import org.apache.storm.metric.api.IMetric; -import org.apache.storm.metric.api.IReducer; -import org.junit.Test; - -import java.util.HashMap; - -/** - * FlinkTopologyContext.getSources(componentId) and FlinkTopologyContext.getTargets(componentId) are not tested here, - * because those are tested in StormWrapperSetupHelperTest. - */ -public class FlinkTopologyContextTest extends AbstractTest { - - @Test(expected = UnsupportedOperationException.class) - public void testAddTaskHook() { - new FlinkTopologyContext(new StormTopology(new HashMap(), - new HashMap(), new HashMap()), null, null, - null, null, null, null, null, null, null, null, null, null, null, null, null) - .addTaskHook(null); - } - - @Test(expected = UnsupportedOperationException.class) - public void testGetHooks() { - new FlinkTopologyContext(new StormTopology(new HashMap(), - new HashMap(), new HashMap()), null, null, - null, null, null, null, null, null, null, null, null, null, null, null, null) - .getHooks(); - } - - @SuppressWarnings("rawtypes") - @Test(expected = UnsupportedOperationException.class) - public void testRegisteredMetric1() { - new FlinkTopologyContext(new StormTopology(new HashMap(), - new HashMap(), new HashMap()), null, null, - null, null, null, null, null, null, null, null, null, null, null, null, null) - .registerMetric(null, (ICombiner) null, 0); - } - - @SuppressWarnings("rawtypes") - @Test(expected = UnsupportedOperationException.class) - public void testRegisteredMetric2() { - new FlinkTopologyContext(new StormTopology(new HashMap(), - new HashMap(), new HashMap()), null, null, - null, null, null, null, null, null, null, null, null, null, null, null, null) - .registerMetric(null, (IReducer) null, 0); - } - - @Test(expected = UnsupportedOperationException.class) - public void testRegisteredMetric3() { - new FlinkTopologyContext(new StormTopology(new HashMap(), - new HashMap(), new HashMap()), null, null, - null, null, null, null, null, null, null, null, null, null, null, null, null) - .registerMetric(null, (IMetric) null, 0); - } - - @Test(expected = UnsupportedOperationException.class) - public void testGetRegisteredMetricByName() { - new FlinkTopologyContext(new StormTopology(new HashMap(), - new HashMap(), new HashMap()), null, null, - null, null, null, null, null, null, null, null, null, null, null, null, null) - .getRegisteredMetricByName(null); - } - - @Test(expected = UnsupportedOperationException.class) - public void testSetAllSubscribedState() { - new FlinkTopologyContext(new StormTopology(new HashMap(), - new HashMap(), new HashMap()), null, null, - null, null, null, null, null, null, null, null, null, null, null, null, null) - .setAllSubscribedState(null); - } - - @Test(expected = UnsupportedOperationException.class) - public void testSetSubscribedState1() { - new FlinkTopologyContext(new StormTopology(new HashMap(), - new HashMap(), new HashMap()), null, null, - null, null, null, null, null, null, null, null, null, null, null, null, null) - .setSubscribedState(null, null); - } - - @Test(expected = UnsupportedOperationException.class) - public void testSetSubscribedState2() { - new FlinkTopologyContext(new StormTopology(new HashMap(), - new HashMap(), new HashMap()), null, null, - null, null, null, null, null, null, null, null, null, null, null, null, null) - .setSubscribedState(null, null, null); - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SetupOutputFieldsDeclarerTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SetupOutputFieldsDeclarerTest.java deleted file mode 100644 index d6575d89d4723c..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SetupOutputFieldsDeclarerTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.storm.util.AbstractTest; - -import org.apache.storm.tuple.Fields; -import org.apache.storm.utils.Utils; -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; - -/** - * Tests for the SetupOutputFieldsDeclarer. - */ -public class SetupOutputFieldsDeclarerTest extends AbstractTest { - - @Test - public void testDeclare() { - final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - - int numberOfAttributes = this.r.nextInt(26); - declarer.declare(createSchema(numberOfAttributes)); - Assert.assertEquals(1, declarer.outputSchemas.size()); - Assert.assertEquals(numberOfAttributes, declarer.outputSchemas.get(Utils.DEFAULT_STREAM_ID) - .intValue()); - - final String sid = "streamId"; - numberOfAttributes = this.r.nextInt(26); - declarer.declareStream(sid, createSchema(numberOfAttributes)); - Assert.assertEquals(2, declarer.outputSchemas.size()); - Assert.assertEquals(numberOfAttributes, declarer.outputSchemas.get(sid).intValue()); - } - - private Fields createSchema(final int numberOfAttributes) { - final ArrayList schema = new ArrayList(numberOfAttributes); - for (int i = 0; i < numberOfAttributes; ++i) { - schema.add("a" + i); - } - return new Fields(schema); - } - - @Test - public void testDeclareDirect() { - new SetupOutputFieldsDeclarer().declare(false, new Fields()); - } - - @Test(expected = UnsupportedOperationException.class) - public void testDeclareDirectFail() { - new SetupOutputFieldsDeclarer().declare(true, new Fields()); - } - - @Test - public void testDeclareStream() { - new SetupOutputFieldsDeclarer().declareStream(Utils.DEFAULT_STREAM_ID, new Fields()); - } - - @Test(expected = IllegalArgumentException.class) - public void testDeclareStreamFail() { - new SetupOutputFieldsDeclarer().declareStream(null, new Fields()); - } - - @Test - public void testDeclareFullStream() { - new SetupOutputFieldsDeclarer().declareStream(Utils.DEFAULT_STREAM_ID, false, new Fields()); - } - - @Test(expected = IllegalArgumentException.class) - public void testDeclareFullStreamFailNonDefaultStream() { - new SetupOutputFieldsDeclarer().declareStream(null, false, new Fields()); - } - - @Test(expected = UnsupportedOperationException.class) - public void testDeclareFullStreamFailDirect() { - new SetupOutputFieldsDeclarer().declareStream(Utils.DEFAULT_STREAM_ID, true, new Fields()); - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutCollectorTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutCollectorTest.java deleted file mode 100644 index b91871aa499c1e..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutCollectorTest.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.api.java.tuple.Tuple; -import org.apache.flink.storm.util.AbstractTest; -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext; - -import org.apache.storm.tuple.Values; -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.List; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -/** - * Tests for the SpoutCollector. - */ -public class SpoutCollectorTest extends AbstractTest { - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Test - public void testSpoutStormCollector() throws InstantiationException, IllegalAccessException { - for (int numberOfAttributes = -1; numberOfAttributes < 26; ++numberOfAttributes) { - final SourceContext flinkCollector = mock(SourceContext.class); - Tuple flinkTuple = null; - final Values tuple = new Values(); - - SpoutCollector collector; - - final String streamId = "streamId"; - HashMap attributes = new HashMap(); - attributes.put(streamId, numberOfAttributes); - - if (numberOfAttributes == -1) { - collector = new SpoutCollector(attributes, -1, flinkCollector); - tuple.add(new Integer(this.r.nextInt())); - } else { - collector = new SpoutCollector(attributes, -1, flinkCollector); - flinkTuple = Tuple.getTupleClass(numberOfAttributes).newInstance(); - - for (int i = 0; i < numberOfAttributes; ++i) { - tuple.add(new Integer(this.r.nextInt())); - flinkTuple.setField(tuple.get(i), i); - } - } - - final List taskIds; - final Object messageId = new Integer(this.r.nextInt()); - - taskIds = collector.emit(streamId, tuple, messageId); - - Assert.assertNull(taskIds); - - if (numberOfAttributes == -1) { - verify(flinkCollector).collect(tuple.get(0)); - } else { - verify(flinkCollector).collect(flinkTuple); - } - } - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Test - public void testSpoutStormCollectorWithTaskId() throws InstantiationException, IllegalAccessException { - for (int numberOfAttributes = 0; numberOfAttributes < 25; ++numberOfAttributes) { - final SourceContext flinkCollector = mock(SourceContext.class); - final int taskId = 42; - final String streamId = "streamId"; - - HashMap attributes = new HashMap(); - attributes.put(streamId, numberOfAttributes); - - SpoutCollector collector = new SpoutCollector(attributes, taskId, flinkCollector); - - final Values tuple = new Values(); - final Tuple flinkTuple = Tuple.getTupleClass(numberOfAttributes + 1).newInstance(); - - for (int i = 0; i < numberOfAttributes; ++i) { - tuple.add(new Integer(this.r.nextInt())); - flinkTuple.setField(tuple.get(i), i); - } - flinkTuple.setField(taskId, numberOfAttributes); - - final List taskIds; - final Object messageId = new Integer(this.r.nextInt()); - - taskIds = collector.emit(streamId, tuple, messageId); - - Assert.assertNull(taskIds); - - verify(flinkCollector).collect(flinkTuple); - } - } - - @SuppressWarnings("unchecked") - @Test(expected = UnsupportedOperationException.class) - public void testTooManyAttributes() { - HashMap attributes = new HashMap(); - attributes.put("", 26); - - new SpoutCollector(attributes, -1, mock(SourceContext.class)); - } - - @SuppressWarnings("unchecked") - @Test(expected = UnsupportedOperationException.class) - public void testTooManyAttributesWithTaskId() { - HashMap attributes = new HashMap(); - attributes.put("", 25); - - new SpoutCollector(attributes, 42, mock(SourceContext.class)); - } - - @SuppressWarnings("unchecked") - @Test(expected = UnsupportedOperationException.class) - public void testRawStreamWithTaskId() { - HashMap attributes = new HashMap(); - attributes.put("", -1); - - new SpoutCollector(attributes, 42, mock(SourceContext.class)); - } - - @SuppressWarnings("unchecked") - @Test(expected = UnsupportedOperationException.class) - public void testEmitDirect() { - new SpoutCollector(mock(HashMap.class), -1, mock(SourceContext.class)).emitDirect( - 0, null, null, (Object) null); - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutWrapperTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutWrapperTest.java deleted file mode 100644 index e6d861bdeb0e84..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutWrapperTest.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.java.tuple.Tuple1; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.storm.util.AbstractTest; -import org.apache.flink.storm.util.FiniteSpout; -import org.apache.flink.storm.util.FiniteTestSpout; -import org.apache.flink.storm.util.StormConfig; -import org.apache.flink.storm.util.TestDummySpout; -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext; -import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.tuple.Fields; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.util.LinkedList; -import java.util.Map; -import java.util.Map.Entry; - -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * Tests for the SpoutWrapper. - */ -@RunWith(PowerMockRunner.class) -@PrepareForTest(WrapperSetupHelper.class) -@PowerMockIgnore({"javax.management.*", "com.sun.jndi.*", "org.apache.log4j.*"}) -public class SpoutWrapperTest extends AbstractTest { - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Test - public void testRunPrepare() throws Exception { - final StormConfig stormConfig = new StormConfig(); - stormConfig.put(this.r.nextInt(), this.r.nextInt()); - final Configuration flinkConfig = new Configuration(); - flinkConfig.setInteger("testKey", this.r.nextInt()); - - final ExecutionConfig taskConfig = mock(ExecutionConfig.class); - when(taskConfig.getGlobalJobParameters()).thenReturn(null).thenReturn(stormConfig) - .thenReturn(flinkConfig); - - final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class); - when(taskContext.getExecutionConfig()).thenReturn(taskConfig); - when(taskContext.getTaskName()).thenReturn("name"); - - final IRichSpout spout = mock(IRichSpout.class); - SpoutWrapper spoutWrapper = new SpoutWrapper(spout); - spoutWrapper.setRuntimeContext(taskContext); - spoutWrapper.cancel(); - - // test without configuration - spoutWrapper.run(mock(SourceContext.class)); - verify(spout).open(any(Map.class), any(TopologyContext.class), - any(SpoutOutputCollector.class)); - - // test with StormConfig - spoutWrapper.run(mock(SourceContext.class)); - verify(spout).open(eq(stormConfig), any(TopologyContext.class), - any(SpoutOutputCollector.class)); - - // test with Configuration - final TestDummySpout testSpout = new TestDummySpout(); - spoutWrapper = new SpoutWrapper(testSpout); - spoutWrapper.setRuntimeContext(taskContext); - spoutWrapper.cancel(); - - spoutWrapper.run(mock(SourceContext.class)); - for (Entry entry : flinkConfig.toMap().entrySet()) { - Assert.assertEquals(entry.getValue(), testSpout.config.get(entry.getKey())); - } - } - - @SuppressWarnings("unchecked") - @Test - public void testRunExecuteFixedNumber() throws Exception { - final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - declarer.declare(new Fields("dummy")); - PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments() - .thenReturn(declarer); - - final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class); - when(taskContext.getExecutionConfig()).thenReturn(mock(ExecutionConfig.class)); - when(taskContext.getTaskName()).thenReturn("name"); - - final IRichSpout spout = mock(IRichSpout.class); - final int numberOfCalls = this.r.nextInt(50); - final SpoutWrapper spoutWrapper = new SpoutWrapper(spout, - numberOfCalls); - spoutWrapper.setRuntimeContext(taskContext); - - spoutWrapper.run(mock(SourceContext.class)); - verify(spout, times(numberOfCalls)).nextTuple(); - } - - @Test - public void testRunExecuteFinite() throws Exception { - final int numberOfCalls = this.r.nextInt(50); - - final LinkedList> expectedResult = new LinkedList>(); - for (int i = numberOfCalls - 1; i >= 0; --i) { - expectedResult.add(new Tuple1(new Integer(i))); - } - - final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class); - when(taskContext.getExecutionConfig()).thenReturn(mock(ExecutionConfig.class)); - when(taskContext.getTaskName()).thenReturn("name"); - - final FiniteTestSpout spout = new FiniteTestSpout(numberOfCalls); - final SpoutWrapper> spoutWrapper = new SpoutWrapper>( - spout, -1); - spoutWrapper.setRuntimeContext(taskContext); - - final TestContext collector = new TestContext(); - spoutWrapper.run(collector); - - Assert.assertEquals(expectedResult, collector.result); - } - - @SuppressWarnings("unchecked") - @Test - public void runAndExecuteFiniteSpout() throws Exception { - final FiniteSpout stormSpout = mock(FiniteSpout.class); - when(stormSpout.reachedEnd()).thenReturn(false, false, false, true, false, false, true); - - final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class); - when(taskContext.getExecutionConfig()).thenReturn(mock(ExecutionConfig.class)); - when(taskContext.getTaskName()).thenReturn("name"); - - final SpoutWrapper wrapper = new SpoutWrapper(stormSpout); - wrapper.setRuntimeContext(taskContext); - - wrapper.run(mock(SourceContext.class)); - verify(stormSpout, times(3)).nextTuple(); - } - - @SuppressWarnings("unchecked") - @Test - public void runAndExecuteFiniteSpout2() throws Exception { - final FiniteSpout stormSpout = mock(FiniteSpout.class); - when(stormSpout.reachedEnd()).thenReturn(true, false, true, false, true, false, true); - - final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class); - when(taskContext.getExecutionConfig()).thenReturn(mock(ExecutionConfig.class)); - when(taskContext.getTaskName()).thenReturn("name"); - - final SpoutWrapper wrapper = new SpoutWrapper(stormSpout); - wrapper.setRuntimeContext(taskContext); - - wrapper.run(mock(SourceContext.class)); - verify(stormSpout, never()).nextTuple(); - } - - @Test - public void testCancel() throws Exception { - final int numberOfCalls = 5 + this.r.nextInt(5); - - final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class); - when(taskContext.getExecutionConfig()).thenReturn(mock(ExecutionConfig.class)); - when(taskContext.getTaskName()).thenReturn("name"); - - final IRichSpout spout = new FiniteTestSpout(numberOfCalls); - - final SpoutWrapper> spoutWrapper = new SpoutWrapper>(spout); - spoutWrapper.setRuntimeContext(taskContext); - - spoutWrapper.cancel(); - final TestContext collector = new TestContext(); - spoutWrapper.run(collector); - - Assert.assertEquals(new LinkedList>(), collector.result); - } - - @Test - public void testClose() throws Exception { - final IRichSpout spout = mock(IRichSpout.class); - final SpoutWrapper> spoutWrapper = new SpoutWrapper>(spout); - - spoutWrapper.close(); - - verify(spout).close(); - } - -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/StormTupleTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/StormTupleTest.java deleted file mode 100644 index 2ff6c4598855d0..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/StormTupleTest.java +++ /dev/null @@ -1,710 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.api.java.tuple.Tuple; -import org.apache.flink.api.java.tuple.Tuple5; -import org.apache.flink.configuration.ConfigConstants; -import org.apache.flink.storm.util.AbstractTest; - -import org.apache.storm.generated.GlobalStreamId; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.MessageId; -import org.apache.storm.tuple.Values; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.mockito.Mockito.mock; - -/** - * Tests for the StormTuple. - */ -public class StormTupleTest extends AbstractTest { - private static final String fieldName = "fieldName"; - private static final String fieldNamePojo = "member"; - - private int arity, index; - - @Override - @Before - public void prepare() { - super.prepare(); - this.arity = 1 + r.nextInt(25); - this.index = r.nextInt(this.arity); - } - - @Test - public void nonTupleTest() { - final Object flinkTuple = this.r.nextInt(); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertSame(flinkTuple, tuple.getValue(0)); - - final List values = tuple.getValues(); - Assert.assertEquals(1, values.size()); - Assert.assertEquals(flinkTuple, values.get(0)); - } - - @Test - public void tupleTest() throws InstantiationException, IllegalAccessException { - for (int numberOfAttributes = 0; numberOfAttributes < 26; ++numberOfAttributes) { - final Object[] data = new Object[numberOfAttributes]; - - final Tuple flinkTuple = Tuple.getTupleClass(numberOfAttributes).newInstance(); - for (int i = 0; i < numberOfAttributes; ++i) { - data[i] = this.r.nextInt(); - flinkTuple.setField(data[i], i); - } - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - final List values = tuple.getValues(); - - Assert.assertEquals(numberOfAttributes, values.size()); - for (int i = 0; i < numberOfAttributes; ++i) { - Assert.assertEquals(flinkTuple.getField(i), values.get(i)); - } - - Assert.assertEquals(numberOfAttributes, tuple.size()); - } - } - - @Test - public void tupleTestWithTaskId() throws InstantiationException, IllegalAccessException { - for (int numberOfAttributes = 1; numberOfAttributes < 26; ++numberOfAttributes) { - final Object[] data = new Object[numberOfAttributes]; - - final Tuple flinkTuple = Tuple.getTupleClass(numberOfAttributes).newInstance(); - for (int i = 0; i < numberOfAttributes - 1; ++i) { - data[i] = this.r.nextInt(); - flinkTuple.setField(data[i], i); - } - - final StormTuple tuple = new StormTuple(flinkTuple, null, 0, null, null, - null); - final List values = tuple.getValues(); - - Assert.assertEquals(numberOfAttributes - 1, values.size()); - for (int i = 0; i < numberOfAttributes - 1; ++i) { - Assert.assertEquals(flinkTuple.getField(i), values.get(i)); - } - - Assert.assertEquals(numberOfAttributes - 1, tuple.size()); - } - } - - @Test - public void testBinary() { - final byte[] data = new byte[this.r.nextInt(15)]; - this.r.nextBytes(data); - - final int index = this.r.nextInt(5); - final Tuple flinkTuple = new Tuple5(); - flinkTuple.setField(data, index); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple.getField(index), tuple.getBinary(index)); - } - - @Test - public void testBoolean() { - final Boolean flinkTuple = this.r.nextBoolean(); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple, tuple.getBoolean(0)); - } - - @Test - public void testByte() { - final Byte flinkTuple = (byte) this.r.nextInt(); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, null); - Assert.assertEquals(flinkTuple, tuple.getByte(0)); - } - - @Test - public void testDouble() { - final Double flinkTuple = this.r.nextDouble(); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple, tuple.getDouble(0)); - } - - @Test - public void testFloat() { - final Float flinkTuple = this.r.nextFloat(); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple, tuple.getFloat(0)); - } - - @Test - public void testInteger() { - final Integer flinkTuple = this.r.nextInt(); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple, tuple.getInteger(0)); - } - - @Test - public void testLong() { - final Long flinkTuple = this.r.nextLong(); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, null); - Assert.assertEquals(flinkTuple, tuple.getLong(0)); - } - - @Test - public void testShort() { - final Short flinkTuple = (short) this.r.nextInt(); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple, tuple.getShort(0)); - } - - @Test - public void testString() { - final byte[] data = new byte[this.r.nextInt(15)]; - this.r.nextBytes(data); - final String flinkTuple = new String(data, ConfigConstants.DEFAULT_CHARSET); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple, tuple.getString(0)); - } - - @Test - public void testBinaryTuple() { - final byte[] data = new byte[this.r.nextInt(15)]; - this.r.nextBytes(data); - - final int index = this.r.nextInt(5); - final Tuple flinkTuple = new Tuple5(); - flinkTuple.setField(data, index); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple.getField(index), tuple.getBinary(index)); - } - - @Test - public void testBooleanTuple() { - final Boolean data = this.r.nextBoolean(); - - final int index = this.r.nextInt(5); - final Tuple flinkTuple = new Tuple5(); - flinkTuple.setField(data, index); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple.getField(index), tuple.getBoolean(index)); - } - - @Test - public void testByteTuple() { - final Byte data = (byte) this.r.nextInt(); - - final int index = this.r.nextInt(5); - final Tuple flinkTuple = new Tuple5(); - flinkTuple.setField(data, index); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple.getField(index), tuple.getByte(index)); - } - - @Test - public void testDoubleTuple() { - final Double data = this.r.nextDouble(); - - final int index = this.r.nextInt(5); - final Tuple flinkTuple = new Tuple5(); - flinkTuple.setField(data, index); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple.getField(index), tuple.getDouble(index)); - } - - @Test - public void testFloatTuple() { - final Float data = this.r.nextFloat(); - - final int index = this.r.nextInt(5); - final Tuple flinkTuple = new Tuple5(); - flinkTuple.setField(data, index); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple.getField(index), tuple.getFloat(index)); - } - - @Test - public void testIntegerTuple() { - final Integer data = this.r.nextInt(); - - final int index = this.r.nextInt(5); - final Tuple flinkTuple = new Tuple5(); - flinkTuple.setField(data, index); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple.getField(index), tuple.getInteger(index)); - } - - @Test - public void testLongTuple() { - final Long data = this.r.nextLong(); - - final int index = this.r.nextInt(5); - final Tuple flinkTuple = new Tuple5(); - flinkTuple.setField(data, index); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple.getField(index), tuple.getLong(index)); - } - - @Test - public void testShortTuple() { - final Short data = (short) this.r.nextInt(); - - final int index = this.r.nextInt(5); - final Tuple flinkTuple = new Tuple5(); - flinkTuple.setField(data, index); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple.getField(index), tuple.getShort(index)); - } - - @Test - public void testStringTuple() { - final byte[] rawdata = new byte[this.r.nextInt(15)]; - this.r.nextBytes(rawdata); - final String data = new String(rawdata, ConfigConstants.DEFAULT_CHARSET); - - final int index = this.r.nextInt(5); - final Tuple flinkTuple = new Tuple5(); - flinkTuple.setField(data, index); - - final StormTuple tuple = new StormTuple(flinkTuple, null, -1, null, null, - null); - Assert.assertEquals(flinkTuple.getField(index), tuple.getString(index)); - } - - @Test - public void testContains() throws Exception { - Fields schema = new Fields("a1", "a2"); - StormTuple tuple = new StormTuple(Tuple.getTupleClass(1).newInstance(), - schema, -1, null, null, null); - - Assert.assertTrue(tuple.contains("a1")); - Assert.assertTrue(tuple.contains("a2")); - Assert.assertFalse(tuple.contains("a3")); - } - - @Test - public void testGetFields() throws Exception { - Fields schema = new Fields(); - Assert.assertSame(schema, - new StormTuple(null, schema, -1, null, null, null).getFields()); - - } - - @Test - public void testFieldIndex() throws Exception { - Fields schema = new Fields("a1", "a2"); - StormTuple tuple = new StormTuple(Tuple.getTupleClass(1).newInstance(), - schema, -1, null, null, null); - - Assert.assertEquals(0, tuple.fieldIndex("a1")); - Assert.assertEquals(1, tuple.fieldIndex("a2")); - } - - @Test - public void testSelect() throws Exception { - Tuple tuple = Tuple.getTupleClass(arity).newInstance(); - Values values = new Values(); - - ArrayList attributeNames = new ArrayList(arity); - ArrayList filterNames = new ArrayList(arity); - - for (int i = 0; i < arity; ++i) { - tuple.setField(i, i); - attributeNames.add("a" + i); - - if (r.nextBoolean()) { - filterNames.add("a" + i); - values.add(i); - } - } - Fields schema = new Fields(attributeNames); - Fields selector = new Fields(filterNames); - - Assert.assertEquals(values, - new StormTuple<>(tuple, schema, -1, null, null, null).select(selector)); - } - - @Test - public void testGetValueByField() throws Exception { - Object value = mock(Object.class); - StormTuple tuple = testGetByField(arity, index, value); - Assert.assertSame(value, tuple.getValueByField(fieldName)); - - } - - @Test - public void testGetValueByFieldPojo() throws Exception { - Object value = mock(Object.class); - TestPojoMember pojo = new TestPojoMember(value); - StormTuple> tuple = new StormTuple>(pojo, - null, -1, null, null, null); - Assert.assertSame(value, tuple.getValueByField(fieldNamePojo)); - } - - @Test - public void testGetValueByFieldPojoGetter() throws Exception { - Object value = mock(Object.class); - TestPojoGetter pojo = new TestPojoGetter(value); - StormTuple> tuple = new StormTuple>(pojo, - null, -1, null, null, null); - Assert.assertSame(value, tuple.getValueByField(fieldNamePojo)); - } - - @Test - public void testGetStringByField() throws Exception { - String value = "stringValue"; - StormTuple tuple = testGetByField(arity, index, value); - Assert.assertSame(value, tuple.getStringByField(fieldName)); - } - - @Test - public void testGetStringByFieldPojo() throws Exception { - String value = "stringValue"; - TestPojoMember pojo = new TestPojoMember(value); - StormTuple> tuple = new StormTuple>(pojo, - null, -1, null, null, null); - Assert.assertSame(value, tuple.getStringByField(fieldNamePojo)); - } - - @Test - public void testGetStringByFieldPojoGetter() throws Exception { - String value = "stringValue"; - TestPojoGetter pojo = new TestPojoGetter(value); - StormTuple> tuple = new StormTuple>(pojo, - null, -1, null, null, null); - Assert.assertSame(value, tuple.getStringByField(fieldNamePojo)); - } - - @Test - public void testGetIntegerByField() throws Exception { - Integer value = r.nextInt(); - StormTuple tuple = testGetByField(arity, index, value); - Assert.assertSame(value, tuple.getIntegerByField(fieldName)); - } - - @Test - public void testGetIntegerByFieldPojo() throws Exception { - Integer value = r.nextInt(); - TestPojoMember pojo = new TestPojoMember(value); - StormTuple> tuple = new StormTuple>(pojo, - null, -1, null, null, null); - Assert.assertSame(value, tuple.getIntegerByField(fieldNamePojo)); - } - - @Test - public void testGetIntegerByFieldPojoGetter() throws Exception { - Integer value = r.nextInt(); - TestPojoGetter pojo = new TestPojoGetter(value); - StormTuple> tuple = new StormTuple>(pojo, - null, -1, null, null, null); - Assert.assertSame(value, tuple.getIntegerByField(fieldNamePojo)); - } - - @Test - public void testGetLongByField() throws Exception { - Long value = r.nextLong(); - StormTuple tuple = testGetByField(arity, index, value); - Assert.assertSame(value, tuple.getLongByField(fieldName)); - } - - @Test - public void testGetLongByFieldPojo() throws Exception { - Long value = r.nextLong(); - TestPojoMember pojo = new TestPojoMember(value); - StormTuple> tuple = new StormTuple>(pojo, null, - -1, null, null, null); - Assert.assertSame(value, tuple.getLongByField(fieldNamePojo)); - } - - @Test - public void testGetLongByFieldPojoGetter() throws Exception { - Long value = r.nextLong(); - TestPojoGetter pojo = new TestPojoGetter(value); - StormTuple> tuple = new StormTuple>(pojo, null, - -1, null, null, null); - Assert.assertSame(value, tuple.getLongByField(fieldNamePojo)); - } - - @Test - public void testGetBooleanByField() throws Exception { - Boolean value = r.nextBoolean(); - StormTuple tuple = testGetByField(arity, index, value); - Assert.assertEquals(value, tuple.getBooleanByField(fieldName)); - } - - @Test - public void testGetBooleanByFieldPojo() throws Exception { - Boolean value = r.nextBoolean(); - TestPojoMember pojo = new TestPojoMember(value); - StormTuple> tuple = new StormTuple>(pojo, - null, -1, null, null, null); - Assert.assertSame(value, tuple.getBooleanByField(fieldNamePojo)); - } - - @Test - public void testGetBooleanByFieldPojoGetter() throws Exception { - Boolean value = r.nextBoolean(); - TestPojoGetter pojo = new TestPojoGetter(value); - StormTuple> tuple = new StormTuple>(pojo, - null, -1, null, null, null); - Assert.assertSame(value, tuple.getBooleanByField(fieldNamePojo)); - } - - @Test - public void testGetShortByField() throws Exception { - Short value = (short) r.nextInt(); - StormTuple tuple = testGetByField(arity, index, value); - Assert.assertSame(value, tuple.getShortByField(fieldName)); - } - - @Test - public void testGetShortByFieldPojo() throws Exception { - Short value = (short) r.nextInt(); - TestPojoMember pojo = new TestPojoMember(value); - StormTuple> tuple = new StormTuple>(pojo, - null, - -1, null, null, null); - Assert.assertSame(value, tuple.getShortByField(fieldNamePojo)); - } - - @Test - public void testGetShortByFieldPojoGetter() throws Exception { - Short value = (short) r.nextInt(); - TestPojoGetter pojo = new TestPojoGetter(value); - StormTuple> tuple = new StormTuple>(pojo, - null, - -1, null, null, null); - Assert.assertSame(value, tuple.getShortByField(fieldNamePojo)); - } - - @Test - public void testGetByteByField() throws Exception { - Byte value = new Byte((byte) r.nextInt()); - StormTuple tuple = testGetByField(arity, index, value); - Assert.assertSame(value, tuple.getByteByField(fieldName)); - } - - @Test - public void testGetByteByFieldPojo() throws Exception { - Byte value = new Byte((byte) r.nextInt()); - TestPojoMember pojo = new TestPojoMember(value); - StormTuple> tuple = new StormTuple>(pojo, - null, - -1, null, null, null); - Assert.assertSame(value, tuple.getByteByField(fieldNamePojo)); - } - - @Test - public void testGetByteByFieldPojoGetter() throws Exception { - Byte value = new Byte((byte) r.nextInt()); - TestPojoGetter pojo = new TestPojoGetter(value); - StormTuple> tuple = new StormTuple>(pojo, - null, - -1, null, null, null); - Assert.assertSame(value, tuple.getByteByField(fieldNamePojo)); - } - - @Test - public void testGetDoubleByField() throws Exception { - Double value = r.nextDouble(); - StormTuple tuple = testGetByField(arity, index, value); - Assert.assertSame(value, tuple.getDoubleByField(fieldName)); - } - - @Test - public void testGetDoubleByFieldPojo() throws Exception { - Double value = r.nextDouble(); - TestPojoMember pojo = new TestPojoMember(value); - StormTuple> tuple = new StormTuple>(pojo, - null, -1, null, null, null); - Assert.assertSame(value, tuple.getDoubleByField(fieldNamePojo)); - } - - @Test - public void testGetDoubleByFieldPojoGetter() throws Exception { - Double value = r.nextDouble(); - TestPojoGetter pojo = new TestPojoGetter(value); - StormTuple> tuple = new StormTuple>(pojo, - null, -1, null, null, null); - Assert.assertSame(value, tuple.getDoubleByField(fieldNamePojo)); - } - - @Test - public void testGetFloatByField() throws Exception { - Float value = r.nextFloat(); - StormTuple tuple = testGetByField(arity, index, value); - Assert.assertSame(value, tuple.getFloatByField(fieldName)); - } - - @Test - public void testGetFloatByFieldPojo() throws Exception { - Float value = r.nextFloat(); - TestPojoMember pojo = new TestPojoMember(value); - StormTuple> tuple = new StormTuple>(pojo, - null, - -1, null, null, null); - Assert.assertSame(value, tuple.getFloatByField(fieldNamePojo)); - } - - @Test - public void testGetFloatByFieldPojoGetter() throws Exception { - Float value = r.nextFloat(); - TestPojoGetter pojo = new TestPojoGetter(value); - StormTuple> tuple = new StormTuple>(pojo, - null, - -1, null, null, null); - Assert.assertSame(value, tuple.getFloatByField(fieldNamePojo)); - } - - @Test - public void testGetBinaryByField() throws Exception { - byte[] data = new byte[1 + r.nextInt(20)]; - r.nextBytes(data); - StormTuple tuple = testGetByField(arity, index, data); - Assert.assertSame(data, tuple.getBinaryByField(fieldName)); - } - - @Test - public void testGetBinaryFieldPojo() throws Exception { - byte[] data = new byte[1 + r.nextInt(20)]; - r.nextBytes(data); - TestPojoMember pojo = new TestPojoMember(data); - StormTuple> tuple = new StormTuple>(pojo, - null, -1, null, null, null); - Assert.assertSame(data, tuple.getBinaryByField(fieldNamePojo)); - } - - @Test - public void testGetBinaryByFieldPojoGetter() throws Exception { - byte[] data = new byte[1 + r.nextInt(20)]; - r.nextBytes(data); - TestPojoGetter pojo = new TestPojoGetter(data); - StormTuple> tuple = new StormTuple>(pojo, - null, -1, null, null, null); - Assert.assertSame(data, tuple.getBinaryByField(fieldNamePojo)); - } - - private StormTuple testGetByField(int arity, int index, T value) - throws Exception { - - assert (index < arity); - - Tuple tuple = Tuple.getTupleClass(arity).newInstance(); - tuple.setField(value, index); - - ArrayList attributeNames = new ArrayList(arity); - for (int i = 0; i < arity; ++i) { - if (i == index) { - attributeNames.add(fieldName); - } else { - attributeNames.add("" + i); - } - } - Fields schema = new Fields(attributeNames); - - return new StormTuple<>(tuple, schema, -1, null, null, null); - } - - @Test - public void testGetSourceGlobalStreamid() { - GlobalStreamId globalStreamid = new StormTuple<>(null, null, -1, "streamId", "componentID", - null).getSourceGlobalStreamid(); - Assert.assertEquals("streamId", globalStreamid.get_streamId()); - Assert.assertEquals("componentID", globalStreamid.get_componentId()); - } - - @Test - public void testGetSourceComponent() { - String sourceComponent = new StormTuple<>(null, null, -1, null, "componentID", null) - .getSourceComponent(); - Assert.assertEquals("componentID", sourceComponent); - } - - @Test - public void testGetSourceTask() { - int sourceTaskId = new StormTuple<>(null, null, 42, null, null, null).getSourceTask(); - Assert.assertEquals(42, sourceTaskId); - } - - @Test - public void testGetSourceStreamId() { - String sourceStreamId = new StormTuple<>(null, null, -1, "streamId", null, null) - .getSourceStreamId(); - Assert.assertEquals("streamId", sourceStreamId); - } - - @Test - public void testGetMessageId() { - MessageId messageId = MessageId.makeUnanchored(); - StormTuple stormTuple = new StormTuple<>(null, null, -1, null, null, messageId); - Assert.assertSame(messageId, stormTuple.getMessageId()); - } - - private static class TestPojoMember { - public T member; - - public TestPojoMember(T value) { - this.member = value; - } - } - - private static class TestPojoGetter { - private T member; - - public TestPojoGetter(T value) { - this.member = value; - } - - public T getMember() { - return this.member; - } - } -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/TestContext.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/TestContext.java deleted file mode 100644 index 58aad7b79b20d5..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/TestContext.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.api.java.tuple.Tuple1; -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext; -import org.apache.flink.streaming.api.watermark.Watermark; - -import java.util.LinkedList; - -class TestContext implements SourceContext> { - public LinkedList> result = new LinkedList>(); - - public TestContext() { - } - - @Override - public void collect(final Tuple1 record) { - this.result.add(record.copy()); - } - - @Override - public void collectWithTimestamp(Tuple1 element, long timestamp) { - this.result.add(element.copy()); - } - - @Override - public void emitWatermark(Watermark mark) { - throw new UnsupportedOperationException(); - } - - @Override - public void markAsTemporarilyIdle() { - throw new UnsupportedOperationException(); - } - - @Override - public Object getCheckpointLock() { - return null; - } - - @Override - public void close() { - - } -} diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupHelperTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupHelperTest.java deleted file mode 100644 index 3118d6bd6a506e..00000000000000 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupHelperTest.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.storm.wrappers; - -import org.apache.flink.storm.util.AbstractTest; - -import org.apache.storm.topology.IComponent; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.tuple.Fields; -import org.apache.storm.utils.Utils; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.util.HashMap; -import java.util.HashSet; - -import static java.util.Collections.singleton; -import static org.mockito.Mockito.mock; - -/** - * Tests for the WrapperSetupHelper. - */ -@RunWith(PowerMockRunner.class) -@PrepareForTest(WrapperSetupHelper.class) -@PowerMockIgnore({"javax.*", "org.apache.log4j.*"}) -public class WrapperSetupHelperTest extends AbstractTest { - - @Test - public void testEmptyDeclarerBolt() { - IComponent boltOrSpout; - - if (this.r.nextBoolean()) { - boltOrSpout = mock(IRichSpout.class); - } else { - boltOrSpout = mock(IRichBolt.class); - } - - Assert.assertEquals(new HashMap(), - WrapperSetupHelper.getNumberOfAttributes(boltOrSpout, null)); - } - - @Test(expected = IllegalArgumentException.class) - public void testRawType() throws Exception { - IComponent boltOrSpout; - - if (this.r.nextBoolean()) { - boltOrSpout = mock(IRichSpout.class); - } else { - boltOrSpout = mock(IRichBolt.class); - } - - final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - declarer.declare(new Fields("dummy1", "dummy2")); - PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer); - - WrapperSetupHelper.getNumberOfAttributes(boltOrSpout, - new HashSet(singleton(Utils.DEFAULT_STREAM_ID))); - } - - @Test(expected = IllegalArgumentException.class) - public void testToManyAttributes() throws Exception { - IComponent boltOrSpout; - - if (this.r.nextBoolean()) { - boltOrSpout = mock(IRichSpout.class); - } else { - boltOrSpout = mock(IRichBolt.class); - } - - final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - final String[] schema = new String[26]; - for (int i = 0; i < schema.length; ++i) { - schema[i] = "a" + i; - } - declarer.declare(new Fields(schema)); - PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer); - - WrapperSetupHelper.getNumberOfAttributes(boltOrSpout, null); - } - - @Test - public void testTupleTypes() throws Exception { - for (int i = -1; i < 26; ++i) { - this.testTupleTypes(i); - } - } - - private void testTupleTypes(final int numberOfAttributes) throws Exception { - String[] schema; - if (numberOfAttributes == -1) { - schema = new String[1]; - } else { - schema = new String[numberOfAttributes]; - } - for (int i = 0; i < schema.length; ++i) { - schema[i] = "a" + i; - } - - IComponent boltOrSpout; - if (this.r.nextBoolean()) { - boltOrSpout = mock(IRichSpout.class); - } else { - boltOrSpout = mock(IRichBolt.class); - } - - final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer(); - declarer.declare(new Fields(schema)); - PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer); - - HashMap attributes = new HashMap(); - attributes.put(Utils.DEFAULT_STREAM_ID, numberOfAttributes); - - Assert.assertEquals(attributes, WrapperSetupHelper.getNumberOfAttributes( - boltOrSpout, - numberOfAttributes == -1 ? new HashSet(singleton(Utils.DEFAULT_STREAM_ID)) : null)); - } -} diff --git a/flink-contrib/flink-storm/src/test/resources/log4j-test.properties b/flink-contrib/flink-storm/src/test/resources/log4j-test.properties deleted file mode 100644 index 881dc0609b8fdb..00000000000000 --- a/flink-contrib/flink-storm/src/test/resources/log4j-test.properties +++ /dev/null @@ -1,27 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -# Set root logger level to DEBUG and its only appender to A1. -log4j.rootLogger=OFF, A1 - -# A1 is set to be a ConsoleAppender. -log4j.appender.A1=org.apache.log4j.ConsoleAppender - -# A1 uses PatternLayout. -log4j.appender.A1.layout=org.apache.log4j.PatternLayout -log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/flink-contrib/pom.xml b/flink-contrib/pom.xml index 7ea9b19731e038..ad81ad6b0e780a 100644 --- a/flink-contrib/pom.xml +++ b/flink-contrib/pom.xml @@ -37,8 +37,6 @@ under the License. pom - flink-storm - flink-storm-examples flink-connector-wikiedits diff --git a/tools/travis/stage.sh b/tools/travis/stage.sh index 53f6beee5efce4..1e43fe8a412b14 100644 --- a/tools/travis/stage.sh +++ b/tools/travis/stage.sh @@ -39,8 +39,6 @@ flink-streaming-java,\ flink-streaming-scala" MODULES_LIBRARIES="\ -flink-contrib/flink-storm,\ -flink-contrib/flink-storm-examples,\ flink-libraries/flink-cep,\ flink-libraries/flink-cep-scala,\ flink-libraries/flink-gelly,\ From 157a26724557b287b3dff5b9e24aff447acae824 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Fri, 11 Jan 2019 09:22:23 +0100 Subject: [PATCH 111/147] Fix queryable-state nightly test --- flink-end-to-end-tests/test-scripts/queryable_state_base.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/flink-end-to-end-tests/test-scripts/queryable_state_base.sh b/flink-end-to-end-tests/test-scripts/queryable_state_base.sh index 65ddef47e44930..2801da4dd62693 100644 --- a/flink-end-to-end-tests/test-scripts/queryable_state_base.sh +++ b/flink-end-to-end-tests/test-scripts/queryable_state_base.sh @@ -24,6 +24,7 @@ function link_queryable_state_lib { echo "Failed to move flink-queryable-state-runtime from opt/ to lib/. Exiting" exit 1 fi + set_conf "queryable-state.enable" "true" } function unlink_queryable_state_lib { From e0efabe8884f22b4a1c7ab9df3274b3fca03dcfb Mon Sep 17 00:00:00 2001 From: Shimin Yang Date: Thu, 6 Dec 2018 14:39:46 +0800 Subject: [PATCH 112/147] [FLINK-11084][datastream] Forbid using two consecutive split transformations This closes #7258 --- .../streaming/api/datastream/DataStream.java | 2 + .../streaming/api/datastream/SplitStream.java | 1 + .../api/graph/StreamGraphGenerator.java | 18 ++++ .../flink/streaming/api/DataStreamTest.java | 100 ++++++++++++++++-- .../streaming/api/scala/DataStream.scala | 6 ++ 5 files changed, 121 insertions(+), 6 deletions(-) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java index 271d9bec33f0e6..a0af5f1a8c317a 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java @@ -234,7 +234,9 @@ public final DataStream union(DataStream... streams) { * {@link org.apache.flink.streaming.api.collector.selector.OutputSelector} * for directing the tuples. * @return The {@link SplitStream} + * @deprecated Please use side ouput instead. */ + @Deprecated public SplitStream split(OutputSelector outputSelector) { return new SplitStream<>(this, clean(outputSelector)); } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SplitStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SplitStream.java index 0beae32435d4ca..7f28dc7b497d67 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SplitStream.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/SplitStream.java @@ -33,6 +33,7 @@ * @param The type of the elements in the Stream */ +@Deprecated @PublicEvolving public class SplitStream extends DataStream { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java index 2c4ae4a993dfa5..82400036c3c83a 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java @@ -258,6 +258,8 @@ private Collection transformSplit(SplitTransformation split) { StreamTransformation input = split.getInput(); Collection resultIds = transform(input); + validateSplitTransformation(input); + // the recursive transform call might have transformed this already if (alreadyTransformed.containsKey(split)) { return alreadyTransformed.get(split); @@ -643,4 +645,20 @@ private String determineSlotSharingGroup(String specifiedGroup, Collection void validateSplitTransformation(StreamTransformation input) { + if (input instanceof SelectTransformation || input instanceof SplitTransformation) { + throw new IllegalStateException("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); + } else if (input instanceof SideOutputTransformation) { + throw new IllegalStateException("Split after side-outputs are not supported. Splits are deprecated. Please use side-outputs."); + } else if (input instanceof UnionTransformation) { + for (StreamTransformation transformation : ((UnionTransformation) input).getInputs()) { + validateSplitTransformation(transformation); + } + } else if (input instanceof PartitionTransformation) { + validateSplitTransformation(((PartitionTransformation) input).getInput()); + } else { + return; + } + } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java index aa6774dab8fc95..215e5485648f4a 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java @@ -79,6 +79,7 @@ import org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner; import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner; import org.apache.flink.util.Collector; +import org.apache.flink.util.OutputTag; import org.apache.flink.util.TestLogger; import org.hamcrest.core.StringStartsWith; @@ -952,12 +953,7 @@ public boolean filter(Integer value) throws Exception { fail(e.getMessage()); } - OutputSelector outputSelector = new OutputSelector() { - @Override - public Iterable select(Integer value) { - return null; - } - }; + OutputSelector outputSelector = new DummyOutputSelector<>(); SplitStream split = unionFilter.split(outputSelector); split.select("dummy").addSink(new DiscardingSink()); @@ -1091,6 +1087,91 @@ public void testChannelSelectors() { assertTrue(globalPartitioner instanceof GlobalPartitioner); } + ///////////////////////////////////////////////////////////// + // Split testing + ///////////////////////////////////////////////////////////// + + @Test + public void testConsecutiveSplitRejection() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + DataStreamSource src = env.fromElements(0, 0); + + OutputSelector outputSelector = new DummyOutputSelector<>(); + + src.split(outputSelector).split(outputSelector).addSink(new DiscardingSink<>()); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); + + env.getStreamGraph(); + } + + @Test + public void testSplitAfterSideOutputRejection() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + DataStreamSource src = env.fromElements(0, 0); + + OutputTag outputTag = new OutputTag("dummy"){}; + OutputSelector outputSelector = new DummyOutputSelector<>(); + + src.getSideOutput(outputTag).split(outputSelector).addSink(new DiscardingSink<>()); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Split after side-outputs are not supported. Splits are deprecated. Please use side-outputs."); + + env.getStreamGraph(); + } + + @Test + public void testSelectBetweenConsecutiveSplitRejection() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + DataStreamSource src = env.fromElements(0, 0); + + OutputSelector outputSelector = new DummyOutputSelector<>(); + + src.split(outputSelector).select("dummy").split(outputSelector).addSink(new DiscardingSink<>()); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); + + env.getStreamGraph(); + } + + @Test + public void testUnionBetweenConsecutiveSplitRejection() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + DataStreamSource src = env.fromElements(0, 0); + + OutputSelector outputSelector = new DummyOutputSelector<>(); + + src.split(outputSelector).select("dummy").union(src.map(x -> x)).split(outputSelector).addSink(new DiscardingSink<>()); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); + + env.getStreamGraph(); + } + + @Test + public void testKeybyBetweenConsecutiveSplitRejection() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + DataStreamSource src = env.fromElements(0, 0); + + OutputSelector outputSelector = new DummyOutputSelector<>(); + + src.split(outputSelector).select("dummy").keyBy(x -> x).split(outputSelector).addSink(new DiscardingSink<>()); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); + + env.getStreamGraph(); + } + ///////////////////////////////////////////////////////////// // KeyBy testing ///////////////////////////////////////////////////////////// @@ -1427,4 +1508,11 @@ public int getI() { return i; } } + + private class DummyOutputSelector implements OutputSelector { + @Override + public Iterable select(Integer value) { + return null; + } + } } diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala index 23d216549f836a..15dca2c9c07523 100644 --- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala +++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala @@ -915,13 +915,19 @@ class DataStream[T](stream: JavaStream[T]) { * Operator used for directing tuples to specific named outputs using an * OutputSelector. Calling this method on an operator creates a new * [[SplitStream]]. + * + * @deprecated Please use side output instead. */ + @deprecated def split(selector: OutputSelector[T]): SplitStream[T] = asScalaStream(stream.split(selector)) /** * Creates a new [[SplitStream]] that contains only the elements satisfying the * given output selector predicate. + * + * @deprecated Please use side output instead. */ + @deprecated def split(fun: T => TraversableOnce[String]): SplitStream[T] = { if (fun == null) { throw new NullPointerException("OutputSelector must not be null.") From be4706171f68e487ce3f409232e829b681ffd2cd Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Fri, 11 Jan 2019 17:20:02 +0800 Subject: [PATCH 113/147] [FLINK-11224][scala-shell] Log is missing in scala-shell Logging configuration was set only for scala-shell in yarn mode. This commit sets the configuration for local and remote mode in start-scala-shell.sh script as well. --- .../start-script/start-scala-shell.sh | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/flink-scala-shell/start-script/start-scala-shell.sh b/flink-scala-shell/start-script/start-scala-shell.sh index 489d50b25d51c1..4bc5d25fc9538b 100644 --- a/flink-scala-shell/start-script/start-scala-shell.sh +++ b/flink-scala-shell/start-script/start-scala-shell.sh @@ -82,15 +82,26 @@ do fi done -log_setting="" +if [ "$FLINK_IDENT_STRING" = "" ]; then + FLINK_IDENT_STRING="$USER" +fi + +MODE=$1 +LOG=$FLINK_LOG_DIR/flink-$FLINK_IDENT_STRING-scala-shell-$MODE-$HOSTNAME.log -if [[ $1 = "yarn" ]] +if [[ ($MODE = "local") || ($MODE = "remote") ]] then -FLINK_CLASSPATH=$FLINK_CLASSPATH:$HADOOP_CLASSPATH:$HADOOP_CONF_DIR:$YARN_CONF_DIR -log=$FLINK_LOG_DIR/flink-$FLINK_IDENT_STRING-scala-shell-yarn-$HOSTNAME.log -log_setting="-Dlog.file="$log" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/log4j-yarn-session.properties -Dlogback.configurationFile=file:"$FLINK_CONF_DIR"/logback-yarn.xml" + LOG4J_CONFIG=log4j.properties + LOGBACK_CONFIG=logback.xml +elif [[ $1 = "yarn" ]] +then + LOG4J_CONFIG=log4j-yarn-session.properties + LOGBACK_CONFIG=logback-yarn.xml + FLINK_CLASSPATH=$FLINK_CLASSPATH:$HADOOP_CLASSPATH:$HADOOP_CONF_DIR:$YARN_CONF_DIR fi +log_setting="-Dlog.file="$LOG" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/$LOG4J_CONFIG -Dlogback.configurationFile=file:"$FLINK_CONF_DIR"/$LOGBACK_CONFIG" + if ${EXTERNAL_LIB_FOUND} then java -Dscala.color -cp "$FLINK_CLASSPATH" $log_setting org.apache.flink.api.scala.FlinkShell $@ --addclasspath "$EXT_CLASSPATH" From 1a4eb097a9454db7cc5972fda43e0b15084371bd Mon Sep 17 00:00:00 2001 From: intsmaze <979222969@qq.com> Date: Fri, 11 Jan 2019 19:01:55 +0800 Subject: [PATCH 114/147] [hotfix][docs] Fix NullPointerException Offset is Long type, the default value is null, and executing offset+=1 in Java 1.8.x will throw a NullPointerException --- docs/dev/stream/state/state.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/stream/state/state.md b/docs/dev/stream/state/state.md index a3109f2c2fa4b4..8fe2981a20a07e 100644 --- a/docs/dev/stream/state/state.md +++ b/docs/dev/stream/state/state.md @@ -640,7 +640,7 @@ public static class CounterSource implements ListCheckpointed { /** current offset for exactly once semantics */ - private Long offset; + private Long offset = 0L; /** flag for job cancellation */ private volatile boolean isRunning = true; From 205f9064897bfcf5f8988d38deb5a407f9a7cae2 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 11 Jan 2019 13:11:46 +0100 Subject: [PATCH 115/147] [FLINK-11306][jdcs] Bump derby from 10.10.1.1 to 10.14.2.0 Derby has two CVE's: https://ossindex.sonatype.org/vuln/b01781a6-31db-4dbc-b091-d683aaeb98e6 https://ossindex.sonatype.org/vuln/bbe6594a-c255-41a7-a397-d762d1aa8b00 Although it is only a test dependency, updating would be nice. --- flink-connectors/flink-jdbc/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-connectors/flink-jdbc/pom.xml b/flink-connectors/flink-jdbc/pom.xml index 165154e3819467..4c53310ac259b3 100644 --- a/flink-connectors/flink-jdbc/pom.xml +++ b/flink-connectors/flink-jdbc/pom.xml @@ -56,7 +56,7 @@ under the License. org.apache.derby derby - 10.10.1.1 + 10.14.2.0 test From 3903b817cb20524aec2c0fbee763e6bf73f5f989 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Tue, 8 Jan 2019 13:06:50 +0100 Subject: [PATCH 116/147] [FLINK-10848] Remove container requests after successful container allocation This commit removes container requests after containers have been allocated. This prevents that we will request more and more containers from Yarn in case of a recovery. Since we cannot rely on the reported container Resource, we remove the container request by using the requested Resource. This is due Yarn's DefaultResourceCalculator which neglects the number of vCores when allocating containers. This closes #7430. --- .../flink/yarn/YarnFlinkResourceManager.java | 73 ++++++++++--- .../flink/yarn/YarnResourceManager.java | 103 ++++++++++++------ .../src/main/resources/log4j.properties | 2 +- .../yarn/YarnFlinkResourceManagerTest.java | 25 ++--- .../flink/yarn/YarnResourceManagerTest.java | 19 +++- .../src/test/resources/log4j-test.properties | 2 +- 6 files changed, 151 insertions(+), 73 deletions(-) diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java index 8e686bbbe34c10..e8e55c33303618 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java @@ -41,15 +41,20 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.NMClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.slf4j.Logger; +import javax.annotation.Nonnull; + import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -72,6 +77,8 @@ public class YarnFlinkResourceManager extends FlinkResourceManager containersInLaunch; @@ -314,6 +321,21 @@ protected void fatalError(String message, Throwable error) { @Override protected void requestNewWorkers(int numWorkers) { + final Resource capability = getContainerResource(); + + for (int i = 0; i < numWorkers; i++) { + numPendingContainerRequests++; + LOG.info("Requesting new TaskManager container with {} megabytes memory. Pending requests: {}", + capability.getMemory(), numPendingContainerRequests); + + resourceManagerClient.addContainerRequest(createContainerRequest(capability)); + } + + // make sure we transmit the request fast and receive fast news of granted allocations + resourceManagerClient.setHeartbeatInterval(FAST_YARN_HEARTBEAT_INTERVAL_MS); + } + + private Resource getContainerResource() { final long mem = taskManagerParameters.taskManagerTotalMemoryMB(); final int containerMemorySizeMB; @@ -325,25 +347,15 @@ protected void requestNewWorkers(int numWorkers) { mem, containerMemorySizeMB); } - for (int i = 0; i < numWorkers; i++) { - numPendingContainerRequests++; - LOG.info("Requesting new TaskManager container with {} megabytes memory. Pending requests: {}", - containerMemorySizeMB, numPendingContainerRequests); - - // Priority for worker containers - priorities are intra-application - Priority priority = Priority.newInstance(0); - - // Resource requirements for worker containers - int taskManagerSlots = taskManagerParameters.numSlots(); - int vcores = config.getInteger(YarnConfigOptions.VCORES, Math.max(taskManagerSlots, 1)); - Resource capability = Resource.newInstance(containerMemorySizeMB, vcores); - - resourceManagerClient.addContainerRequest( - new AMRMClient.ContainerRequest(capability, null, null, priority)); - } + // Resource requirements for worker containers + int taskManagerSlots = taskManagerParameters.numSlots(); + int vcores = config.getInteger(YarnConfigOptions.VCORES, Math.max(taskManagerSlots, 1)); + return Resource.newInstance(containerMemorySizeMB, vcores); + } - // make sure we transmit the request fast and receive fast news of granted allocations - resourceManagerClient.setHeartbeatInterval(FAST_YARN_HEARTBEAT_INTERVAL_MS); + @Nonnull + private AMRMClient.ContainerRequest createContainerRequest(Resource capability) { + return new AMRMClient.ContainerRequest(capability, null, null, RM_REQUEST_PRIORITY); } @Override @@ -434,7 +446,14 @@ private void containersAllocated(List containers) { final int numRequired = getDesignatedWorkerPoolSize(); final int numRegistered = getNumberOfStartedTaskManagers(); + final Collection pendingRequests = getPendingRequests(); + final Iterator pendingRequestsIterator = pendingRequests.iterator(); + for (Container container : containers) { + if (numPendingContainerRequests > 0) { + numPendingContainerRequests -= 1; + resourceManagerClient.removeContainerRequest(pendingRequestsIterator.next()); + } numPendingContainerRequests = Math.max(0, numPendingContainerRequests - 1); LOG.info("Received new container: {} - Remaining pending container requests: {}", container.getId(), numPendingContainerRequests); @@ -487,6 +506,24 @@ private void containersAllocated(List containers) { triggerCheckWorkers(); } + private Collection getPendingRequests() { + final List> matchingRequests = resourceManagerClient.getMatchingRequests(RM_REQUEST_PRIORITY, ResourceRequest.ANY, getContainerResource()); + + final Collection result; + + if (matchingRequests.isEmpty()) { + result = Collections.emptyList(); + } else { + result = new ArrayList<>(matchingRequests.get(0)); + } + + Preconditions.checkState( + result.size() == numPendingContainerRequests, + "The RMClient's and YarnResourceManagers internal state about the number of pending container requests has diverged. Number client's pending container requests %s != Number RM's pending container requests %s.", result.size(), numPendingContainerRequests); + + return result; + } + /** * Invoked when the ResourceManager informs of completed containers. * Called via an actor message by the callback from the ResourceManager client. diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java index 6ff5cd66487112..609ab42f590f0e 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java @@ -18,6 +18,7 @@ package org.apache.flink.yarn; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.ConfigurationUtils; @@ -41,6 +42,7 @@ import org.apache.flink.runtime.rpc.RpcService; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; +import org.apache.flink.util.Preconditions; import org.apache.flink.yarn.configuration.YarnConfigOptions; import org.apache.hadoop.yarn.api.ApplicationConstants; @@ -52,15 +54,20 @@ import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.NMClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -73,6 +80,7 @@ */ public class YarnResourceManager extends ResourceManager implements AMRMClientAsync.CallbackHandler { + private static final Priority RM_REQUEST_PRIORITY = Priority.newInstance(1); /** The process environment variables. */ private final Map env; @@ -119,6 +127,8 @@ public class YarnResourceManager extends ResourceManager impleme private final Collection slotsPerWorker; + private final Resource resource; + public YarnResourceManager( RpcService rpcService, String resourceManagerEndpointId, @@ -169,6 +179,7 @@ public YarnResourceManager( this.numberOfTaskSlots = flinkConfig.getInteger(TaskManagerOptions.NUM_TASK_SLOTS); this.defaultTaskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(flinkConfig).getMebiBytes(); this.defaultCpus = flinkConfig.getInteger(YarnConfigOptions.VCORES, numberOfTaskSlots); + this.resource = Resource.newInstance(defaultTaskManagerMemoryMB, defaultCpus); this.slotsPerWorker = createSlotsPerWorker(numberOfTaskSlots); } @@ -292,17 +303,19 @@ protected void internalDeregisterApplication( @Override public Collection startNewWorker(ResourceProfile resourceProfile) { - // Priority for worker containers - priorities are intra-application - //TODO: set priority according to the resource allocated - Priority priority = Priority.newInstance(generatePriority(resourceProfile)); - int mem = resourceProfile.getMemoryInMB() < 0 ? defaultTaskManagerMemoryMB : resourceProfile.getMemoryInMB(); - int vcore = resourceProfile.getCpuCores() < 1 ? defaultCpus : (int) resourceProfile.getCpuCores(); - Resource capability = Resource.newInstance(mem, vcore); - requestYarnContainer(capability, priority); + Preconditions.checkArgument( + ResourceProfile.UNKNOWN.equals(resourceProfile), + "The YarnResourceManager does not support custom ResourceProfiles yet. It assumes that all containers have the same resources."); + requestYarnContainer(); return slotsPerWorker; } + @VisibleForTesting + Resource getContainerResource() { + return resource; + } + @Override public boolean stopWorker(final YarnWorkerNode workerNode) { final Container container = workerNode.getContainer(); @@ -343,8 +356,7 @@ public void onContainersCompleted(final List statuses) { if (yarnWorkerNode != null) { // Container completed unexpectedly ~> start a new one - final Container container = yarnWorkerNode.getContainer(); - requestYarnContainerIfRequired(container.getResource(), yarnWorkerNode.getContainer().getPriority()); + requestYarnContainerIfRequired(); } // Eagerly close the connection with task manager. closeTaskManagerConnection(resourceId, new Exception(containerStatus.getDiagnostics())); @@ -356,6 +368,9 @@ public void onContainersCompleted(final List statuses) { @Override public void onContainersAllocated(List containers) { runAsync(() -> { + final Collection pendingRequests = getPendingRequests(); + final Iterator pendingRequestsIterator = pendingRequests.iterator(); + for (Container container : containers) { log.info( "Received new container: {} - Remaining pending container requests: {}", @@ -363,7 +378,7 @@ public void onContainersAllocated(List containers) { numPendingContainerRequests); if (numPendingContainerRequests > 0) { - numPendingContainerRequests--; + removeContainerRequest(pendingRequestsIterator.next()); final String containerIdStr = container.getId().toString(); final ResourceID resourceId = new ResourceID(containerIdStr); @@ -385,7 +400,7 @@ public void onContainersAllocated(List containers) { workerNodeMap.remove(resourceId); resourceManagerClient.releaseAssignedContainer(container.getId()); // and ask for a new one - requestYarnContainerIfRequired(container.getResource(), container.getPriority()); + requestYarnContainerIfRequired(); } } else { // return the excessive containers @@ -402,6 +417,36 @@ public void onContainersAllocated(List containers) { }); } + private void removeContainerRequest(AMRMClient.ContainerRequest pendingContainerRequest) { + numPendingContainerRequests--; + + log.info("Removing container request {}. Pending container requests {}.", pendingContainerRequest, numPendingContainerRequests); + + resourceManagerClient.removeContainerRequest(pendingContainerRequest); + } + + private Collection getPendingRequests() { + final List> matchingRequests = resourceManagerClient.getMatchingRequests( + RM_REQUEST_PRIORITY, + ResourceRequest.ANY, + getContainerResource()); + + final Collection matchingContainerRequests; + + if (matchingRequests.isEmpty()) { + matchingContainerRequests = Collections.emptyList(); + } else { + final Collection collection = matchingRequests.get(0); + matchingContainerRequests = new ArrayList<>(collection); + } + + Preconditions.checkState( + matchingContainerRequests.size() == numPendingContainerRequests, + "The RMClient's and YarnResourceManagers internal state about the number of pending container requests has diverged. Number client's pending container requests %s != Number RM's pending container requests %s.", matchingContainerRequests.size(), numPendingContainerRequests); + + return matchingContainerRequests; + } + @Override public void onShutdownRequest() { shutDown(); @@ -456,17 +501,17 @@ private static Tuple2 parseHostPort(String address) { /** * Request new container if pending containers cannot satisfies pending slot requests. */ - private void requestYarnContainerIfRequired(Resource resource, Priority priority) { + private void requestYarnContainerIfRequired() { int requiredTaskManagerSlots = getNumberRequiredTaskManagerSlots(); int pendingTaskManagerSlots = numPendingContainerRequests * numberOfTaskSlots; if (requiredTaskManagerSlots > pendingTaskManagerSlots) { - requestYarnContainer(resource, priority); + requestYarnContainer(); } } - private void requestYarnContainer(Resource resource, Priority priority) { - resourceManagerClient.addContainerRequest(new AMRMClient.ContainerRequest(resource, null, null, priority)); + private void requestYarnContainer() { + resourceManagerClient.addContainerRequest(getContainerRequest()); // make sure we transmit the request fast and receive fast news of granted allocations resourceManagerClient.setHeartbeatInterval(FAST_YARN_HEARTBEAT_INTERVAL_MS); @@ -478,6 +523,16 @@ private void requestYarnContainer(Resource resource, Priority priority) { numPendingContainerRequests); } + @Nonnull + @VisibleForTesting + AMRMClient.ContainerRequest getContainerRequest() { + return new AMRMClient.ContainerRequest( + getContainerResource(), + null, + null, + RM_REQUEST_PRIORITY); + } + private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource, String containerId, String host) throws Exception { // init the ContainerLaunchContext @@ -514,22 +569,4 @@ private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource .put(ENV_FLINK_NODE_ID, host); return taskExecutorLaunchContext; } - - - - /** - * Generate priority by given resource profile. - * Priority is only used for distinguishing request of different resource. - * @param resourceProfile The resource profile of a request - * @return The priority of this resource profile. - */ - private int generatePriority(ResourceProfile resourceProfile) { - if (resourcePriorities.containsKey(resourceProfile)) { - return resourcePriorities.get(resourceProfile); - } else { - int priority = resourcePriorities.size(); - resourcePriorities.put(resourceProfile, priority); - return priority; - } - } } diff --git a/flink-yarn/src/main/resources/log4j.properties b/flink-yarn/src/main/resources/log4j.properties index b2ad0d3c465bcb..e84cd49f01aeaa 100644 --- a/flink-yarn/src/main/resources/log4j.properties +++ b/flink-yarn/src/main/resources/log4j.properties @@ -17,7 +17,7 @@ ################################################################################ # Convenience file for local debugging of the JobManager/TaskManager. -log4j.rootLogger=INFO, console +log4j.rootLogger=OFF, console log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.layout=org.apache.log4j.PatternLayout log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java index 10b2ce97d6fe48..4108249548e56a 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFlinkResourceManagerTest.java @@ -43,6 +43,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.NMClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; @@ -52,7 +54,6 @@ import org.junit.Test; import org.mockito.Matchers; import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import java.util.ArrayList; import java.util.Collections; @@ -69,7 +70,10 @@ import scala.concurrent.duration.FiniteDuration; import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -128,20 +132,6 @@ public void testYarnFlinkResourceManagerJobManagerLostLeadership() throws Except containerList.add(mockContainer); } - doAnswer(new Answer() { - int counter = 0; - @Override - public Object answer(InvocationOnMock invocation) throws Throwable { - if (counter < containerList.size()) { - callbackHandler.onContainersAllocated( - Collections.singletonList( - containerList.get(counter++) - )); - } - return null; - } - }).when(resourceManagerClient).addContainerRequest(Matchers.any(AMRMClient.ContainerRequest.class)); - final CompletableFuture resourceManagerFuture = new CompletableFuture<>(); final CompletableFuture leaderGatewayFuture = new CompletableFuture<>(); @@ -191,6 +181,9 @@ public Object answer(InvocationOnMock invocation) throws Throwable { nodeManagerClient )); + doReturn(Collections.singletonList(Collections.nCopies(numInitialTaskManagers, new AMRMClient.ContainerRequest(Resource.newInstance(1024 * 1024, 1), null, null, Priority.newInstance(0))))) + .when(resourceManagerClient).getMatchingRequests(any(Priority.class), anyString(), any(Resource.class)); + leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); final AkkaActorGateway leader1Gateway = new AkkaActorGateway(leader1, leaderSessionID); @@ -203,6 +196,8 @@ public Object answer(InvocationOnMock invocation) throws Throwable { resourceManagerGateway.tell(new RegisterResourceManagerSuccessful(leader1, Collections.emptyList())); + callbackHandler.onContainersAllocated(containerList); + for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java index d41d42d7a05f97..368e95cc76030d 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java @@ -105,8 +105,10 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -243,7 +245,7 @@ class Context { final HardwareDescription hardwareDescription = new HardwareDescription(1, 2L, 3L, 4L); // domain objects for test purposes - final ResourceProfile resourceProfile1 = new ResourceProfile(1.0, 200); + final ResourceProfile resourceProfile1 = ResourceProfile.UNKNOWN; public ContainerId task = ContainerId.newInstance( ApplicationAttemptId.newInstance(ApplicationId.newInstance(1L, 0), 0), 1); @@ -351,7 +353,7 @@ void runTest(RunnableWithException testMethod) throws Exception { } } - private static Container mockContainer(String host, int port, int containerId) { + private static Container mockContainer(String host, int port, int containerId, Resource resource) { Container mockContainer = mock(Container.class); NodeId mockNodeId = NodeId.newInstance(host, port); @@ -365,7 +367,7 @@ private static Container mockContainer(String host, int port, int containerId) { when(mockContainer.getId()).thenReturn(mockContainerId); when(mockContainer.getNodeId()).thenReturn(mockNodeId); - when(mockContainer.getResource()).thenReturn(Resource.newInstance(200, 1)); + when(mockContainer.getResource()).thenReturn(resource); when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED); return mockContainer; @@ -397,7 +399,10 @@ public void testStopWorker() throws Exception { registerSlotRequestFuture.get(); // Callback from YARN when container is allocated. - Container testingContainer = mockContainer("container", 1234, 1); + Container testingContainer = mockContainer("container", 1234, 1, resourceManager.getContainerResource()); + + doReturn(Collections.singletonList(Collections.singletonList(resourceManager.getContainerRequest()))) + .when(mockResourceManagerClient).getMatchingRequests(any(Priority.class), anyString(), any(Resource.class)); resourceManager.onContainersAllocated(ImmutableList.of(testingContainer)); verify(mockResourceManagerClient).addContainerRequest(any(AMRMClient.ContainerRequest.class)); @@ -492,10 +497,14 @@ public void testOnContainerCompleted() throws Exception { registerSlotRequestFuture.get(); // Callback from YARN when container is allocated. - Container testingContainer = mockContainer("container", 1234, 1); + Container testingContainer = mockContainer("container", 1234, 1, resourceManager.getContainerResource()); + + doReturn(Collections.singletonList(Collections.singletonList(resourceManager.getContainerRequest()))) + .when(mockResourceManagerClient).getMatchingRequests(any(Priority.class), anyString(), any(Resource.class)); resourceManager.onContainersAllocated(ImmutableList.of(testingContainer)); verify(mockResourceManagerClient).addContainerRequest(any(AMRMClient.ContainerRequest.class)); + verify(mockResourceManagerClient).removeContainerRequest(any(AMRMClient.ContainerRequest.class)); verify(mockNMClient).startContainer(eq(testingContainer), any(ContainerLaunchContext.class)); // Callback from YARN when container is Completed, pending request can not be fulfilled by pending diff --git a/flink-yarn/src/test/resources/log4j-test.properties b/flink-yarn/src/test/resources/log4j-test.properties index 2226f68653181a..5b1e4ed8e7a747 100644 --- a/flink-yarn/src/test/resources/log4j-test.properties +++ b/flink-yarn/src/test/resources/log4j-test.properties @@ -18,7 +18,7 @@ # Set root logger level to OFF to not flood build logs # set manually to INFO for debugging purposes -log4j.rootLogger=OFF, testlogger +log4j.rootLogger=INFO, testlogger # A1 is set to be a ConsoleAppender. log4j.appender.testlogger=org.apache.log4j.ConsoleAppender From 43ec81f67e948e06f683b2987a10a8c1248e36b4 Mon Sep 17 00:00:00 2001 From: sunjincheng121 Date: Mon, 14 Jan 2019 09:25:42 +0800 Subject: [PATCH 117/147] [hotfix] [docs] Fix kafka doc mistake classname --- docs/dev/connectors/kafka.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/connectors/kafka.md b/docs/dev/connectors/kafka.md index b7f65b5cd1c986..936d36b86e3b3b 100644 --- a/docs/dev/connectors/kafka.md +++ b/docs/dev/connectors/kafka.md @@ -547,7 +547,7 @@ In the meanwhile, a possible workaround is to send *heartbeat messages* to all c ## Kafka Producer -Flink’s Kafka Producer is called `FlinkKafkaProducer011` (or `010` for Kafka 0.10.0.x versions, etc. or just `FlinkKafkaConsumer` for Kafka >= 1.0.0 versions). +Flink’s Kafka Producer is called `FlinkKafkaProducer011` (or `010` for Kafka 0.10.0.x versions, etc. or just `FlinkKafkaProducer` for Kafka >= 1.0.0 versions). It allows writing a stream of records to one or more Kafka topics. Example: From 6c8abc13a80655470732dc948b2d127aaf2b9802 Mon Sep 17 00:00:00 2001 From: ifndef-SleePy Date: Mon, 14 Jan 2019 14:17:14 +0800 Subject: [PATCH 118/147] [FLINK-11312][docs] Fix broken link to Zookeeper getting started --- docs/ops/jobmanager_high_availability.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ops/jobmanager_high_availability.md b/docs/ops/jobmanager_high_availability.md index 2c2f1a6ba19ed1..82297477435231 100644 --- a/docs/ops/jobmanager_high_availability.md +++ b/docs/ops/jobmanager_high_availability.md @@ -44,7 +44,7 @@ As an example, consider the following setup with three JobManager instances: To enable JobManager High Availability you have to set the **high-availability mode** to *zookeeper*, configure a **ZooKeeper quorum** and set up a **masters file** with all JobManagers hosts and their web UI ports. -Flink leverages **[ZooKeeper](http://zookeeper.apache.org)** for *distributed coordination* between all running JobManager instances. ZooKeeper is a separate service from Flink, which provides highly reliable distributed coordination via leader election and light-weight consistent state storage. Check out [ZooKeeper's Getting Started Guide](http://zookeeper.apache.org/doc/trunk/zookeeperStarted.html) for more information about ZooKeeper. Flink includes scripts to [bootstrap a simple ZooKeeper](#bootstrap-zookeeper) installation. +Flink leverages **[ZooKeeper](http://zookeeper.apache.org)** for *distributed coordination* between all running JobManager instances. ZooKeeper is a separate service from Flink, which provides highly reliable distributed coordination via leader election and light-weight consistent state storage. Check out [ZooKeeper's Getting Started Guide](http://zookeeper.apache.org/doc/current/zookeeperStarted.html) for more information about ZooKeeper. Flink includes scripts to [bootstrap a simple ZooKeeper](#bootstrap-zookeeper) installation. #### Masters File (masters) From f305b64c290f55fcd527de9d848cfbe0ad79bca8 Mon Sep 17 00:00:00 2001 From: intsmaze Date: Mon, 14 Jan 2019 16:42:12 +0800 Subject: [PATCH 119/147] [hotfix] Add dependency snippet for RocksDBStateBackend to state_backends.md In order to use the RocksDBStateBackend, the user has to add the flink-statebackend-rocksdb dependency to his project. This commit adds this information to the state_backends.md. This closes #7480. --- docs/ops/state/state_backends.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/ops/state/state_backends.md b/docs/ops/state/state_backends.md index 70663f696b2159..c1678036875f75 100644 --- a/docs/ops/state/state_backends.md +++ b/docs/ops/state/state_backends.md @@ -146,6 +146,16 @@ env.setStateBackend(new FsStateBackend("hdfs://namenode:40010/flink/checkpoints" +If you want to use the `RocksDBStateBackend`, then you have to add the following dependency to your Flink project. + +{% highlight xml %} + + org.apache.flink + flink-statebackend-rocksdb{{ site.scala_version_suffix }} + {{ site.version }} + +{% endhighlight %} + ### Setting Default State Backend From 7206d20f00790801514b2a894814175ea8c5e372 Mon Sep 17 00:00:00 2001 From: Benchao Li Date: Mon, 14 Jan 2019 18:28:36 +0800 Subject: [PATCH 120/147] [hotfix] Fix typo in restart strategies documenation (#7479) --- docs/dev/restart_strategies.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/restart_strategies.md b/docs/dev/restart_strategies.md index 49419e192de91a..5be430e7f4804f 100644 --- a/docs/dev/restart_strategies.md +++ b/docs/dev/restart_strategies.md @@ -264,7 +264,7 @@ env.setRestartStrategy(RestartStrategies.noRestart()) ### Fallback Restart Strategy The cluster defined restart strategy is used. -This helpful for streaming programs which enable checkpointing. -Per default, a fixed delay restart strategy is chosen if there is no other restart strategy defined. +This is helpful for streaming programs which enable checkpointing. +By default, a fixed delay restart strategy is chosen if there is no other restart strategy defined. {% top %} From c9af982e830fb4b89defdce910cce3bb99000364 Mon Sep 17 00:00:00 2001 From: Yangze Guo Date: Mon, 14 Jan 2019 10:28:08 +0800 Subject: [PATCH 121/147] [hotfix][docs] Fix typo in classloading doc --- docs/monitoring/debugging_classloading.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/monitoring/debugging_classloading.md b/docs/monitoring/debugging_classloading.md index fe2c36d1f14fba..52d15280932682 100644 --- a/docs/monitoring/debugging_classloading.md +++ b/docs/monitoring/debugging_classloading.md @@ -85,7 +85,6 @@ classes are loaded dynamically when the jobs are submitted. In setups where dynamic classloading is involved (sessions), there is a hierarchy of typically two ClassLoaders: (1) Java's *application classloader*, which has all classes in the classpath, and (2) the dynamic *user code classloader*. for loading classes from the user-code jar(s). The user-code ClassLoader has the application classloader as its parent. -cases. By default, Flink inverts classloading order, meaning it looks into the user code classloader first, and only looks into the parent (application classloader) if the class is not part of the dynamically loaded user code. From 2be178c36bb1fae19d3089be43d789c66813f191 Mon Sep 17 00:00:00 2001 From: JaryZhen Date: Mon, 14 Jan 2019 12:30:10 +0800 Subject: [PATCH 122/147] [FLINK-11304][docs][table] Fix typo in time attributes doc This closes #7477. --- docs/dev/table/streaming/time_attributes.md | 24 +++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/dev/table/streaming/time_attributes.md b/docs/dev/table/streaming/time_attributes.md index 101bad68b80822..9522909ae8e3dc 100644 --- a/docs/dev/table/streaming/time_attributes.md +++ b/docs/dev/table/streaming/time_attributes.md @@ -264,7 +264,7 @@ Moreover, the `DataStream` returned by the `getDataStream()` method must have wa
{% highlight java %} // define a table source with a rowtime attribute -public class UserActionSource implements StreamTableSource, DefinedRowtimeAttribute { +public class UserActionSource implements StreamTableSource, DefinedRowtimeAttributes { @Override public TypeInformation getReturnType() { @@ -284,9 +284,15 @@ public class UserActionSource implements StreamTableSource, DefinedRowtimeA } @Override - public String getRowtimeAttribute() { + public List getRowtimeAttributeDescriptors() { // Mark the "UserActionTime" attribute as event-time attribute. - return "UserActionTime"; + // We create one attribute descriptor of "UserActionTime". + RowtimeAttributeDescriptor rowtimeAttrDescr = new RowtimeAttributeDescriptor( + "UserActionTime", + new ExistingField("UserActionTime"), + new AscendingTimestamps()); + List listRowtimeAttrDescr = Collections.singletonList(rowtimeAttrDescr); + return listRowtimeAttrDescr; } } @@ -301,7 +307,7 @@ WindowedTable windowedTable = tEnv
{% highlight scala %} // define a table source with a rowtime attribute -class UserActionSource extends StreamTableSource[Row] with DefinedRowtimeAttribute { +class UserActionSource extends StreamTableSource[Row] with DefinedRowtimeAttributes { override def getReturnType = { val names = Array[String]("Username" , "Data", "UserActionTime") @@ -317,9 +323,15 @@ class UserActionSource extends StreamTableSource[Row] with DefinedRowtimeAttribu stream } - override def getRowtimeAttribute = { + override def getRowtimeAttributeDescriptors: util.List[RowtimeAttributeDescriptor] = { // Mark the "UserActionTime" attribute as event-time attribute. - "UserActionTime" + // We create one attribute descriptor of "UserActionTime". + val rowtimeAttrDescr = new RowtimeAttributeDescriptor( + "UserActionTime", + new ExistingField("UserActionTime"), + new AscendingTimestamps) + val listRowtimeAttrDescr = Collections.singletonList(rowtimeAttrDescr) + listRowtimeAttrDescr } } From 1325599153b162fc85679589cab0c2691bf398f1 Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Mon, 14 Jan 2019 14:05:46 +0100 Subject: [PATCH 123/147] [hotfix][docs][table] Fix more typos in time attributes doc --- docs/dev/table/streaming/time_attributes.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/dev/table/streaming/time_attributes.md b/docs/dev/table/streaming/time_attributes.md index 9522909ae8e3dc..9014d2a0007f14 100644 --- a/docs/dev/table/streaming/time_attributes.md +++ b/docs/dev/table/streaming/time_attributes.md @@ -256,9 +256,12 @@ val windowedTable = table.window(Tumble over 10.minutes on 'UserActionTime as 'u ### Using a TableSource -The event time attribute is defined by a `TableSource` that implements the `DefinedRowtimeAttribute` interface. The `getRowtimeAttribute()` method returns the name of an existing field that carries the event time attribute of the table and is of type `LONG` or `TIMESTAMP`. +The event time attribute is defined by a `TableSource` that implements the `DefinedRowtimeAttributes` interface. The `getRowtimeAttributeDescriptors()` method returns a list of `RowtimeAttributeDescriptor` for describing the final name of a time attribute, a timestamp extractor to derive the values of the attribute, and the watermark strategy associated with the attribute. -Moreover, the `DataStream` returned by the `getDataStream()` method must have watermarks assigned that are aligned with the defined time attribute. Please note that the timestamps of the `DataStream` (the ones which are assigned by a `TimestampAssigner`) are ignored. Only the values of the `TableSource`'s rowtime attribute are relevant. +Please make sure that the `DataStream` returned by the `getDataStream()` method is aligned with the defined time attribute. +The timestamps of the `DataStream` (the ones which are assigned by a `TimestampAssigner`) are only considered if a `StreamRecordTimestamp` timestamp extractor is defined. +Watermarks of a `DataStream` are only preserved if a `PreserveWatermarks` watermark strategy is defined. +Otherwise, only the values of the `TableSource`'s rowtime attribute are relevant.
From e5e6274f962497e3c3594d4a6f5fe686bb06587e Mon Sep 17 00:00:00 2001 From: JaryZhen Date: Tue, 15 Jan 2019 12:52:40 +0800 Subject: [PATCH 124/147] [hotfix] [docs] Fix typo in Table SQL doc This closes #7491. --- docs/dev/table/sourceSinks.md | 2 +- docs/dev/table/sql.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/table/sourceSinks.md b/docs/dev/table/sourceSinks.md index ab58a725b2ae48..483ef20e64ef4d 100644 --- a/docs/dev/table/sourceSinks.md +++ b/docs/dev/table/sourceSinks.md @@ -205,7 +205,7 @@ DefinedRowtimeAttributes { * `getRowtimeAttributeDescriptors()`: Returns a list of `RowtimeAttributeDescriptor`. A `RowtimeAttributeDescriptor` describes a rowtime attribute with the following properties: * `attributeName`: The name of the rowtime attribute in the table schema. The field must be defined with type `Types.SQL_TIMESTAMP`. - * `timestampExtractor`: The timestamp extractor extracts the timestamp from a record with the return type. For example, it can convert convert a Long field into a timestamp or parse a String-encoded timestamp. Flink comes with a set of built-in `TimestampExtractor` implementation for common use cases. It is also possible to provide a custom implementation. + * `timestampExtractor`: The timestamp extractor extracts the timestamp from a record with the return type. For example, it can convert a Long field into a timestamp or parse a String-encoded timestamp. Flink comes with a set of built-in `TimestampExtractor` implementation for common use cases. It is also possible to provide a custom implementation. * `watermarkStrategy`: The watermark strategy defines how watermarks are generated for the rowtime attribute. Flink comes with a set of built-in `WatermarkStrategy` implementations for common use cases. It is also possible to provide a custom implementation. Attention Although the `getRowtimeAttributeDescriptors()` method returns a list of descriptors, only a single rowtime attribute is support at the moment. We plan to remove this restriction in the future and support tables with more than one rowtime attribute. diff --git a/docs/dev/table/sql.md b/docs/dev/table/sql.md index 1e7ef83fcb7237..14df44131f9621 100644 --- a/docs/dev/table/sql.md +++ b/docs/dev/table/sql.md @@ -93,7 +93,7 @@ val result2 = tableEnv.sqlQuery( // SQL update with a registered table // create and register a TableSink -TableSink csvSink = new CsvTableSink("/path/to/file", ...) +val csvSink: CsvTableSink = new CsvTableSink("/path/to/file", ...) val fieldNames: Array[String] = Array("product", "amount") val fieldTypes: Array[TypeInformation[_]] = Array(Types.STRING, Types.INT) tableEnv.registerTableSink("RubberOrders", fieldNames, fieldTypes, csvSink) From baea9a35a9dc209229fae043991b394c141b1265 Mon Sep 17 00:00:00 2001 From: leesf <490081539@qq.com> Date: Mon, 14 Jan 2019 23:24:56 +0800 Subject: [PATCH 125/147] [hotfix] [datastream] Remove irrelevant comments and fix typo in SourceStreamTask This closes #7485. --- .../flink/streaming/runtime/tasks/SourceStreamTask.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java index 644d57c8abcee2..8bd3534f44f067 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTask.java @@ -53,10 +53,8 @@ public SourceStreamTask(Environment env) { @Override protected void init() { - // does not hold any resources, so no initialization needed - // we check if the source is actually inducing the checkpoints, rather - // than the trigger ch + // than the trigger SourceFunction source = headOperator.getUserFunction(); if (source instanceof ExternallyInducedSource) { externallyInducedCheckpoints = true; From e7b02b42364563dfa20c8189da50d1a0a7529f4a Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 14 Jan 2019 14:54:32 +0100 Subject: [PATCH 126/147] [hotfix] [fs] Add clock as an argument in Javadocs The clock parameter is missing in the docstring. This closes #7484. --- .../apache/flink/streaming/connectors/fs/bucketing/Bucketer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/Bucketer.java b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/Bucketer.java index f2eebf377efcb9..e712b2e69b9565 100644 --- a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/Bucketer.java +++ b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/Bucketer.java @@ -38,6 +38,7 @@ public interface Bucketer extends Serializable { /** * Returns the {@link Path} of a bucket file. * + * @param clock The current system time in milliseconds. * @param basePath The base path containing all the buckets. * @param element The current element being processed. * From 02078e0e8830877d774b17923436bf5102490f4c Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 15 Dec 2018 06:37:20 +0800 Subject: [PATCH 127/147] [FLINK-11156][tests, runtime] Reconcile ZooKeeperCompletedCheckpointStoreMockitoTest with JDK 9 --- .../ZooKeeperCompletedCheckpointStore.java | 76 ++++++++++++++----- ...erCompletedCheckpointStoreMockitoTest.java | 22 ++---- 2 files changed, 63 insertions(+), 35 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java index 51f40085c3fa3b..e87ca36233178c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.checkpoint; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobmanager.HighAvailabilityMode; @@ -32,6 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayDeque; import java.util.ArrayList; @@ -107,34 +109,70 @@ public class ZooKeeperCompletedCheckpointStore implements CompletedCheckpointSto * @throws Exception */ public ZooKeeperCompletedCheckpointStore( - int maxNumberOfCheckpointsToRetain, - CuratorFramework client, - String checkpointsPath, - RetrievableStateStorageHelper stateStorage, - Executor executor) throws Exception { + int maxNumberOfCheckpointsToRetain, + CuratorFramework client, + String checkpointsPath, + RetrievableStateStorageHelper stateStorage, + Executor executor + ) throws Exception { + this(maxNumberOfCheckpointsToRetain, + adaptNameSpace(client, checkpointsPath), + stateStorage, + executor); - checkArgument(maxNumberOfCheckpointsToRetain >= 1, "Must retain at least one checkpoint."); - checkNotNull(stateStorage, "State storage"); - - this.maxNumberOfCheckpointsToRetain = maxNumberOfCheckpointsToRetain; + LOG.info("Initialized in '{}'.", checkpointsPath); + } - checkNotNull(client, "Curator client"); - checkNotNull(checkpointsPath, "Checkpoints path"); + @VisibleForTesting + ZooKeeperCompletedCheckpointStore( + int maxNumberOfCheckpointsToRetain, + CuratorFramework client, + String checkpointsPath, + Executor executor, + ZooKeeperStateHandleStore checkpointsInZooKeeper + ) throws Exception { + this(maxNumberOfCheckpointsToRetain, + adaptNameSpace(client, checkpointsPath), + executor, + checkpointsInZooKeeper); - // Ensure that the checkpoints path exists - client.newNamespaceAwareEnsurePath(checkpointsPath) - .ensure(client.getZookeeperClient()); + LOG.info("Initialized in '{}'.", checkpointsPath); + } - // All operations will have the path as root - this.client = client.usingNamespace(client.getNamespace() + checkpointsPath); + private ZooKeeperCompletedCheckpointStore( + int maxNumberOfCheckpointsToRetain, + CuratorFramework client, + RetrievableStateStorageHelper stateStorage, + Executor executor + ) { + this(maxNumberOfCheckpointsToRetain, + client, + executor, + new ZooKeeperStateHandleStore<>(client, stateStorage)); + } - this.checkpointsInZooKeeper = new ZooKeeperStateHandleStore<>(this.client, stateStorage); + private ZooKeeperCompletedCheckpointStore( + int maxNumberOfCheckpointsToRetain, + @Nonnull CuratorFramework client, + @Nonnull Executor executor, + @Nonnull ZooKeeperStateHandleStore checkpointsInZooKeeper + ) { + checkArgument(maxNumberOfCheckpointsToRetain >= 1, "Must retain at least one checkpoint."); + this.maxNumberOfCheckpointsToRetain = maxNumberOfCheckpointsToRetain; + this.client = client; + this.executor = executor; + this.checkpointsInZooKeeper = checkpointsInZooKeeper; this.completedCheckpoints = new ArrayDeque<>(maxNumberOfCheckpointsToRetain + 1); + } - this.executor = checkNotNull(executor); + private static CuratorFramework adaptNameSpace(CuratorFramework client, String checkpointsPath) throws Exception { + // Ensure that the checkpoints path exists + client.newNamespaceAwareEnsurePath(checkpointsPath) + .ensure(client.getZookeeperClient()); - LOG.info("Initialized in '{}'.", checkpointsPath); + // All operations will have the path as root + return client.usingNamespace(client.getNamespace() + checkpointsPath); } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreMockitoTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreMockitoTest.java index e9b90b7261140a..5fe0f41fc52722 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreMockitoTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreMockitoTest.java @@ -34,12 +34,9 @@ import org.apache.curator.framework.api.ErrorListenerPathable; import org.apache.curator.utils.EnsurePath; import org.junit.Test; -import org.junit.runner.RunWith; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; import java.io.IOException; import java.util.ArrayList; @@ -55,21 +52,18 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.powermock.api.mockito.PowerMockito.doAnswer; -import static org.powermock.api.mockito.PowerMockito.doThrow; -import static org.powermock.api.mockito.PowerMockito.whenNew; /** * Mockito based tests for the {@link ZooKeeperStateHandleStore}. */ -@RunWith(PowerMockRunner.class) -@PrepareForTest(ZooKeeperCompletedCheckpointStore.class) public class ZooKeeperCompletedCheckpointStoreMockitoTest extends TestLogger { /** @@ -125,7 +119,6 @@ public void testCheckpointRecovery() throws Exception { final RetrievableStateStorageHelper storageHelperMock = mock(RetrievableStateStorageHelper.class); ZooKeeperStateHandleStore zooKeeperStateHandleStoreMock = spy(new ZooKeeperStateHandleStore<>(client, storageHelperMock)); - whenNew(ZooKeeperStateHandleStore.class).withAnyArguments().thenReturn(zooKeeperStateHandleStoreMock); doReturn(checkpointsInZooKeeper).when(zooKeeperStateHandleStoreMock).getAllAndLock(); final int numCheckpointsToRetain = 1; @@ -165,14 +158,13 @@ public Void answer(InvocationOnMock invocation) throws Throwable { }); final String checkpointsPath = "foobar"; - final RetrievableStateStorageHelper stateStorage = mock(RetrievableStateStorageHelper.class); ZooKeeperCompletedCheckpointStore zooKeeperCompletedCheckpointStore = new ZooKeeperCompletedCheckpointStore( numCheckpointsToRetain, client, checkpointsPath, - stateStorage, - Executors.directExecutor()); + Executors.directExecutor(), + zooKeeperStateHandleStoreMock); zooKeeperCompletedCheckpointStore.recover(); @@ -213,7 +205,6 @@ public void testAddCheckpointWithFailedRemove() throws Exception { ZooKeeperStateHandleStore zookeeperStateHandleStoreMock = spy(new ZooKeeperStateHandleStore<>(client, storageHelperMock)); - whenNew(ZooKeeperStateHandleStore.class).withAnyArguments().thenReturn(zookeeperStateHandleStoreMock); doAnswer(new Answer>() { @Override @@ -231,14 +222,13 @@ public RetrievableStateHandle answer(InvocationOnMock invoc final int numCheckpointsToRetain = 1; final String checkpointsPath = "foobar"; - final RetrievableStateStorageHelper stateSotrage = mock(RetrievableStateStorageHelper.class); ZooKeeperCompletedCheckpointStore zooKeeperCompletedCheckpointStore = new ZooKeeperCompletedCheckpointStore( numCheckpointsToRetain, client, checkpointsPath, - stateSotrage, - Executors.directExecutor()); + Executors.directExecutor(), + zookeeperStateHandleStoreMock); for (long i = 0; i <= numCheckpointsToRetain; ++i) { CompletedCheckpoint checkpointToAdd = mock(CompletedCheckpoint.class); From eff261efee8c99a0ecc88c06aa07db9a553e6bdd Mon Sep 17 00:00:00 2001 From: Gary Yao Date: Tue, 8 Jan 2019 21:02:16 +0100 Subject: [PATCH 128/147] [FLINK-11156][tests, runtime] Simplify ZooKeeperCompletedCheckpointStore constructor --- .../ZooKeeperCompletedCheckpointStore.java | 88 +++---------------- .../flink/runtime/util/ZooKeeperUtils.java | 48 +++++++++- .../zookeeper/ZooKeeperStateHandleStore.java | 12 +++ .../zookeeper/ZooKeeperUtilityFactory.java | 11 +-- ...oKeeperCompletedCheckpointStoreITCase.java | 10 ++- ...erCompletedCheckpointStoreMockitoTest.java | 15 +--- ...ZooKeeperCompletedCheckpointStoreTest.java | 10 ++- .../ZooKeeperStateHandleStoreTest.java | 15 ++++ 8 files changed, 104 insertions(+), 105 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java index e87ca36233178c..12cc8eb66abada 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java @@ -18,22 +18,17 @@ package org.apache.flink.runtime.checkpoint; -import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobmanager.HighAvailabilityMode; import org.apache.flink.runtime.state.RetrievableStateHandle; -import org.apache.flink.runtime.zookeeper.RetrievableStateStorageHelper; import org.apache.flink.runtime.zookeeper.ZooKeeperStateHandleStore; import org.apache.flink.util.FlinkException; import org.apache.flink.util.function.ThrowingConsumer; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.utils.ZKPaths; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayDeque; import java.util.ArrayList; @@ -75,9 +70,6 @@ public class ZooKeeperCompletedCheckpointStore implements CompletedCheckpointSto private static final Comparator, String>> STRING_COMPARATOR = Comparator.comparing(o -> o.f1); - /** Curator ZooKeeper client. */ - private final CuratorFramework client; - /** Completed checkpoints in ZooKeeper. */ private final ZooKeeperStateHandleStore checkpointsInZooKeeper; @@ -100,79 +92,23 @@ public class ZooKeeperCompletedCheckpointStore implements CompletedCheckpointSto * least 1). Adding more checkpoints than this results * in older checkpoints being discarded. On recovery, * we will only start with a single checkpoint. - * @param client The Curator ZooKeeper client - * @param checkpointsPath The ZooKeeper path for the checkpoints (needs to - * start with a '/') - * @param stateStorage State storage to be used to persist the completed - * checkpoint - * @param executor to execute blocking calls - * @throws Exception + * @param checkpointsInZooKeeper Completed checkpoints in ZooKeeper + * @param executor to execute blocking calls */ public ZooKeeperCompletedCheckpointStore( - int maxNumberOfCheckpointsToRetain, - CuratorFramework client, - String checkpointsPath, - RetrievableStateStorageHelper stateStorage, - Executor executor - ) throws Exception { - this(maxNumberOfCheckpointsToRetain, - adaptNameSpace(client, checkpointsPath), - stateStorage, - executor); - - LOG.info("Initialized in '{}'.", checkpointsPath); - } - - @VisibleForTesting - ZooKeeperCompletedCheckpointStore( - int maxNumberOfCheckpointsToRetain, - CuratorFramework client, - String checkpointsPath, - Executor executor, - ZooKeeperStateHandleStore checkpointsInZooKeeper - ) throws Exception { - this(maxNumberOfCheckpointsToRetain, - adaptNameSpace(client, checkpointsPath), - executor, - checkpointsInZooKeeper); - - LOG.info("Initialized in '{}'.", checkpointsPath); - } - - private ZooKeeperCompletedCheckpointStore( - int maxNumberOfCheckpointsToRetain, - CuratorFramework client, - RetrievableStateStorageHelper stateStorage, - Executor executor - ) { - this(maxNumberOfCheckpointsToRetain, - client, - executor, - new ZooKeeperStateHandleStore<>(client, stateStorage)); - } + int maxNumberOfCheckpointsToRetain, + ZooKeeperStateHandleStore checkpointsInZooKeeper, + Executor executor) { - private ZooKeeperCompletedCheckpointStore( - int maxNumberOfCheckpointsToRetain, - @Nonnull CuratorFramework client, - @Nonnull Executor executor, - @Nonnull ZooKeeperStateHandleStore checkpointsInZooKeeper - ) { checkArgument(maxNumberOfCheckpointsToRetain >= 1, "Must retain at least one checkpoint."); this.maxNumberOfCheckpointsToRetain = maxNumberOfCheckpointsToRetain; - this.client = client; - this.executor = executor; - this.checkpointsInZooKeeper = checkpointsInZooKeeper; - this.completedCheckpoints = new ArrayDeque<>(maxNumberOfCheckpointsToRetain + 1); - } - private static CuratorFramework adaptNameSpace(CuratorFramework client, String checkpointsPath) throws Exception { - // Ensure that the checkpoints path exists - client.newNamespaceAwareEnsurePath(checkpointsPath) - .ensure(client.getZookeeperClient()); + this.checkpointsInZooKeeper = checkNotNull(checkpointsInZooKeeper); + + this.completedCheckpoints = new ArrayDeque<>(maxNumberOfCheckpointsToRetain + 1); - // All operations will have the path as root - return client.usingNamespace(client.getNamespace() + checkpointsPath); + this.executor = checkNotNull(executor); } @Override @@ -345,11 +281,7 @@ public void shutdown(JobStatus jobStatus) throws Exception { } completedCheckpoints.clear(); - - String path = "/" + client.getNamespace(); - - LOG.info("Removing {} from ZooKeeper", path); - ZKPaths.deleteChildren(client.getZookeeperClient().getZooKeeper(), path, true); + checkpointsInZooKeeper.deleteChildren(); } else { LOG.info("Suspending"); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java index 039dcf41d8d999..86b7e6b1d11b87 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java @@ -294,12 +294,31 @@ public static CompletedCheckpointStore createCompletedCheckpoints( checkpointsPath += ZooKeeperSubmittedJobGraphStore.getPathForJob(jobId); - return new ZooKeeperCompletedCheckpointStore( + final ZooKeeperCompletedCheckpointStore zooKeeperCompletedCheckpointStore = new ZooKeeperCompletedCheckpointStore( maxNumberOfCheckpointsToRetain, - client, - checkpointsPath, - stateStorage, + createZooKeeperStateHandleStore(client, checkpointsPath, stateStorage), executor); + + LOG.info("Initialized {} in '{}'.", ZooKeeperCompletedCheckpointStore.class.getSimpleName(), checkpointsPath); + return zooKeeperCompletedCheckpointStore; + } + + /** + * Creates an instance of {@link ZooKeeperStateHandleStore}. + * + * @param client ZK client + * @param path Path to use for the client namespace + * @param stateStorage RetrievableStateStorageHelper that persist the actual state and whose + * returned state handle is then written to ZooKeeper + * @param Type of state + * @return {@link ZooKeeperStateHandleStore} instance + * @throws Exception ZK errors + */ + public static ZooKeeperStateHandleStore createZooKeeperStateHandleStore( + final CuratorFramework client, + final String path, + final RetrievableStateStorageHelper stateStorage) throws Exception { + return new ZooKeeperStateHandleStore<>(useNamespaceAndEnsurePath(client, path), stateStorage); } /** @@ -362,6 +381,27 @@ public static String generateZookeeperPath(String root, String namespace) { return root + namespace; } + /** + * Returns a facade of the client that uses the specified namespace, and ensures that all nodes + * in the path exist. + * + * @param client ZK client + * @param path the new namespace + * @return ZK Client that uses the new namespace + * @throws Exception ZK errors + */ + public static CuratorFramework useNamespaceAndEnsurePath(final CuratorFramework client, final String path) throws Exception { + Preconditions.checkNotNull(client, "client must not be null"); + Preconditions.checkNotNull(path, "path must not be null"); + + // Ensure that the checkpoints path exists + client.newNamespaceAwareEnsurePath(path) + .ensure(client.getZookeeperClient()); + + // All operations will have the path as root + return client.usingNamespace(generateZookeeperPath(client.getNamespace(), path)); + } + /** * Secure {@link ACLProvider} implementation. */ diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/ZooKeeperStateHandleStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/ZooKeeperStateHandleStore.java index 2cb1ccc5071606..62630104942c85 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/ZooKeeperStateHandleStore.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/ZooKeeperStateHandleStore.java @@ -24,6 +24,7 @@ import org.apache.flink.util.InstantiationUtil; import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.utils.ZKPaths; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; @@ -424,6 +425,17 @@ public void releaseAll() throws Exception { } } + /** + * Recursively deletes all children. + * + * @throws Exception ZK errors + */ + public void deleteChildren() throws Exception { + final String path = "/" + client.getNamespace(); + LOG.info("Removing {} from ZooKeeper", path); + ZKPaths.deleteChildren(client.getZookeeperClient().getZooKeeper(), path, true); + } + // --------------------------------------------------------------------------------------------------------- // Protected methods // --------------------------------------------------------------------------------------------------------- diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/ZooKeeperUtilityFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/ZooKeeperUtilityFactory.java index 3e294e0dbddea7..3a9ae9228679e3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/ZooKeeperUtilityFactory.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/ZooKeeperUtilityFactory.java @@ -80,13 +80,10 @@ public ZooKeeperStateHandleStore createZooKeeperStat String zkStateHandleStorePath, RetrievableStateStorageHelper stateStorageHelper) throws Exception { - facade.newNamespaceAwareEnsurePath(zkStateHandleStorePath).ensure(facade.getZookeeperClient()); - CuratorFramework stateHandleStoreFacade = facade.usingNamespace( - ZooKeeperUtils.generateZookeeperPath( - facade.getNamespace(), - zkStateHandleStorePath)); - - return new ZooKeeperStateHandleStore<>(stateHandleStoreFacade, stateStorageHelper); + return ZooKeeperUtils.createZooKeeperStateHandleStore( + facade, + zkStateHandleStorePath, + stateStorageHelper); } /** diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java index f493d6f967b404..c0c18773e6f301 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreITCase.java @@ -22,7 +22,9 @@ import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.state.RetrievableStateHandle; import org.apache.flink.runtime.state.SharedStateRegistry; +import org.apache.flink.runtime.util.ZooKeeperUtils; import org.apache.flink.runtime.zookeeper.RetrievableStateStorageHelper; +import org.apache.flink.runtime.zookeeper.ZooKeeperStateHandleStore; import org.apache.flink.runtime.zookeeper.ZooKeeperTestEnvironment; import org.apache.curator.framework.CuratorFramework; @@ -67,10 +69,14 @@ public void cleanUp() throws Exception { @Override protected ZooKeeperCompletedCheckpointStore createCompletedCheckpoints(int maxNumberOfCheckpointsToRetain) throws Exception { - return new ZooKeeperCompletedCheckpointStore(maxNumberOfCheckpointsToRetain, + final ZooKeeperStateHandleStore checkpointsInZooKeeper = ZooKeeperUtils.createZooKeeperStateHandleStore( ZOOKEEPER.getClient(), CHECKPOINT_PATH, - new HeapStateStorageHelper(), + new TestingRetrievableStateStorageHelper<>()); + + return new ZooKeeperCompletedCheckpointStore( + maxNumberOfCheckpointsToRetain, + checkpointsInZooKeeper, Executors.directExecutor()); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreMockitoTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreMockitoTest.java index 5fe0f41fc52722..c8fcfb90711d7f 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreMockitoTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreMockitoTest.java @@ -157,14 +157,10 @@ public Void answer(InvocationOnMock invocation) throws Throwable { } }); - final String checkpointsPath = "foobar"; - ZooKeeperCompletedCheckpointStore zooKeeperCompletedCheckpointStore = new ZooKeeperCompletedCheckpointStore( numCheckpointsToRetain, - client, - checkpointsPath, - Executors.directExecutor(), - zooKeeperStateHandleStoreMock); + zooKeeperStateHandleStoreMock, + Executors.directExecutor()); zooKeeperCompletedCheckpointStore.recover(); @@ -221,14 +217,11 @@ public RetrievableStateHandle answer(InvocationOnMock invoc doThrow(new Exception()).when(zookeeperStateHandleStoreMock).releaseAndTryRemove(anyString()); final int numCheckpointsToRetain = 1; - final String checkpointsPath = "foobar"; ZooKeeperCompletedCheckpointStore zooKeeperCompletedCheckpointStore = new ZooKeeperCompletedCheckpointStore( numCheckpointsToRetain, - client, - checkpointsPath, - Executors.directExecutor(), - zookeeperStateHandleStoreMock); + zookeeperStateHandleStoreMock, + Executors.directExecutor()); for (long i = 0; i <= numCheckpointsToRetain; ++i) { CompletedCheckpoint checkpointToAdd = mock(CompletedCheckpoint.class); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java index a9cba8861c044a..bff6779336f088 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java @@ -25,6 +25,7 @@ import org.apache.flink.runtime.state.SharedStateRegistry; import org.apache.flink.runtime.util.ZooKeeperUtils; import org.apache.flink.runtime.zookeeper.ZooKeeperResource; +import org.apache.flink.runtime.zookeeper.ZooKeeperStateHandleStore; import org.apache.flink.util.TestLogger; import org.apache.curator.framework.CuratorFramework; @@ -117,11 +118,14 @@ public void testDiscardingCheckpointsAtShutDown() throws Exception { @Nonnull private ZooKeeperCompletedCheckpointStore createZooKeeperCheckpointStore(CuratorFramework client) throws Exception { - return new ZooKeeperCompletedCheckpointStore( - 1, + final ZooKeeperStateHandleStore checkpointsInZooKeeper = ZooKeeperUtils.createZooKeeperStateHandleStore( client, "/checkpoints", - new TestingRetrievableStateStorageHelper<>(), + new TestingRetrievableStateStorageHelper<>()); + + return new ZooKeeperCompletedCheckpointStore( + 1, + checkpointsInZooKeeper, Executors.directExecutor()); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/zookeeper/ZooKeeperStateHandleStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/zookeeper/ZooKeeperStateHandleStoreTest.java index 3c37faef3aae07..161b3b8b57e654 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/zookeeper/ZooKeeperStateHandleStoreTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/zookeeper/ZooKeeperStateHandleStoreTest.java @@ -42,10 +42,13 @@ import java.util.List; import java.util.Set; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.spy; @@ -685,6 +688,18 @@ public void testReleaseAll() throws Exception { assertEquals(0, stat.getNumChildren()); } + @Test + public void testDeleteAllShouldRemoveAllPaths() throws Exception { + final ZooKeeperStateHandleStore zkStore = new ZooKeeperStateHandleStore<>( + ZooKeeperUtils.useNamespaceAndEnsurePath(ZOOKEEPER.getClient(), "/path"), + new LongStateStorage()); + + zkStore.addAndLock("/state", 1L); + zkStore.deleteChildren(); + + assertThat(zkStore.getAllPaths(), is(empty())); + } + // --------------------------------------------------------------------------------------------- // Simple test helpers // --------------------------------------------------------------------------------------------- From 384692866113ae8f303c1ffcf206aca950c7d4d6 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Tue, 15 Jan 2019 14:31:21 +0100 Subject: [PATCH 129/147] Add link to KeyedProcessFunction to side output documentation --- docs/dev/stream/side_output.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/dev/stream/side_output.md b/docs/dev/stream/side_output.md index 4632995c72e459..7ca195d7022472 100644 --- a/docs/dev/stream/side_output.md +++ b/docs/dev/stream/side_output.md @@ -58,6 +58,7 @@ contains. Emitting data to a side output is possible from the following functions: - [ProcessFunction]({{ site.baseurl }}/dev/stream/operators/process_function.html) +- [KeyedProcessFunction]({{ site.baseurl }}/dev/stream/operators/process_function.html#the-keyedprocessfunction) - CoProcessFunction - [ProcessWindowFunction]({{ site.baseurl }}/dev/stream/operators/windows.html#processwindowfunction) - ProcessAllWindowFunction From ea0f283ffdbb492ea7f2d22e79c57ca7ae6ca181 Mon Sep 17 00:00:00 2001 From: hequn8128 Date: Tue, 15 Jan 2019 18:31:57 +0800 Subject: [PATCH 130/147] [FLINK-11331][table][docs] Fix errors in tableApi.md and functions.md This closes #7494 --- docs/dev/table/functions.md | 52 ++++++++++++++++++++++++++++++++++++- docs/dev/table/tableApi.md | 4 +-- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/docs/dev/table/functions.md b/docs/dev/table/functions.md index 682f519eb3ce62..f41139d14957c7 100644 --- a/docs/dev/table/functions.md +++ b/docs/dev/table/functions.md @@ -3573,7 +3573,18 @@ NUMERIC.years

Creates an interval of months for NUMERIC years.

- + + + {% highlight java %} +NUMERIC.quarter +NUMERIC.quarters +{% endhighlight %} + + +

Creates an interval of months for NUMERIC quarters.

+

E.g., 2.quarters returns 6.

+ + {% highlight java %} @@ -3586,6 +3597,19 @@ NUMERIC.months + + + {% highlight java %} +NUMERIC.week +NUMERIC.weeks +{% endhighlight %} + + +

Creates an interval of milliseconds for NUMERIC weeks.

+

E.g., 2.weeks returns 1209600000.

+ + + {% highlight java %} @@ -3831,6 +3855,19 @@ NUMERIC.years + + + {% highlight scala %} +NUMERIC.quarter +NUMERIC.quarters +{% endhighlight %} + + +

Creates an interval of months for NUMERIC quarters.

+

E.g., 2.quarters returns 6.

+ + + {% highlight scala %} @@ -3843,6 +3880,19 @@ NUMERIC.months + + + {% highlight scala %} +NUMERIC.week +NUMERIC.weeks +{% endhighlight %} + + +

Creates an interval of milliseconds for NUMERIC weeks.

+

E.g., 2.weeks returns 1209600000.

+ + + {% highlight scala %} diff --git a/docs/dev/table/tableApi.md b/docs/dev/table/tableApi.md index e44df24ad5ec96..d44eebddf53519 100644 --- a/docs/dev/table/tableApi.md +++ b/docs/dev/table/tableApi.md @@ -109,7 +109,7 @@ Table orders = tEnv.scan("Orders"); // schema (a, b, c, rowtime) Table result = orders .filter("a.isNotNull && b.isNotNull && c.isNotNull") - .select("a.lowerCase(), b, rowtime") + .select("a.lowerCase() as a, b, rowtime") .window(Tumble.over("1.hour").on("rowtime").as("hourlyWindow")) .groupBy("hourlyWindow, a") .select("a, hourlyWindow.end as hour, b.avg as avgBillingAmount"); @@ -128,7 +128,7 @@ val orders: Table = tEnv.scan("Orders") // schema (a, b, c, rowtime) val result: Table = orders .filter('a.isNotNull && 'b.isNotNull && 'c.isNotNull) - .select('a.lowerCase(), 'b, 'rowtime) + .select('a.lowerCase() as 'a, 'b, 'rowtime) .window(Tumble over 1.hour on 'rowtime as 'hourlyWindow) .groupBy('hourlyWindow, 'a) .select('a, 'hourlyWindow.end as 'hour, 'b.avg as 'avgBillingAmount) From ecb57a913e3284e96ddec29811d5836187294c5a Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Mon, 23 Apr 2018 18:58:15 +0200 Subject: [PATCH 131/147] [hotfix][docs] fix some typos --- docs/dev/projectsetup/scala_api_quickstart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/projectsetup/scala_api_quickstart.md b/docs/dev/projectsetup/scala_api_quickstart.md index bcb404591fa367..da38eb7cdd6219 100644 --- a/docs/dev/projectsetup/scala_api_quickstart.md +++ b/docs/dev/projectsetup/scala_api_quickstart.md @@ -30,7 +30,7 @@ under the License. ## Build Tools Flink projects can be built with different build tools. -In order to get quickly started, Flink provides project templates for the following build tools: +In order to get started quickly, Flink provides project templates for the following build tools: - [SBT](#sbt) - [Maven](#maven) From bfd13024cab0a561fe3246845e6e0901d7a04cd4 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Mon, 23 Apr 2018 18:58:28 +0200 Subject: [PATCH 132/147] [FLINK-9222][docs] add documentation for setting up Gradle projects --- docs/_config.yml | 2 + docs/dev/projectsetup/java_api_quickstart.md | 228 ++++++++++++++++++- 2 files changed, 223 insertions(+), 7 deletions(-) diff --git a/docs/_config.yml b/docs/_config.yml index aea41bd4729306..5a83c9a95e4e45 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -34,6 +34,8 @@ version_title: "1.8-SNAPSHOT" # Branch on Github for this version github_branch: "master" +# Plain Scala version is needed for e.g. the Gradle quickstart. +scala_version: "2.11" # This suffix is appended to the Scala-dependent Maven artifact names scala_version_suffix: "_2.11" diff --git a/docs/dev/projectsetup/java_api_quickstart.md b/docs/dev/projectsetup/java_api_quickstart.md index 03d75c130720bb..91ef70e9326ced 100644 --- a/docs/dev/projectsetup/java_api_quickstart.md +++ b/docs/dev/projectsetup/java_api_quickstart.md @@ -26,14 +26,24 @@ under the License. * This will be replaced by the TOC {:toc} -Start working on your Flink Java program in a few simple steps. +## Build Tools -## Requirements +Flink projects can be built with different build tools. +In order to get started quickly, Flink provides project templates for the following build tools: + +- [Maven](#maven) +- [Gradle](#gradle) + +These templates help you to set up the project structure and to create the initial build files. + +## Maven + +### Requirements The only requirements are working __Maven 3.0.4__ (or higher) and __Java 8.x__ installations. -## Create Project +### Create Project Use one of the following commands to __create a project__: @@ -69,7 +79,7 @@ Use one of the following commands to __create a project__: {% endunless %}
-## Inspect Project +### Inspect Project There will be a new directory in your working directory. If you've used the _curl_ approach, the directory is called `quickstart`. Otherwise, @@ -108,7 +118,7 @@ In Eclipse, choose box: `-Xmx800m`. In IntelliJ IDEA recommended way to change JVM options is from the `Help | Edit Custom VM Options` menu. See [this article](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties) for details. -## Build Project +### Build Project If you want to __build/package your project__, go to your project directory and run the '`mvn clean package`' command. @@ -116,8 +126,212 @@ You will __find a JAR file__ that contains your application, plus connectors and that you may have added as dependencies to the application: `target/-.jar`. __Note:__ If you use a different class than *StreamingJob* as the application's main class / entry point, -we recommend you change the `mainClass` setting in the `pom.xml` file accordingly. That way, the Flink -can run time application from the JAR file without additionally specifying the main class. +we recommend you change the `mainClass` setting in the `pom.xml` file accordingly. That way, Flink +can run the application from the JAR file without additionally specifying the main class. + +## Gradle + +### Requirements + +The only requirements are working __Gradle 3.x__ (or higher) and __Java 8.x__ installations. + +### Create Project + +Use one of the following commands to __create a project__: + + +
+
+ + +
+ +
+ {% highlight gradle %} +buildscript { + repositories { + jcenter() // this applies only to the Gradle 'Shadow' plugin + } + dependencies { + classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4' + } +} + +plugins { + id 'java' + id 'application' + // shadow plugin to produce fat JARs + id 'com.github.johnrengelman.shadow' version '2.0.4' +} + + +// artifact properties +group = 'org.myorg.quickstart' +version = '0.1-SNAPSHOT' +mainClassName = 'org.myorg.quickstart.StreamingJob' +description = """Flink Quickstart Job""" + +ext { + javaVersion = '1.8' + flinkVersion = '{{ site.version }}' + scalaBinaryVersion = '{{ site.scala_version }}' + slf4jVersion = '1.7.7' + log4jVersion = '1.2.17' +} + + +sourceCompatibility = javaVersion +targetCompatibility = javaVersion +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} + +applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"] + +task wrapper(type: Wrapper) { + gradleVersion = '3.1' +} + +// declare where to find the dependencies of your project +repositories { + mavenCentral() + maven { url "https://repository.apache.org/content/repositories/snapshots/" } +} + +// NOTE: We cannot use "compileOnly" or "shadow" configurations since then we could not run code +// in the IDE or with "gradle run". We also cannot exclude transitive dependencies from the +// shadowJar yet (see https://github.com/johnrengelman/shadow/issues/159). +// -> Explicitly define the // libraries we want to be included in the "flinkShadowJar" configuration! +configurations { + flinkShadowJar // dependencies which go into the shadowJar + + // always exclude these (also from transitive dependencies) since they are provided by Flink + flinkShadowJar.exclude group: 'org.apache.flink', module: 'force-shading' + flinkShadowJar.exclude group: 'com.google.code.findbugs', module: 'jsr305' + flinkShadowJar.exclude group: 'org.slf4j' + flinkShadowJar.exclude group: 'log4j' +} + +// declare the dependencies for your production and test code +dependencies { + // -------------------------------------------------------------- + // Compile-time dependencies that should NOT be part of the + // shadow jar and are provided in the lib folder of Flink + // -------------------------------------------------------------- + compile "org.apache.flink:flink-java:${flinkVersion}" + compile "org.apache.flink:flink-streaming-java_${scalaBinaryVersion}:${flinkVersion}" + + // -------------------------------------------------------------- + // Dependencies that should be part of the shadow jar, e.g. + // connectors. These must be in the flinkShadowJar configuration! + // -------------------------------------------------------------- + //flinkShadowJar "org.apache.flink:flink-connector-kafka-0.11_${scalaBinaryVersion}:${flinkVersion}" + + compile "log4j:log4j:${log4jVersion}" + compile "org.slf4j:slf4j-log4j12:${slf4jVersion}" + + // Add test dependencies here. + // testCompile "junit:junit:4.12" +} + +// make compileOnly dependencies available for tests: +sourceSets { + main.compileClasspath += configurations.flinkShadowJar + main.runtimeClasspath += configurations.flinkShadowJar + + test.compileClasspath += configurations.flinkShadowJar + test.runtimeClasspath += configurations.flinkShadowJar + + javadoc.classpath += configurations.flinkShadowJar +} + +run.classpath = sourceSets.main.runtimeClasspath + +jar { + manifest { + attributes 'Built-By': System.getProperty('user.name'), + 'Build-Jdk': System.getProperty('java.version') + } +} + +shadowJar { + configurations = [project.configurations.flinkShadowJar] +} + {% endhighlight %} +
+
+ {% highlight gradle %} +rootProject.name = 'quickstart' + {% endhighlight %} +
+
+
+ +
+ {% highlight bash %} + bash -c "$(curl https://flink.apache.org/q/gradle-quickstart.sh)" -- {{site.version}} {{site.scala_version}} + {% endhighlight %} + This allows you to name your newly created project. It will interactively ask + you for the project name, organization (also used for the package name), project version, + Scala and Flink version. +
+
+ +### Inspect Project + +There will be a new directory in your working directory based on the +project name you provided, e.g. for `quickstart`: + +{% highlight bash %} +$ tree quickstart/ +quickstart/ +├── README +├── build.gradle +├── settings.gradle +└── src + └── main + ├── java + │   └── org + │   └── myorg + │   └── quickstart + │   ├── BatchJob.java + │   └── StreamingJob.java + └── resources + └── log4j.properties +{% endhighlight %} + +The sample project is a __Gradle project__, which contains two classes: _StreamingJob_ and _BatchJob_ are the basic skeleton programs for a *DataStream* and *DataSet* program. +The _main_ method is the entry point of the program, both for in-IDE testing/execution and for proper deployments. + +We recommend you __import this project into your IDE__ to develop and +test it. IntelliJ IDEA supports Gradle projects after installing the `Gradle` plugin. +Eclipse does so via the [Eclipse Buildship](https://projects.eclipse.org/projects/tools.buildship) plugin +(make sure to specify a Gradle version >= 3.0 in the last step of the import wizard; the `shadow` plugin requires it). +You may also use [Gradle's IDE integration](https://docs.gradle.org/current/userguide/userguide.html#ide-integration) +to create project files from Gradle. + + +*A note to Mac OS X users*: The default JVM heapsize for Java may be too +small for Flink. You have to manually increase it. In Eclipse, choose +`Run Configurations -> Arguments` and write into the `VM Arguments` +box: `-Xmx800m`. + +### Build Project + +If you want to __build/package your project__, go to your project directory and +run the '`gradle clean shadowJar`' command. +You will __find a JAR file__ that contains your application, plus connectors and libraries +that you may have added as dependencies to the application: `build/libs/--all.jar`. + +__Note:__ If you use a different class than *StreamingJob* as the application's main class / entry point, +we recommend you change the `mainClassName` setting in the `build.gradle` file accordingly. That way, Flink +can run the application from the JAR file without additionally specifying the main class. ## Next Steps From bfbf98697bc4dacab5b689fcbdbfcb080b331801 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Tue, 10 Jul 2018 13:44:41 +0200 Subject: [PATCH 133/147] [hotfix][quickstart] make the note about IDE memory requirements non-specific to MacOS X users --- docs/dev/projectsetup/java_api_quickstart.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/dev/projectsetup/java_api_quickstart.md b/docs/dev/projectsetup/java_api_quickstart.md index 91ef70e9326ced..fd2eef503790e8 100644 --- a/docs/dev/projectsetup/java_api_quickstart.md +++ b/docs/dev/projectsetup/java_api_quickstart.md @@ -111,11 +111,9 @@ allows to [import Maven projects](http://books.sonatype.com/m2eclipse-book/refer Some Eclipse bundles include that plugin by default, others require you to install it manually. -*A note to Mac OS X users*: The default JVM heapsize for Java may be too +*Please note*: The default JVM heapsize for Java may be too small for Flink. You have to manually increase it. -In Eclipse, choose -`Run Configurations -> Arguments` and write into the `VM Arguments` -box: `-Xmx800m`. +In Eclipse, choose `Run Configurations -> Arguments` and write into the `VM Arguments` box: `-Xmx800m`. In IntelliJ IDEA recommended way to change JVM options is from the `Help | Edit Custom VM Options` menu. See [this article](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties) for details. ### Build Project @@ -317,10 +315,10 @@ You may also use [Gradle's IDE integration](https://docs.gradle.org/current/user to create project files from Gradle. -*A note to Mac OS X users*: The default JVM heapsize for Java may be too -small for Flink. You have to manually increase it. In Eclipse, choose -`Run Configurations -> Arguments` and write into the `VM Arguments` -box: `-Xmx800m`. +*Please note*: The default JVM heapsize for Java may be too +small for Flink. You have to manually increase it. +In Eclipse, choose `Run Configurations -> Arguments` and write into the `VM Arguments` box: `-Xmx800m`. +In IntelliJ IDEA recommended way to change JVM options is from the `Help | Edit Custom VM Options` menu. See [this article](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties) for details. ### Build Project From a1c64b99a73ed7cbb91f7db983196981ee8af330 Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 20 Dec 2018 11:54:55 +0100 Subject: [PATCH 134/147] [FLINK-11023][kafka] Add NOTICE files --- flink-connectors/flink-sql-connector-kafka-0.10/pom.xml | 7 ++++++- .../src/main/resources/META-INF/NOTICE | 9 +++++++++ flink-connectors/flink-sql-connector-kafka-0.11/pom.xml | 7 ++++++- .../src/main/resources/META-INF/NOTICE | 9 +++++++++ flink-connectors/flink-sql-connector-kafka-0.9/pom.xml | 5 ++++- .../src/main/resources/META-INF/NOTICE | 9 +++++++++ flink-connectors/flink-sql-connector-kafka/pom.xml | 7 ++++++- .../src/main/resources/META-INF/NOTICE | 9 +++++++++ 8 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 flink-connectors/flink-sql-connector-kafka-0.10/src/main/resources/META-INF/NOTICE create mode 100644 flink-connectors/flink-sql-connector-kafka-0.11/src/main/resources/META-INF/NOTICE create mode 100644 flink-connectors/flink-sql-connector-kafka-0.9/src/main/resources/META-INF/NOTICE create mode 100644 flink-connectors/flink-sql-connector-kafka/src/main/resources/META-INF/NOTICE diff --git a/flink-connectors/flink-sql-connector-kafka-0.10/pom.xml b/flink-connectors/flink-sql-connector-kafka-0.10/pom.xml index 06027573b32a6f..0411ab92cc538c 100644 --- a/flink-connectors/flink-sql-connector-kafka-0.10/pom.xml +++ b/flink-connectors/flink-sql-connector-kafka-0.10/pom.xml @@ -66,9 +66,14 @@ under the License. - *:* + org.apache.kafka:* kafka/kafka-version.properties + LICENSE + + NOTICE diff --git a/flink-connectors/flink-sql-connector-kafka-0.10/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-sql-connector-kafka-0.10/src/main/resources/META-INF/NOTICE new file mode 100644 index 00000000000000..abe30980272ac3 --- /dev/null +++ b/flink-connectors/flink-sql-connector-kafka-0.10/src/main/resources/META-INF/NOTICE @@ -0,0 +1,9 @@ +flink-sql-connector-kafka-0.10 +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.kafka:kafka-clients:0.10.2.1 diff --git a/flink-connectors/flink-sql-connector-kafka-0.11/pom.xml b/flink-connectors/flink-sql-connector-kafka-0.11/pom.xml index 4b031a13a726bb..2401d310c353eb 100644 --- a/flink-connectors/flink-sql-connector-kafka-0.11/pom.xml +++ b/flink-connectors/flink-sql-connector-kafka-0.11/pom.xml @@ -67,9 +67,14 @@ under the License. - *:* + org.apache.kafka:* kafka/kafka-version.properties + LICENSE + + NOTICE diff --git a/flink-connectors/flink-sql-connector-kafka-0.11/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-sql-connector-kafka-0.11/src/main/resources/META-INF/NOTICE new file mode 100644 index 00000000000000..6edfd53a673b96 --- /dev/null +++ b/flink-connectors/flink-sql-connector-kafka-0.11/src/main/resources/META-INF/NOTICE @@ -0,0 +1,9 @@ +flink-sql-connector-kafka-0.9 +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.kafka:kafka-clients:0.11.0.2 diff --git a/flink-connectors/flink-sql-connector-kafka-0.9/pom.xml b/flink-connectors/flink-sql-connector-kafka-0.9/pom.xml index 4668ee5f64b3f8..9185171a9fc9d2 100644 --- a/flink-connectors/flink-sql-connector-kafka-0.9/pom.xml +++ b/flink-connectors/flink-sql-connector-kafka-0.9/pom.xml @@ -65,9 +65,12 @@ under the License. - *:* + org.apache.kafka:* kafka/kafka-version.properties + LICENSE + + NOTICE diff --git a/flink-connectors/flink-sql-connector-kafka-0.9/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-sql-connector-kafka-0.9/src/main/resources/META-INF/NOTICE new file mode 100644 index 00000000000000..8738d64a85c01b --- /dev/null +++ b/flink-connectors/flink-sql-connector-kafka-0.9/src/main/resources/META-INF/NOTICE @@ -0,0 +1,9 @@ +flink-sql-connector-kafka-0.9 +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.kafka:kafka-clients:0.9.0.1 diff --git a/flink-connectors/flink-sql-connector-kafka/pom.xml b/flink-connectors/flink-sql-connector-kafka/pom.xml index 8a2609ed6a7e08..bc65b3db59047f 100644 --- a/flink-connectors/flink-sql-connector-kafka/pom.xml +++ b/flink-connectors/flink-sql-connector-kafka/pom.xml @@ -66,9 +66,14 @@ under the License. - *:* + org.apache.kafka:* kafka/kafka-version.properties + LICENSE + + NOTICE diff --git a/flink-connectors/flink-sql-connector-kafka/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-sql-connector-kafka/src/main/resources/META-INF/NOTICE new file mode 100644 index 00000000000000..2ca1bb4acf0ef6 --- /dev/null +++ b/flink-connectors/flink-sql-connector-kafka/src/main/resources/META-INF/NOTICE @@ -0,0 +1,9 @@ +flink-sql-connector-kafka +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.kafka:kafka-clients:2.0.1 From 9c07822acedfd7e0582481de620e5af205f4aef3 Mon Sep 17 00:00:00 2001 From: Addison Higham Date: Tue, 8 Jan 2019 15:32:28 -0700 Subject: [PATCH 135/147] [FLINK-11187] [s3] Use file over stream for writes This changes the S3AccessHelper API to take a file instead of an input stream. This allows s3 client to properly reset a file instead of a file over stream for writes. This fixes an issue where the underlying s3 implementation has an intermittent failure, tries to reset the stream, fails to do so, and results in hung requests with delayed errors. --- .../utils/RefCountedBufferingFileStream.java | 7 ++----- .../s3/common/utils/RefCountedFSOutputStream.java | 8 ++++---- .../writer/RecoverableMultiPartUploadImpl.java | 9 ++++----- .../flink/fs/s3/common/writer/S3AccessHelper.java | 12 +++++------- .../utils/RefCountedBufferingFileStreamTest.java | 3 ++- .../writer/RecoverableMultiPartUploadImplTest.java | 14 +++++++------- .../S3RecoverableFsDataOutputStreamTest.java | 9 ++++----- .../flink/fs/s3hadoop/HadoopS3AccessHelper.java | 9 ++++----- 8 files changed, 32 insertions(+), 39 deletions(-) diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStream.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStream.java index 8f3aff899f1a14..29f2590803cc8e 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStream.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStream.java @@ -24,9 +24,6 @@ import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.StandardOpenOption; import static org.apache.flink.util.Preconditions.checkArgument; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -64,8 +61,8 @@ public RefCountedBufferingFileStream( } @Override - public InputStream getInputStream() throws IOException { - return Files.newInputStream(currentTmpFile.getFile().toPath(), StandardOpenOption.READ); + public File getInputFile() { + return currentTmpFile.getFile(); } @Override diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFSOutputStream.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFSOutputStream.java index d4b962e146396b..d51e37e8bfbf34 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFSOutputStream.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFSOutputStream.java @@ -21,8 +21,8 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.core.fs.FSDataOutputStream; +import java.io.File; import java.io.IOException; -import java.io.InputStream; /** * A {@link FSDataOutputStream} with the {@link RefCounted} functionality. @@ -31,11 +31,11 @@ public abstract class RefCountedFSOutputStream extends FSDataOutputStream implements RefCounted { /** - * Gets an {@link InputStream} that allows to read the contents of the file. + * Gets the underlying {@link File} that allows to read the contents of the file. * - * @return An input stream to the contents of the file. + * @return A handle to the File object. */ - public abstract InputStream getInputStream() throws IOException; + public abstract File getInputFile(); /** * Checks if the file is closed for writes. diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImpl.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImpl.java index 9d88e65f64d753..0d0998a0620026 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImpl.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImpl.java @@ -30,7 +30,6 @@ import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Deque; @@ -173,8 +172,8 @@ private String safelyUploadSmallPart(@Nullable RefCountedFSOutputStream file) th // first, upload the trailing data file. during that time, other in-progress uploads may complete. final String incompletePartObjectName = createIncompletePartObjectName(); file.retain(); - try (InputStream inputStream = file.getInputStream()) { - s3AccessHelper.putObject(incompletePartObjectName, inputStream, file.getPos()); + try { + s3AccessHelper.putObject(incompletePartObjectName, file.getInputFile()); } finally { file.release(); @@ -315,8 +314,8 @@ private static class UploadTask implements Runnable { @Override public void run() { - try (final InputStream inputStream = file.getInputStream()) { - final UploadPartResult result = s3AccessHelper.uploadPart(objectName, uploadId, partNumber, inputStream, file.getPos()); + try { + final UploadPartResult result = s3AccessHelper.uploadPart(objectName, uploadId, partNumber, file.getInputFile(), file.getPos()); future.complete(new PartETag(result.getPartNumber(), result.getETag())); file.release(); } diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3AccessHelper.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3AccessHelper.java index bcdea3c00b060e..593d9d36de2c19 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3AccessHelper.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3AccessHelper.java @@ -28,7 +28,6 @@ import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -59,25 +58,24 @@ public interface S3AccessHelper { * @param key the key this MPU is associated with. * @param uploadId the id of the MPU. * @param partNumber the number of the part being uploaded (has to be in [1 ... 10000]). - * @param file the (local) file holding the part to be uploaded. + * @param inputFile the (local) file holding the part to be uploaded. * @param length the length of the part. * @return The {@link UploadPartResult result} of the attempt to upload the part. * @throws IOException */ - UploadPartResult uploadPart(String key, String uploadId, int partNumber, InputStream file, long length) throws IOException; + UploadPartResult uploadPart(String key, String uploadId, int partNumber, File inputFile, long length) throws IOException; /** - * Uploads an object to S3. Contrary to the {@link #uploadPart(String, String, int, InputStream, long)} method, + * Uploads an object to S3. Contrary to the {@link #uploadPart(String, String, int, File, long)} method, * this object is not going to be associated to any MPU and, as such, it is not subject to the garbage collection * policies specified for your S3 bucket. * * @param key the key used to identify this part. - * @param file the (local) file holding the data to be uploaded. - * @param length the size of the data to be uploaded. + * @param inputFile the (local) file holding the data to be uploaded. * @return The {@link PutObjectResult result} of the attempt to stage the incomplete part. * @throws IOException */ - PutObjectResult putObject(String key, InputStream file, long length) throws IOException; + PutObjectResult putObject(String key, File inputFile) throws IOException; /** * Finalizes a Multi-Part Upload. diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStreamTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStreamTest.java index 5b7d1ccfb9f2db..50ea9bd64a6e1f 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStreamTest.java +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStreamTest.java @@ -24,6 +24,7 @@ import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; @@ -121,7 +122,7 @@ public void testFlush() throws IOException { Assert.assertEquals(contentToWrite.length, stream.getPos()); final byte[] contentRead = new byte[contentToWrite.length]; - stream.getInputStream().read(contentRead, 0, contentRead.length); + new FileInputStream(stream.getInputFile()).read(contentRead, 0, contentRead.length); Assert.assertTrue(Arrays.equals(contentToWrite, contentRead)); stream.release(); diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImplTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImplTest.java index 673796d6c80cbb..0194065633f305 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImplTest.java +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImplTest.java @@ -35,8 +35,8 @@ import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -361,14 +361,14 @@ public String startMultiPartUpload(String key) throws IOException { } @Override - public UploadPartResult uploadPart(String key, String uploadId, int partNumber, InputStream file, long length) throws IOException { - final byte[] content = getFileContentBytes(file, MathUtils.checkedDownCast(length)); + public UploadPartResult uploadPart(String key, String uploadId, int partNumber, File inputFile, long length) throws IOException { + final byte[] content = getFileContentBytes(inputFile, MathUtils.checkedDownCast(length)); return storeAndGetUploadPartResult(key, partNumber, content); } @Override - public PutObjectResult putObject(String key, InputStream file, long length) throws IOException { - final byte[] content = getFileContentBytes(file, MathUtils.checkedDownCast(length)); + public PutObjectResult putObject(String key, File inputFile) throws IOException { + final byte[] content = getFileContentBytes(inputFile, MathUtils.checkedDownCast(inputFile.length())); return storeAndGetPutObjectResult(key, content); } @@ -397,9 +397,9 @@ public ObjectMetadata getObjectMetadata(String key) throws IOException { return null; } - private byte[] getFileContentBytes(InputStream file, int length) throws IOException { + private byte[] getFileContentBytes(File file, int length) throws IOException { final byte[] content = new byte[length]; - file.read(content, 0, length); + new FileInputStream(file).read(content, 0, length); return content; } diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStreamTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStreamTest.java index 7a32392f885e2f..14ed2e294f7d2c 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStreamTest.java +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStreamTest.java @@ -39,8 +39,8 @@ import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.FileAlreadyExistsException; @@ -314,10 +314,9 @@ private static byte[] bytesOf(String str) { private static byte[] readFileContents(RefCountedFSOutputStream file) throws IOException { final byte[] content = new byte[MathUtils.checkedDownCast(file.getPos())]; - try (InputStream inputStream = file.getInputStream()) { - int bytesRead = inputStream.read(content, 0, content.length); // TODO: 10/2/18 see if closed in download - Assert.assertEquals(file.getPos(), bytesRead); - } + File inputFile = file.getInputFile(); + long bytesRead = new FileInputStream(inputFile).read(content, 0, MathUtils.checkedDownCast(inputFile.length())); + Assert.assertEquals(file.getPos(), bytesRead); return content; } diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3AccessHelper.java b/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3AccessHelper.java index b9612adf63a66e..4d37ce0ba058de 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3AccessHelper.java +++ b/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3AccessHelper.java @@ -38,7 +38,6 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -68,15 +67,15 @@ public String startMultiPartUpload(String key) throws IOException { } @Override - public UploadPartResult uploadPart(String key, String uploadId, int partNumber, InputStream inputStream, long length) throws IOException { + public UploadPartResult uploadPart(String key, String uploadId, int partNumber, File inputFile, long length) throws IOException { final UploadPartRequest uploadRequest = s3accessHelper.newUploadPartRequest( - key, uploadId, partNumber, MathUtils.checkedDownCast(length), inputStream, null, 0L); + key, uploadId, partNumber, MathUtils.checkedDownCast(length), null, inputFile, 0L); return s3accessHelper.uploadPart(uploadRequest); } @Override - public PutObjectResult putObject(String key, InputStream inputStream, long length) throws IOException { - final PutObjectRequest putRequest = s3accessHelper.createPutObjectRequest(key, inputStream, length); + public PutObjectResult putObject(String key, File inputFile) throws IOException { + final PutObjectRequest putRequest = s3accessHelper.createPutObjectRequest(key, inputFile); return s3accessHelper.putObject(putRequest); } From b56197a3f3e7e062bc2666990efdc5fe48708b71 Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 10 Jan 2019 14:46:34 +0100 Subject: [PATCH 136/147] [hotfix][build] Append shade-plugin transformers in child modules --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 46a3a85f445e91..dc6bcad8c3cbe1 100644 --- a/pom.xml +++ b/pom.xml @@ -1525,7 +1525,7 @@ under the License. org.apache.flink:force-shading - + From 979ebd212885f5b46ba1b60beea95080e0a7bc0f Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 9 Jan 2019 16:20:51 +0100 Subject: [PATCH 137/147] [FLINK-11289][examples] Rework examples to account for licensing --- NOTICE-binary | 63 ++++++++++- .../flink-connector-twitter/pom.xml | 3 - flink-dist/pom.xml | 14 +++ flink-dist/src/main/assemblies/bin.xml | 24 ++++ .../pom.xml | 106 ++++++++++++++++++ .../src/main/resources/META-INF/NOTICE | 9 ++ .../flink-examples-streaming-twitter/pom.xml | 94 ++++++++++++++++ .../flink-examples-build-helper/pom.xml | 40 +++++++ .../flink-examples-streaming/pom.xml | 74 ------------ flink-examples/pom.xml | 1 + 10 files changed, 347 insertions(+), 81 deletions(-) create mode 100644 flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/pom.xml create mode 100644 flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/src/main/resources/META-INF/NOTICE create mode 100644 flink-examples/flink-examples-build-helper/flink-examples-streaming-twitter/pom.xml create mode 100644 flink-examples/flink-examples-build-helper/pom.xml diff --git a/NOTICE-binary b/NOTICE-binary index 2619dbab859a64..9028f72aaa4f35 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -9,12 +9,8 @@ Copyright 2014-2018 The Apache Software Foundation This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) - com.chuusai:shapeless_2.11:2.3.2 -- com.google.guava:guava:14.0.1 -- com.twitter:hbc-core:2.2.0 -- com.twitter:joauth:6.0.2 - log4j:log4j:1.2.17 - net.sf.opencsv:opencsv:2.3 -- org.apache.kafka:kafka-clients:jar:0.10.2.1 - org.scalanlp:breeze_2.11:0.13 - org.scalanlp:breeze-macros_2.11:0.13 - org.typelevel:macro-compat_2.11:1.1.1 @@ -5806,3 +5802,62 @@ This project bundles the following dependencies under the Apache Software Licens - io.dropwizard.metrics:metrics-core:3.1.5 - io.dropwizard.metrics:metrics-graphite:3.1.5 + +// NOTICE file corresponding to the section 4d of The Apache License, +// Version 2.0, in this case for Apache Flink +// ------------------------------------------------------------------ + +Apache Flink +Copyright 2006-2019 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +flink-examples-streaming-state-machine +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.kafka:kafka-clients:0.10.2.1 + + +flink-connector-kafka-0.10 +Copyright 2014-2019 The Apache Software Foundation + +flink-connector-kafka-0.9 +Copyright 2014-2019 The Apache Software Foundation + +flink-connector-kafka-base +Copyright 2014-2019 The Apache Software Foundation + +// ------------------------------------------------------------------ +// NOTICE file corresponding to the section 4d of The Apache License, +// Version 2.0, in this case for Apache Flink +// ------------------------------------------------------------------ + +Apache Flink +Copyright 2006-2019 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + +flink-examples-streaming-twitter +Copyright 2014-2019 The Apache Software Foundation + +Apache HttpCore +Copyright 2005-2017 The Apache Software Foundation + +flink-connector-twitter +Copyright 2014-2018 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.guava:guava:14.0.1 +- com.twitter:hbc-core:2.2.0 +- com.twitter:joauth:6.0.2 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 + +Apache HttpClient +Copyright 1999-2017 The Apache Software Foundation \ No newline at end of file diff --git a/flink-connectors/flink-connector-twitter/pom.xml b/flink-connectors/flink-connector-twitter/pom.xml index b78458ab769582..eca315ff15d485 100644 --- a/flink-connectors/flink-connector-twitter/pom.xml +++ b/flink-connectors/flink-connector-twitter/pom.xml @@ -94,9 +94,6 @@ under the License. - - - diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml index 84339ebbecac71..3ab2343d823a71 100644 --- a/flink-dist/pom.xml +++ b/flink-dist/pom.xml @@ -207,6 +207,20 @@ under the License. provided + + org.apache.flink + flink-examples-streaming-state-machine_${scala.binary.version} + ${project.version} + provided + + + + org.apache.flink + flink-examples-streaming-twitter_${scala.binary.version} + ${project.version} + provided + + org.apache.flink flink-gelly-examples_${scala.binary.version} diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml index 8627f1a56a18a9..438321c9ef4cba 100644 --- a/flink-dist/src/main/assemblies/bin.xml +++ b/flink-dist/src/main/assemblies/bin.xml @@ -194,6 +194,30 @@ under the License. + + + ../flink-examples/flink-examples-build-helper/flink-examples-streaming-twitter/target + examples/streaming + 0644 + + *.jar + + + original-*.jar + + + + ../flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/target + examples/streaming + 0644 + + *.jar + + + original-*.jar + + + ../flink-libraries/flink-gelly-examples/target diff --git a/flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/pom.xml b/flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/pom.xml new file mode 100644 index 00000000000000..4400fb834c57a9 --- /dev/null +++ b/flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/pom.xml @@ -0,0 +1,106 @@ + + + + 4.0.0 + + + flink-examples-build-helper + org.apache.flink + 1.8-SNAPSHOT + .. + + + flink-examples-streaming-state-machine_${scala.binary.version} + flink-examples-streaming-state-machine + jar + + + + org.apache.flink + flink-examples-streaming_${scala.binary.version} + ${project.version} + + + + + StateMachineExample + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + false + false + false + + + org.apache.flink.streaming.examples.statemachine.StateMachineExample + + + + + org.apache.flink:flink-connector-kafka* + org.apache.flink:flink-examples-streaming* + org.apache.kafka:* + + + + + + org.apache.flink:flink-examples-streaming_* + + org/apache/flink/streaming/examples/statemachine/** + + + + org.apache.kafka:* + + LICENSE + + NOTICE + + + + + + + + + + + diff --git a/flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/src/main/resources/META-INF/NOTICE b/flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/src/main/resources/META-INF/NOTICE new file mode 100644 index 00000000000000..7a2f9e33340c91 --- /dev/null +++ b/flink-examples/flink-examples-build-helper/flink-examples-streaming-state-machine/src/main/resources/META-INF/NOTICE @@ -0,0 +1,9 @@ +flink-examples-streaming-state-machine +Copyright 2014-2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.kafka:kafka-clients:0.10.2.1 diff --git a/flink-examples/flink-examples-build-helper/flink-examples-streaming-twitter/pom.xml b/flink-examples/flink-examples-build-helper/flink-examples-streaming-twitter/pom.xml new file mode 100644 index 00000000000000..a602959100943c --- /dev/null +++ b/flink-examples/flink-examples-build-helper/flink-examples-streaming-twitter/pom.xml @@ -0,0 +1,94 @@ + + + + 4.0.0 + + + flink-examples-build-helper + org.apache.flink + 1.8-SNAPSHOT + .. + + + flink-examples-streaming-twitter_${scala.binary.version} + flink-examples-streaming-twitter + jar + + + + org.apache.flink + flink-examples-streaming_${scala.binary.version} + ${project.version} + + + + + Twitter + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + false + false + false + + + org.apache.flink.streaming.examples.twitter.TwitterExample + + + + + org.apache.flink:flink-connector-twitter* + org.apache.flink:flink-examples-streaming* + + + + + org.apache.flink:*flink-examples-streaming_* + + org/apache/flink/streaming/examples/twitter/** + + + + + + + + + + + diff --git a/flink-examples/flink-examples-build-helper/pom.xml b/flink-examples/flink-examples-build-helper/pom.xml new file mode 100644 index 00000000000000..344057c01a0944 --- /dev/null +++ b/flink-examples/flink-examples-build-helper/pom.xml @@ -0,0 +1,40 @@ + + + + 4.0.0 + + + flink-examples + org.apache.flink + 1.8-SNAPSHOT + .. + + + flink-examples-build-helper + pom + This is a utility module for building example jars to be used in flink-dist. + + + flink-examples-streaming-twitter + flink-examples-streaming-state-machine + + + diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml index 30347abfcb7edf..0be7fad3e1d359 100644 --- a/flink-examples/flink-examples-streaming/pom.xml +++ b/flink-examples/flink-examples-streaming/pom.xml @@ -491,80 +491,6 @@ under the License. - - - - - org.apache.maven.plugins - maven-shade-plugin - - - - - state-machine-example - package - - shade - - - false - false - false - - - org.apache.flink.streaming.examples.statemachine.StateMachineExample - - - StateMachineExample - - - * - - org/apache/flink/streaming/examples/statemachine/** - org/apache/flink/streaming/util/serialization/** - org/apache/flink/streaming/connectors/kafka/** - org/apache/kafka/** - kafka/ - - - - - - - - fat-jar-twitter-example - package - - shade - - - false - false - false - - - org.apache.flink.streaming.examples.twitter.TwitterExample - - - Twitter - - - * - - org/apache/flink/twitter/shaded/com/google/common/** - org/apache/flink/streaming/examples/twitter/** - org/apache/flink/streaming/connectors/twitter/** - org/apache/http/** - com/twitter/** - build.properties - - - - - - - - diff --git a/flink-examples/pom.xml b/flink-examples/pom.xml index 165130d3bdfa79..f76652bbffef77 100644 --- a/flink-examples/pom.xml +++ b/flink-examples/pom.xml @@ -36,6 +36,7 @@ under the License. flink-examples-batch flink-examples-streaming flink-examples-table + flink-examples-build-helper From 53034cd70e2a39e08c828bbc812ad8eef93e4f9d Mon Sep 17 00:00:00 2001 From: ZILI CHEN Date: Wed, 16 Jan 2019 20:10:16 +0800 Subject: [PATCH 138/147] [FLINK-11174][prometheus] Add option to not filter label values --- ...us_push_gateway_reporter_configuration.html | 5 +++++ docs/monitoring/metrics.md | 1 + .../prometheus/AbstractPrometheusReporter.java | 18 ++++++++++++++++-- .../PrometheusPushGatewayReporter.java | 2 ++ .../PrometheusPushGatewayReporterOptions.java | 13 +++++++++++++ .../metrics/prometheus/PrometheusReporter.java | 2 ++ 6 files changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/_includes/generated/prometheus_push_gateway_reporter_configuration.html b/docs/_includes/generated/prometheus_push_gateway_reporter_configuration.html index 2d14b50777b474..5eee4c698c28eb 100644 --- a/docs/_includes/generated/prometheus_push_gateway_reporter_configuration.html +++ b/docs/_includes/generated/prometheus_push_gateway_reporter_configuration.html @@ -12,6 +12,11 @@ true Specifies whether to delete metrics from the PushGateway on shutdown. + +
filterLabelValueCharacters
+ true + Specifies whether to filter label value characters. If enabled, all characters not matching [a-zA-Z0-9:_] will be removed, otherwise no characters will be removed. Before disabling this option please ensure that your label values meet the Prometheus requirements. +
host
(none) diff --git a/docs/monitoring/metrics.md b/docs/monitoring/metrics.md index bfa0576c98caf7..ae51f9ee3d4422 100644 --- a/docs/monitoring/metrics.md +++ b/docs/monitoring/metrics.md @@ -652,6 +652,7 @@ of your Flink distribution. Parameters: - `port` - (optional) the port the Prometheus exporter listens on, defaults to [9249](https://github.com/prometheus/prometheus/wiki/Default-port-allocations). In order to be able to run several instances of the reporter on one host (e.g. when one TaskManager is colocated with the JobManager) it is advisable to use a port range like `9250-9260`. +- `filterLabelValueCharacters` - (optional) Specifies whether to filter label value characters. If enabled, all characters not matching \[a-zA-Z0-9:_\] will be removed, otherwise no characters will be removed. Before disabling this option please ensure that your label values meet the [Prometheus requirements](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels). Example configuration: diff --git a/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/AbstractPrometheusReporter.java b/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/AbstractPrometheusReporter.java index 426cd4c4f5ac75..72bff5b7e7a86c 100644 --- a/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/AbstractPrometheusReporter.java +++ b/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/AbstractPrometheusReporter.java @@ -26,6 +26,7 @@ import org.apache.flink.metrics.Histogram; import org.apache.flink.metrics.Meter; import org.apache.flink.metrics.Metric; +import org.apache.flink.metrics.MetricConfig; import org.apache.flink.metrics.MetricGroup; import org.apache.flink.metrics.reporter.MetricReporter; import org.apache.flink.runtime.metrics.groups.AbstractMetricGroup; @@ -46,6 +47,7 @@ import java.util.Map; import java.util.regex.Pattern; +import static org.apache.flink.metrics.prometheus.PrometheusPushGatewayReporterOptions.FILTER_LABEL_VALUE_CHARACTER; /** * base prometheus reporter for prometheus metrics. @@ -75,6 +77,18 @@ static String replaceInvalidChars(final String input) { return UNALLOWED_CHAR_PATTERN.matcher(input).replaceAll("_"); } + private CharacterFilter labelValueCharactersFilter = CHARACTER_FILTER; + + @Override + public void open(MetricConfig config) { + boolean filterLabelValueCharacters = config.getBoolean( + FILTER_LABEL_VALUE_CHARACTER.key(), FILTER_LABEL_VALUE_CHARACTER.defaultValue()); + + if (!filterLabelValueCharacters) { + labelValueCharactersFilter = input -> input; + } + } + @Override public void close() { CollectorRegistry.defaultRegistry.clear(); @@ -88,7 +102,7 @@ public void notifyOfAddedMetric(final Metric metric, final String metricName, fi for (final Map.Entry dimension : group.getAllVariables().entrySet()) { final String key = dimension.getKey(); dimensionKeys.add(CHARACTER_FILTER.filterCharacters(key.substring(1, key.length() - 1))); - dimensionValues.add(CHARACTER_FILTER.filterCharacters(dimension.getValue())); + dimensionValues.add(labelValueCharactersFilter.filterCharacters(dimension.getValue())); } final String scopedMetricName = getScopedName(metricName, group); @@ -173,7 +187,7 @@ public void notifyOfRemovedMetric(final Metric metric, final String metricName, List dimensionValues = new LinkedList<>(); for (final Map.Entry dimension : group.getAllVariables().entrySet()) { - dimensionValues.add(CHARACTER_FILTER.filterCharacters(dimension.getValue())); + dimensionValues.add(labelValueCharactersFilter.filterCharacters(dimension.getValue())); } final String scopedMetricName = getScopedName(metricName, group); diff --git a/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/PrometheusPushGatewayReporter.java b/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/PrometheusPushGatewayReporter.java index 164eaf325c7629..ac09c339c444ac 100644 --- a/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/PrometheusPushGatewayReporter.java +++ b/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/PrometheusPushGatewayReporter.java @@ -48,6 +48,8 @@ public class PrometheusPushGatewayReporter extends AbstractPrometheusReporter im @Override public void open(MetricConfig config) { + super.open(config); + String host = config.getString(HOST.key(), HOST.defaultValue()); int port = config.getInteger(PORT.key(), PORT.defaultValue()); String configuredJobName = config.getString(JOB_NAME.key(), JOB_NAME.defaultValue()); diff --git a/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/PrometheusPushGatewayReporterOptions.java b/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/PrometheusPushGatewayReporterOptions.java index 74fe7cb2128b79..8f1713b4be653d 100644 --- a/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/PrometheusPushGatewayReporterOptions.java +++ b/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/PrometheusPushGatewayReporterOptions.java @@ -20,6 +20,8 @@ import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.configuration.description.Description; +import org.apache.flink.configuration.description.LinkElement; /** * Config options for the {@link PrometheusPushGatewayReporter}. @@ -50,4 +52,15 @@ public class PrometheusPushGatewayReporterOptions { .key("deleteOnShutdown") .defaultValue(true) .withDescription("Specifies whether to delete metrics from the PushGateway on shutdown."); + + public static final ConfigOption FILTER_LABEL_VALUE_CHARACTER = ConfigOptions + .key("filterLabelValueCharacters") + .defaultValue(true) + .withDescription(Description.builder() + .text("Specifies whether to filter label value characters." + + " If enabled, all characters not matching [a-zA-Z0-9:_] will be removed," + + " otherwise no characters will be removed." + + " Before disabling this option please ensure that your" + + " label values meet the %s.", LinkElement.link("https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels", "Prometheus requirements")) + .build()); } diff --git a/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/PrometheusReporter.java b/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/PrometheusReporter.java index 190e120c1c4bbe..4697685b8c0653 100644 --- a/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/PrometheusReporter.java +++ b/flink-metrics/flink-metrics-prometheus/src/main/java/org/apache/flink/metrics/prometheus/PrometheusReporter.java @@ -51,6 +51,8 @@ int getPort() { @Override public void open(MetricConfig config) { + super.open(config); + String portsConfig = config.getString(ARG_PORT, DEFAULT_PORT); Iterator ports = NetUtils.getPortRangeFromString(portsConfig); From 4863bc1aa7c376a996b66953c2cbd7afb6eccc1f Mon Sep 17 00:00:00 2001 From: intsmaze Date: Wed, 16 Jan 2019 20:26:02 +0800 Subject: [PATCH 139/147] [hotfix][docs] Update references to deprecated SinkFunction#invoke(Object) --- docs/dev/stream/state/state.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/stream/state/state.md b/docs/dev/stream/state/state.md index 8fe2981a20a07e..1654c9a77beb66 100644 --- a/docs/dev/stream/state/state.md +++ b/docs/dev/stream/state/state.md @@ -475,7 +475,7 @@ public class BufferingSink } @Override - public void invoke(Tuple2 value) throws Exception { + public void invoke(Tuple2 value, Context contex) throws Exception { bufferedElements.add(value); if (bufferedElements.size() == threshold) { for (Tuple2 element: bufferedElements) { @@ -523,7 +523,7 @@ class BufferingSink(threshold: Int = 0) private val bufferedElements = ListBuffer[(String, Int)]() - override def invoke(value: (String, Int)): Unit = { + override def invoke(value: (String, Int), context: Context): Unit = { bufferedElements += value if (bufferedElements.size == threshold) { for (element <- bufferedElements) { From 9a21cc0b216da5198854ae52add53ffad3af4e60 Mon Sep 17 00:00:00 2001 From: leesf <490081539@qq.com> Date: Wed, 16 Jan 2019 20:27:08 +0800 Subject: [PATCH 140/147] [hotfix] Fix parameter name KeyedCoProcessOperator constructor --- .../streaming/api/operators/co/KeyedCoProcessOperator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/KeyedCoProcessOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/KeyedCoProcessOperator.java index 4e2bfc71261f0b..c7d981cf595fc0 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/KeyedCoProcessOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/KeyedCoProcessOperator.java @@ -53,8 +53,8 @@ public class KeyedCoProcessOperator private transient OnTimerContextImpl onTimerContext; - public KeyedCoProcessOperator(CoProcessFunction flatMapper) { - super(flatMapper); + public KeyedCoProcessOperator(CoProcessFunction coProcessFunction) { + super(coProcessFunction); } @Override From c08db3fda3335402df8f1f770b73007832227aa1 Mon Sep 17 00:00:00 2001 From: "jinhu.wjh" Date: Sun, 30 Dec 2018 02:23:43 +0800 Subject: [PATCH 141/147] [FLINK-11012][tests] Introduce abstract superclass for filesystem IT cases. This closes #7384. --- docs/ops/deployment/oss.md | 6 +- flink-filesystems/flink-hadoop-fs/pom.xml | 15 ++ .../runtime/fs/hdfs/HadoopFileSystem.java | 4 +- .../hdfs/AbstractHadoopFileSystemITTest.java | 162 ++++++++++++++++++ flink-filesystems/flink-oss-fs-hadoop/pom.xml | 17 ++ .../HadoopOSSFileSystemBehaviorITCase.java | 70 ++++++++ .../osshadoop/HadoopOSSFileSystemITCase.java | 73 ++------ flink-filesystems/flink-s3-fs-hadoop/pom.xml | 7 + .../fs/s3hadoop/HadoopS3FileSystemITCase.java | 134 +-------------- flink-filesystems/flink-s3-fs-presto/pom.xml | 8 + .../fs/s3presto/PrestoS3FileSystemITCase.java | 125 +++----------- .../testutils/oss/OSSTestCredentials.java | 99 +++++++++++ 12 files changed, 428 insertions(+), 292 deletions(-) create mode 100644 flink-filesystems/flink-hadoop-fs/src/test/java/org/apache/flink/runtime/fs/hdfs/AbstractHadoopFileSystemITTest.java create mode 100644 flink-filesystems/flink-oss-fs-hadoop/src/test/java/org/apache/flink/fs/osshadoop/HadoopOSSFileSystemBehaviorITCase.java create mode 100644 flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/oss/OSSTestCredentials.java diff --git a/docs/ops/deployment/oss.md b/docs/ops/deployment/oss.md index 543d5a2f176450..b70676711e54bc 100644 --- a/docs/ops/deployment/oss.md +++ b/docs/ops/deployment/oss.md @@ -1,5 +1,5 @@ --- -title: "Aliyun Open Storage Service (OSS)" +title: "Aliyun Object Storage Service (OSS)" nav-title: Aliyun OSS nav-parent_id: deployment nav-pos: 9 @@ -27,9 +27,9 @@ under the License. {:toc} -## OSS: Open Storage Service +## OSS: Object Storage Service -[Aliyun Open Storage Service](https://www.aliyun.com/product/oss) (Aliyun OSS) is widely used especially among China’s cloud users, and it provides cloud object storage for a variety of use cases. +[Aliyun Object Storage Service](https://www.aliyun.com/product/oss) (Aliyun OSS) is widely used especially among China’s cloud users, and it provides cloud object storage for a variety of use cases. [Hadoop file system](http://hadoop.apache.org/docs/current/hadoop-aliyun/tools/hadoop-aliyun/index.html) supports OSS since version 2.9.1. Now, you can also use OSS with Fink for **reading** and **writing data**. diff --git a/flink-filesystems/flink-hadoop-fs/pom.xml b/flink-filesystems/flink-hadoop-fs/pom.xml index 468e3550f5cfb3..8c2d2ff87cc31c 100644 --- a/flink-filesystems/flink-hadoop-fs/pom.xml +++ b/flink-filesystems/flink-hadoop-fs/pom.xml @@ -76,4 +76,19 @@ under the License. + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + diff --git a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopFileSystem.java b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopFileSystem.java index dceca1e36bb07b..6a5976aa9ad427 100644 --- a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopFileSystem.java +++ b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopFileSystem.java @@ -224,8 +224,8 @@ public static org.apache.hadoop.fs.Path toHadoopPath(Path path) { static FileSystemKind getKindForScheme(String scheme) { scheme = scheme.toLowerCase(Locale.US); - if (scheme.startsWith("s3") || scheme.startsWith("emr")) { - // the Amazon S3 storage + if (scheme.startsWith("s3") || scheme.startsWith("emr") || scheme.startsWith("oss")) { + // the Amazon S3 storage or Aliyun OSS storage return FileSystemKind.OBJECT_STORE; } else if (scheme.startsWith("http") || scheme.startsWith("ftp")) { diff --git a/flink-filesystems/flink-hadoop-fs/src/test/java/org/apache/flink/runtime/fs/hdfs/AbstractHadoopFileSystemITTest.java b/flink-filesystems/flink-hadoop-fs/src/test/java/org/apache/flink/runtime/fs/hdfs/AbstractHadoopFileSystemITTest.java new file mode 100644 index 00000000000000..d2695d7a5f66b5 --- /dev/null +++ b/flink-filesystems/flink-hadoop-fs/src/test/java/org/apache/flink/runtime/fs/hdfs/AbstractHadoopFileSystemITTest.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.fs.hdfs; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.fs.FSDataInputStream; +import org.apache.flink.core.fs.FSDataOutputStream; +import org.apache.flink.core.fs.FileStatus; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.core.fs.Path; +import org.apache.flink.util.TestLogger; + +import org.junit.AfterClass; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; + +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertFalse; +import static junit.framework.TestCase.assertNotNull; +import static junit.framework.TestCase.assertTrue; +import static org.apache.flink.core.fs.FileSystemTestUtils.checkPathEventualExistence; + +/** + * Abstract integration test class for implementations of hadoop file system. + */ +public abstract class AbstractHadoopFileSystemITTest extends TestLogger { + + protected static FileSystem fs; + protected static Path basePath; + protected static long deadline; + + public static void checkPathExistence(Path path, + boolean expectedExists, + long deadline) throws IOException, InterruptedException { + if (deadline == 0) { + //strongly consistency + assertEquals(expectedExists, fs.exists(path)); + } else { + //eventually consistency + checkPathEventualExistence(fs, path, expectedExists, deadline); + } + } + + protected void checkEmptyDirectory(Path path) throws IOException, InterruptedException { + checkPathExistence(path, true, deadline); + } + + @Test + public void testSimpleFileWriteAndRead() throws Exception { + final String testLine = "Hello Upload!"; + + final Path path = new Path(basePath, "test.txt"); + + try { + try (FSDataOutputStream out = fs.create(path, FileSystem.WriteMode.OVERWRITE); + OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { + writer.write(testLine); + } + + // just in case, wait for the path to exist + checkPathExistence(path, true, deadline); + + try (FSDataInputStream in = fs.open(path); + InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(ir)) { + String line = reader.readLine(); + assertEquals(testLine, line); + } + } + finally { + fs.delete(path, false); + } + + checkPathExistence(path, false, deadline); + } + + @Test + public void testDirectoryListing() throws Exception { + final Path directory = new Path(basePath, "testdir/"); + + // directory must not yet exist + assertFalse(fs.exists(directory)); + + try { + // create directory + assertTrue(fs.mkdirs(directory)); + + checkEmptyDirectory(directory); + + // directory empty + assertEquals(0, fs.listStatus(directory).length); + + // create some files + final int numFiles = 3; + for (int i = 0; i < numFiles; i++) { + Path file = new Path(directory, "/file-" + i); + try (FSDataOutputStream out = fs.create(file, FileSystem.WriteMode.OVERWRITE); + OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { + writer.write("hello-" + i + "\n"); + } + // just in case, wait for the file to exist (should then also be reflected in the + // directory's file list below) + checkPathExistence(file, true, deadline); + } + + FileStatus[] files = fs.listStatus(directory); + assertNotNull(files); + assertEquals(3, files.length); + + for (FileStatus status : files) { + assertFalse(status.isDir()); + } + + // now that there are files, the directory must exist + assertTrue(fs.exists(directory)); + } + finally { + // clean up + fs.delete(directory, true); + } + + // now directory must be gone + checkPathExistence(directory, false, deadline); + } + + @AfterClass + public static void teardown() throws IOException, InterruptedException { + try { + if (fs != null) { + // clean up + fs.delete(basePath, true); + + // now directory must be gone + checkPathExistence(basePath, false, deadline); + } + } + finally { + FileSystem.initialize(new Configuration()); + } + } +} diff --git a/flink-filesystems/flink-oss-fs-hadoop/pom.xml b/flink-filesystems/flink-oss-fs-hadoop/pom.xml index b4f70a49e45810..85465e410d2ac9 100644 --- a/flink-filesystems/flink-oss-fs-hadoop/pom.xml +++ b/flink-filesystems/flink-oss-fs-hadoop/pom.xml @@ -76,6 +76,23 @@ under the License. test test-jar + + + + org.apache.flink + flink-core + ${project.version} + test + test-jar + + + + org.apache.flink + flink-hadoop-fs + ${project.version} + test + test-jar + diff --git a/flink-filesystems/flink-oss-fs-hadoop/src/test/java/org/apache/flink/fs/osshadoop/HadoopOSSFileSystemBehaviorITCase.java b/flink-filesystems/flink-oss-fs-hadoop/src/test/java/org/apache/flink/fs/osshadoop/HadoopOSSFileSystemBehaviorITCase.java new file mode 100644 index 00000000000000..b07a98b70dbbe2 --- /dev/null +++ b/flink-filesystems/flink-oss-fs-hadoop/src/test/java/org/apache/flink/fs/osshadoop/HadoopOSSFileSystemBehaviorITCase.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.fs.osshadoop; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.core.fs.FileSystemBehaviorTestSuite; +import org.apache.flink.core.fs.FileSystemKind; +import org.apache.flink.core.fs.Path; +import org.apache.flink.testutils.oss.OSSTestCredentials; + +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.UUID; + +/** + * An implementation of the {@link FileSystemBehaviorTestSuite} for the OSS file system. + */ +public class HadoopOSSFileSystemBehaviorITCase extends FileSystemBehaviorTestSuite { + private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID(); + + @BeforeClass + public static void setup() throws IOException { + OSSTestCredentials.assumeCredentialsAvailable(); + + final Configuration conf = new Configuration(); + conf.setString("fs.oss.endpoint", OSSTestCredentials.getOSSEndpoint()); + conf.setString("fs.oss.accessKeyId", OSSTestCredentials.getOSSAccessKey()); + conf.setString("fs.oss.accessKeySecret", OSSTestCredentials.getOSSSecretKey()); + FileSystem.initialize(conf); + } + + @Override + public FileSystem getFileSystem() throws Exception { + return getBasePath().getFileSystem(); + } + + @Override + public Path getBasePath() throws Exception { + return new Path(OSSTestCredentials.getTestBucketUri() + TEST_DATA_DIR); + } + + @Override + public FileSystemKind getFileSystemKind() { + return FileSystemKind.OBJECT_STORE; + } + + @AfterClass + public static void clearFsConfig() throws IOException { + FileSystem.initialize(new Configuration()); + } +} diff --git a/flink-filesystems/flink-oss-fs-hadoop/src/test/java/org/apache/flink/fs/osshadoop/HadoopOSSFileSystemITCase.java b/flink-filesystems/flink-oss-fs-hadoop/src/test/java/org/apache/flink/fs/osshadoop/HadoopOSSFileSystemITCase.java index 33f96b85333d95..4dd5e341fd0aca 100644 --- a/flink-filesystems/flink-oss-fs-hadoop/src/test/java/org/apache/flink/fs/osshadoop/HadoopOSSFileSystemITCase.java +++ b/flink-filesystems/flink-oss-fs-hadoop/src/test/java/org/apache/flink/fs/osshadoop/HadoopOSSFileSystemITCase.java @@ -19,84 +19,47 @@ package org.apache.flink.fs.osshadoop; import org.apache.flink.configuration.Configuration; -import org.apache.flink.core.fs.FSDataInputStream; -import org.apache.flink.core.fs.FSDataOutputStream; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; -import org.apache.flink.util.TestLogger; +import org.apache.flink.runtime.fs.hdfs.AbstractHadoopFileSystemITTest; +import org.apache.flink.testutils.oss.OSSTestCredentials; -import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; +import java.io.IOException; import java.util.UUID; import static junit.framework.TestCase.assertEquals; -import static junit.framework.TestCase.assertTrue; /** * Unit tests for the OSS file system support via AliyunOSSFileSystem. * These tests do actually read from or write to OSS. */ -public class HadoopOSSFileSystemITCase extends TestLogger { +public class HadoopOSSFileSystemITCase extends AbstractHadoopFileSystemITTest { - private static final String ENDPOINT = System.getenv("ARTIFACTS_OSS_ENDPOINT"); - private static final String BUCKET = System.getenv("ARTIFACTS_OSS_BUCKET"); private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID(); - private static final String ACCESS_KEY = System.getenv("ARTIFACTS_OSS_ACCESS_KEY"); - private static final String SECRET_KEY = System.getenv("ARTIFACTS_OSS_SECRET_KEY"); @BeforeClass - public static void checkIfCredentialsArePresent() { - Assume.assumeTrue("Aliyun OSS endpoint not configured, skipping test...", ENDPOINT != null); - Assume.assumeTrue("Aliyun OSS bucket not configured, skipping test...", BUCKET != null); - Assume.assumeTrue("Aliyun OSS access key not configured, skipping test...", ACCESS_KEY != null); - Assume.assumeTrue("Aliyun OSS secret key not configured, skipping test...", SECRET_KEY != null); - } + public static void setup() throws IOException { + OSSTestCredentials.assumeCredentialsAvailable(); - @Test - public void testReadAndWrite() throws Exception { final Configuration conf = new Configuration(); - conf.setString("fs.oss.endpoint", ENDPOINT); - conf.setString("fs.oss.accessKeyId", ACCESS_KEY); - conf.setString("fs.oss.accessKeySecret", SECRET_KEY); - final String testLine = "Aliyun OSS"; - + conf.setString("fs.oss.endpoint", OSSTestCredentials.getOSSEndpoint()); + conf.setString("fs.oss.accessKeyId", OSSTestCredentials.getOSSAccessKey()); + conf.setString("fs.oss.accessKeySecret", OSSTestCredentials.getOSSSecretKey()); FileSystem.initialize(conf); - final Path path = new Path("oss://" + BUCKET + '/' + TEST_DATA_DIR); - final FileSystem fs = path.getFileSystem(); - try { - for (int i = 0; i < 10; ++i) { - final Path file = new Path(path.getPath() + "/test.data." + i); - try (FSDataOutputStream out = fs.create(file, FileSystem.WriteMode.OVERWRITE)) { - try (OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { - writer.write(testLine); - } - } - try (FSDataInputStream in = fs.open(file); - InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8); - BufferedReader reader = new BufferedReader(ir)) { - String line = reader.readLine(); - assertEquals(testLine, line); - } - } - assertTrue(fs.exists(path)); - assertEquals(10, fs.listStatus(path).length); - } finally { - fs.delete(path, true); - } + basePath = new Path(OSSTestCredentials.getTestBucketUri() + TEST_DATA_DIR); + fs = basePath.getFileSystem(); + deadline = 0; } @Test public void testShadedConfigurations() { final Configuration conf = new Configuration(); - conf.setString("fs.oss.endpoint", ENDPOINT); - conf.setString("fs.oss.accessKeyId", ACCESS_KEY); - conf.setString("fs.oss.accessKeySecret", SECRET_KEY); + conf.setString("fs.oss.endpoint", OSSTestCredentials.getOSSEndpoint()); + conf.setString("fs.oss.accessKeyId", OSSTestCredentials.getOSSAccessKey()); + conf.setString("fs.oss.accessKeySecret", OSSTestCredentials.getOSSSecretKey()); conf.setString("fs.oss.credentials.provider", "org.apache.hadoop.fs.aliyun.oss.AliyunCredentialsProvider"); OSSFileSystemFactory ossfsFactory = new OSSFileSystemFactory(); @@ -106,8 +69,8 @@ public void testShadedConfigurations() { assertEquals("org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.aliyun.oss.AliyunCredentialsProvider", configuration.get("fs.oss.credentials.provider")); // should not shaded - assertEquals(ENDPOINT, configuration.get("fs.oss.endpoint")); - assertEquals(ACCESS_KEY, configuration.get("fs.oss.accessKeyId")); - assertEquals(SECRET_KEY, configuration.get("fs.oss.accessKeySecret")); + assertEquals(OSSTestCredentials.getOSSEndpoint(), configuration.get("fs.oss.endpoint")); + assertEquals(OSSTestCredentials.getOSSAccessKey(), configuration.get("fs.oss.accessKeyId")); + assertEquals(OSSTestCredentials.getOSSSecretKey(), configuration.get("fs.oss.accessKeySecret")); } } diff --git a/flink-filesystems/flink-s3-fs-hadoop/pom.xml b/flink-filesystems/flink-s3-fs-hadoop/pom.xml index 1355eda37af8db..530803d6ba87fc 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/pom.xml +++ b/flink-filesystems/flink-s3-fs-hadoop/pom.xml @@ -67,6 +67,13 @@ under the License. test-jar + + org.apache.flink + flink-hadoop-fs + ${project.version} + test + test-jar + diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemITCase.java b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemITCase.java index 70b5bce10a5b17..033bd781b90d87 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemITCase.java +++ b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemITCase.java @@ -19,31 +19,17 @@ package org.apache.flink.fs.s3hadoop; import org.apache.flink.configuration.Configuration; -import org.apache.flink.core.fs.FSDataInputStream; -import org.apache.flink.core.fs.FSDataOutputStream; -import org.apache.flink.core.fs.FileStatus; import org.apache.flink.core.fs.FileSystem; -import org.apache.flink.core.fs.FileSystem.WriteMode; import org.apache.flink.core.fs.Path; +import org.apache.flink.runtime.fs.hdfs.AbstractHadoopFileSystemITTest; import org.apache.flink.testutils.s3.S3TestCredentials; -import org.apache.flink.util.TestLogger; -import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Test; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; import java.util.UUID; -import static org.apache.flink.core.fs.FileSystemTestUtils.checkPathEventualExistence; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; /** * Unit tests for the S3 file system support via Hadoop's {@link org.apache.hadoop.fs.s3a.S3AFileSystem}. @@ -52,17 +38,10 @@ * consistency guarantees * and what the {@link org.apache.hadoop.fs.s3a.S3AFileSystem} offers. */ -public class HadoopS3FileSystemITCase extends TestLogger { - - /** - * Will be updated by {@link #checkCredentialsAndSetup()} if the test is not skipped. - */ - private static FileSystem fileSystem; - - private static Path basePath; +public class HadoopS3FileSystemITCase extends AbstractHadoopFileSystemITTest { @BeforeClass - public static void checkCredentialsAndSetup() throws IOException { + public static void setup() throws IOException { // check whether credentials exist S3TestCredentials.assumeCredentialsAvailable(); @@ -73,112 +52,11 @@ public static void checkCredentialsAndSetup() throws IOException { FileSystem.initialize(conf); basePath = new Path(S3TestCredentials.getTestBucketUri() + "tests-" + UUID.randomUUID()); - fileSystem = basePath.getFileSystem(); + fs = basePath.getFileSystem(); + deadline = System.nanoTime() + 30_000_000_000L; // check for uniqueness of the test directory // directory must not yet exist - assertFalse(fileSystem.exists(basePath)); - } - - @AfterClass - public static void cleanUp() throws IOException, InterruptedException { - try { - if (fileSystem != null) { - final long deadline = System.nanoTime() + 30_000_000_000L; // 30 secs - - // clean up - fileSystem.delete(basePath, true); - - // now directory must be gone - checkPathEventualExistence(fileSystem, basePath, false, deadline); - } - } - finally { - FileSystem.initialize(new Configuration()); - } - } - - @Test - public void testSimpleFileWriteAndRead() throws Exception { - final long deadline = System.nanoTime() + 30_000_000_000L; // 30 secs - final String testLine = "Hello Upload!"; - - final Path path = new Path(basePath, "test.txt"); - final FileSystem fs = path.getFileSystem(); - - try { - try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE); - OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { - writer.write(testLine); - } - - // just in case, wait for the path to exist - checkPathEventualExistence(fs, path, true, deadline); - - try (FSDataInputStream in = fs.open(path); - InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8); - BufferedReader reader = new BufferedReader(ir)) { - String line = reader.readLine(); - assertEquals(testLine, line); - } - } - finally { - fs.delete(path, false); - } - - // now file must be gone (this is eventually-consistent!) - checkPathEventualExistence(fs, path, false, deadline); - } - - @Test - public void testDirectoryListing() throws Exception { - final long deadline = System.nanoTime() + 30_000_000_000L; // 30 secs - - final Path directory = new Path(basePath, "testdir/"); - final FileSystem fs = directory.getFileSystem(); - - // directory must not yet exist - assertFalse(fs.exists(directory)); - - try { - // create directory - assertTrue(fs.mkdirs(directory)); - - checkPathEventualExistence(fs, directory, true, deadline); - - // directory empty - assertEquals(0, fs.listStatus(directory).length); - - // create some files - final int numFiles = 3; - for (int i = 0; i < numFiles; i++) { - Path file = new Path(directory, "/file-" + i); - try (FSDataOutputStream out = fs.create(file, WriteMode.OVERWRITE); - OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { - writer.write("hello-" + i + "\n"); - } - // just in case, wait for the file to exist (should then also be reflected in the - // directory's file list below) - checkPathEventualExistence(fs, file, true, deadline); - } - - FileStatus[] files = fs.listStatus(directory); - assertNotNull(files); - assertEquals(3, files.length); - - for (FileStatus status : files) { - assertFalse(status.isDir()); - } - - // now that there are files, the directory must exist - assertTrue(fs.exists(directory)); - } - finally { - // clean up - fs.delete(directory, true); - } - - // now directory must be gone (this is eventually-consistent, though!) - checkPathEventualExistence(fs, directory, false, deadline); + assertFalse(fs.exists(basePath)); } } diff --git a/flink-filesystems/flink-s3-fs-presto/pom.xml b/flink-filesystems/flink-s3-fs-presto/pom.xml index 913f4ade62fbef..1496de685d10b6 100644 --- a/flink-filesystems/flink-s3-fs-presto/pom.xml +++ b/flink-filesystems/flink-s3-fs-presto/pom.xml @@ -190,6 +190,14 @@ under the License. test test-jar + + + org.apache.flink + flink-hadoop-fs + ${project.version} + test + test-jar + diff --git a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java index 9a008a176d66aa..985ad7a2566763 100644 --- a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java +++ b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java @@ -19,35 +19,23 @@ package org.apache.flink.fs.s3presto; import org.apache.flink.configuration.Configuration; -import org.apache.flink.core.fs.FSDataInputStream; -import org.apache.flink.core.fs.FSDataOutputStream; -import org.apache.flink.core.fs.FileStatus; import org.apache.flink.core.fs.FileSystem; -import org.apache.flink.core.fs.FileSystem.WriteMode; import org.apache.flink.core.fs.Path; +import org.apache.flink.runtime.fs.hdfs.AbstractHadoopFileSystemITTest; import org.apache.flink.testutils.s3.S3TestCredentials; -import org.apache.flink.util.TestLogger; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; import java.util.UUID; import static com.facebook.presto.hive.PrestoS3FileSystem.S3_USE_INSTANCE_CREDENTIALS; -import static org.apache.flink.core.fs.FileSystemTestUtils.checkPathEventualExistence; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -58,7 +46,7 @@ * and what the {@link com.facebook.presto.hive.PrestoS3FileSystem} offers. */ @RunWith(Parameterized.class) -public class PrestoS3FileSystemITCase extends TestLogger { +public class PrestoS3FileSystemITCase extends AbstractHadoopFileSystemITTest { @Parameterized.Parameter public String scheme; @@ -71,13 +59,26 @@ public static List parameters() { private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID(); @BeforeClass - public static void checkIfCredentialsArePresent() { + public static void setup() throws IOException { S3TestCredentials.assumeCredentialsAvailable(); + // initialize configuration with valid credentials + final Configuration conf = new Configuration(); + conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey()); + conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey()); + FileSystem.initialize(conf); + + basePath = new Path(S3TestCredentials.getTestBucketUri() + TEST_DATA_DIR); + fs = basePath.getFileSystem(); + deadline = System.nanoTime() + 30_000_000_000L; + + // check for uniqueness of the test directory + // directory must not yet exist + assertFalse(fs.exists(basePath)); } @Test public void testConfigKeysForwarding() throws Exception { - final Path path = new Path(S3TestCredentials.getTestBucketUri() + TEST_DATA_DIR); + final Path path = basePath; // access without credentials should fail { @@ -140,93 +141,9 @@ public void testConfigKeysForwarding() throws Exception { FileSystem.initialize(new Configuration()); } - @Test - public void testSimpleFileWriteAndRead() throws Exception { - final long deadline = System.nanoTime() + 30_000_000_000L; // 30 secs - final Configuration conf = new Configuration(); - conf.setString("s3.access-key", S3TestCredentials.getS3AccessKey()); - conf.setString("s3.secret-key", S3TestCredentials.getS3SecretKey()); - - final String testLine = "Hello Upload!"; - - FileSystem.initialize(conf); - - final Path path = new Path(S3TestCredentials.getTestBucketUri() + TEST_DATA_DIR + "/test.txt"); - final FileSystem fs = path.getFileSystem(); - - try { - try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE); - OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { - writer.write(testLine); - } - - try (FSDataInputStream in = fs.open(path); - InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8); - BufferedReader reader = new BufferedReader(ir)) { - String line = reader.readLine(); - assertEquals(testLine, line); - } - } - finally { - fs.delete(path, false); - } - - // now file must be gone (this is eventually-consistent!) - checkPathEventualExistence(fs, path, false, deadline); - } - - @Test - public void testDirectoryListing() throws Exception { - final long deadline = System.nanoTime() + 30_000_000_000L; // 30 secs - final Configuration conf = new Configuration(); - conf.setString("s3.access-key", S3TestCredentials.getS3AccessKey()); - conf.setString("s3.secret-key", S3TestCredentials.getS3SecretKey()); - - FileSystem.initialize(conf); - - final Path directory = new Path(S3TestCredentials.getTestBucketUri() + TEST_DATA_DIR + "/testdir/"); - final FileSystem fs = directory.getFileSystem(); - - // directory must not yet exist - assertFalse(fs.exists(directory)); - - try { - // create directory - assertTrue(fs.mkdirs(directory)); - - // seems the presto file system does not assume existence of empty directories in S3 -// assertTrue(fs.exists(directory)); - - // directory empty - assertEquals(0, fs.listStatus(directory).length); - - // create some files - final int numFiles = 3; - for (int i = 0; i < numFiles; i++) { - Path file = new Path(directory, "/file-" + i); - try (FSDataOutputStream out = fs.create(file, WriteMode.OVERWRITE); - OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { - writer.write("hello-" + i + "\n"); - } - } - - FileStatus[] files = fs.listStatus(directory); - assertNotNull(files); - assertEquals(3, files.length); - - for (FileStatus status : files) { - assertFalse(status.isDir()); - } - - // now that there are files, the directory must exist - assertTrue(fs.exists(directory)); - } - finally { - // clean up - fs.delete(directory, true); - } - - // now directory must be gone (this is eventually-consistent!) - checkPathEventualExistence(fs, directory, false, deadline); + @Override + protected void checkEmptyDirectory(Path path) throws IOException, InterruptedException { + // seems the presto file system does not assume existence of empty directories in S3 + // do nothing as before } } diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/oss/OSSTestCredentials.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/oss/OSSTestCredentials.java new file mode 100644 index 00000000000000..052b4cc0572986 --- /dev/null +++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/testutils/oss/OSSTestCredentials.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.testutils.oss; + +import org.junit.Assume; + +import javax.annotation.Nullable; + +/** + * Access to credentials to access OSS buckets during integration tests. + */ +public class OSSTestCredentials { + @Nullable + private static final String ENDPOINT = System.getenv("ARTIFACTS_OSS_ENDPOINT"); + + @Nullable + private static final String BUCKET = System.getenv("ARTIFACTS_OSS_BUCKET"); + + @Nullable + private static final String ACCESS_KEY = System.getenv("ARTIFACTS_OSS_ACCESS_KEY"); + + @Nullable + private static final String SECRET_KEY = System.getenv("ARTIFACTS_OSS_SECRET_KEY"); + + // ------------------------------------------------------------------------ + + public static boolean credentialsAvailable() { + return ENDPOINT != null && BUCKET != null && ACCESS_KEY != null && SECRET_KEY != null; + } + + public static void assumeCredentialsAvailable() { + Assume.assumeTrue("No OSS credentials available in this test's environment", credentialsAvailable()); + } + + /** + * Get OSS endpoint used to connect. + * @return OSS endpoint + */ + public static String getOSSEndpoint() { + if (ENDPOINT != null) { + return ENDPOINT; + } else { + throw new IllegalStateException("OSS endpoint is not available"); + } + } + + /** + * Get OSS access key. + * @return OSS access key + */ + public static String getOSSAccessKey() { + if (ACCESS_KEY != null) { + return ACCESS_KEY; + } else { + throw new IllegalStateException("OSS access key is not available"); + } + } + + /** + * Get OSS secret key. + * @return OSS secret key + */ + public static String getOSSSecretKey() { + if (SECRET_KEY != null) { + return SECRET_KEY; + } else { + throw new IllegalStateException("OSS secret key is not available"); + } + } + + public static String getTestBucketUri() { + return getTestBucketUriWithScheme("oss"); + } + + public static String getTestBucketUriWithScheme(String scheme) { + if (BUCKET != null) { + return scheme + "://" + BUCKET + "/"; + } + else { + throw new IllegalStateException("OSS test bucket is not available"); + } + } +} From 171a3b143b9d70faa6982878fed936bfee580a8f Mon Sep 17 00:00:00 2001 From: "zhuzhu.zz" Date: Thu, 6 Dec 2018 19:37:05 +0800 Subject: [PATCH 142/147] [FLINK-10945] Add an InputDependencyConstraint to avoid resource deadlocks for finite stream jobs when resources are limited This commit adds a job config InputDependencyConstraint, which helps to avoid resource deadlocks in LAZY_FROM_SOURCES scheduling when resources are limited. The InputDependencyConstraint controls across multiple inputs when consumers are scheduled. Currently it supports ANY and ALL. ANY means that any input intermediate result partition must be consumable and ALL means that all input intermediate result partitions (from all inputs) need to be consumable in order to schedule the consumer task. This closes #7255. --- .../flink/api/common/ExecutionConfig.java | 33 ++- .../api/common/InputDependencyConstraint.java | 35 +++ .../plantranslate/JobGraphGenerator.java | 1 + .../InputChannelDeploymentDescriptor.java | 6 +- .../runtime/executiongraph/Execution.java | 46 ++-- .../executiongraph/ExecutionJobVertex.java | 5 + .../executiongraph/ExecutionVertex.java | 37 +++ .../executiongraph/IntermediateResult.java | 12 +- .../IntermediateResultPartition.java | 23 +- .../flink/runtime/jobgraph/JobVertex.java | 12 + .../InputChannelDeploymentDescriptorTest.java | 2 + .../ExecutionVertexInputConstraintTest.java | 214 ++++++++++++++++++ .../IntermediateResultPartitionTest.java | 99 ++++++++ .../api/graph/StreamingJobGraphGenerator.java | 3 + 14 files changed, 499 insertions(+), 29 deletions(-) create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/InputDependencyConstraint.java create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexInputConstraintTest.java create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/IntermediateResultPartitionTest.java diff --git a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java index 16b2adfd1963ca..bee67cb51fdef5 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java @@ -159,6 +159,9 @@ public class ExecutionConfig implements Serializable, Archiveable { + try { + final ExecutionGraph executionGraph = consumerVertex.getExecutionGraph(); + consumerVertex.scheduleForExecution( + executionGraph.getSlotProvider(), + executionGraph.isQueuedSchedulingAllowed(), + LocationPreferenceConstraint.ANY, // there must be at least one known location + Collections.emptySet()); + } catch (Throwable t) { + consumerVertex.fail(new IllegalStateException("Could not schedule consumer " + + "vertex " + consumerVertex, t)); + } + + return null; + }, + executor); + } + void scheduleOrUpdateConsumers(List> allConsumers) { final int numConsumers = allConsumers.size(); @@ -755,23 +776,16 @@ else if (numConsumers == 0) { // TODO The current approach may send many update messages even though the consuming // task has already been deployed with all necessary information. We have to check // whether this is a problem and fix it, if it is. - CompletableFuture.supplyAsync( - () -> { - try { - final ExecutionGraph executionGraph = consumerVertex.getExecutionGraph(); - consumerVertex.scheduleForExecution( - executionGraph.getSlotProvider(), - executionGraph.isQueuedSchedulingAllowed(), - LocationPreferenceConstraint.ANY, // there must be at least one known location - Collections.emptySet()); - } catch (Throwable t) { - consumerVertex.fail(new IllegalStateException("Could not schedule consumer " + - "vertex " + consumerVertex, t)); - } - return null; - }, - executor); + // Schedule the consumer vertex if its inputs constraint is satisfied, otherwise skip the scheduling. + // A shortcut of input constraint check is added for InputDependencyConstraint.ANY since + // at least one of the consumer vertex's inputs is consumable here. This is to avoid the + // O(N) complexity introduced by input constraint check for InputDependencyConstraint.ANY, + // as we do not want the default scheduling performance to be affected. + if (consumerVertex.getInputDependencyConstraint() == InputDependencyConstraint.ANY || + consumerVertex.checkInputDependencyConstraints()) { + scheduleConsumer(consumerVertex); + } // double check to resolve race conditions if (consumerVertex.getExecutionState() == RUNNING) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java index ee735050976757..94ee45d65927d3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.Archiveable; import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.InputDependencyConstraint; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.accumulators.Accumulator; import org.apache.flink.api.common.accumulators.AccumulatorHelper; @@ -366,6 +367,10 @@ public List getInputs() { return inputs; } + public InputDependencyConstraint getInputDependencyConstraint() { + return getJobVertex().getInputDependencyConstraint(); + } + public Either, PermanentBlobKey> getTaskInformationOrBlobKey() throws IOException { // only one thread should offload the task information, so let's also let only one thread // serialize the task information! diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java index a0747296c5399d..1217217333504f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java @@ -20,6 +20,7 @@ import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.Archiveable; +import org.apache.flink.api.common.InputDependencyConstraint; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.time.Time; import org.apache.flink.configuration.JobManagerOptions; @@ -61,6 +62,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -70,6 +72,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.stream.IntStream; import static org.apache.flink.runtime.execution.ExecutionState.FINISHED; @@ -340,6 +343,10 @@ public Map getProduc return resultPartitions; } + public InputDependencyConstraint getInputDependencyConstraint() { + return getJobVertex().getInputDependencyConstraint(); + } + // -------------------------------------------------------------------------------------------- // Graph building // -------------------------------------------------------------------------------------------- @@ -684,6 +691,8 @@ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } + partition.markDataProduced(); + if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition execution.scheduleOrUpdateConsumers(partition.getConsumers()); @@ -726,6 +735,34 @@ List finishAllBlockingPartitions() { } } + /** + * Check whether the InputDependencyConstraint is satisfied for this vertex. + * + * @return whether the input constraint is satisfied + */ + boolean checkInputDependencyConstraints() { + if (getInputDependencyConstraint() == InputDependencyConstraint.ANY) { + // InputDependencyConstraint == ANY + return IntStream.range(0, inputEdges.length).anyMatch(this::isInputConsumable); + } else { + // InputDependencyConstraint == ALL + return IntStream.range(0, inputEdges.length).allMatch(this::isInputConsumable); + } + } + + /** + * Get whether an input of the vertex is consumable. + * An input is consumable when when any partition in it is consumable. + * + * Note that a BLOCKING result partition is only consumable when all partitions in the result are FINISHED. + * + * @return whether the input is consumable + */ + boolean isInputConsumable(int inputNumber) { + return Arrays.stream(inputEdges[inputNumber]).map(ExecutionEdge::getSource).anyMatch( + IntermediateResultPartition::isConsumable); + } + // -------------------------------------------------------------------------------------------- // Notifications from the Execution Attempt // -------------------------------------------------------------------------------------------- diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResult.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResult.java index 313272cf86bfba..6e1d9ba69fbd7f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResult.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResult.java @@ -156,19 +156,17 @@ public int getConnectionIndex() { void resetForNewExecution() { this.numberOfRunningProducers.set(numParallelProducers); + for (IntermediateResultPartition partition : partitions) { + partition.resetForNewExecution(); + } } int decrementNumberOfRunningProducersAndGetRemaining() { return numberOfRunningProducers.decrementAndGet(); } - boolean isConsumable() { - if (resultType.isPipelined()) { - return true; - } - else { - return numberOfRunningProducers.get() == 0; - } + boolean areAllPartitionsFinished() { + return numberOfRunningProducers.get() == 0; } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResultPartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResultPartition.java index 124ceb2b6bcd04..d4e85cf11c10f3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResultPartition.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResultPartition.java @@ -36,6 +36,11 @@ public class IntermediateResultPartition { private List> consumers; + /** + * Whether this partition has produced some data. + */ + private boolean hasDataProduced = false; + public IntermediateResultPartition(IntermediateResult totalResult, ExecutionVertex producer, int partitionNumber) { this.totalResult = totalResult; this.producer = producer; @@ -60,7 +65,7 @@ public IntermediateResultPartitionID getPartitionId() { return partitionId; } - ResultPartitionType getResultType() { + public ResultPartitionType getResultType() { return totalResult.getResultType(); } @@ -68,8 +73,20 @@ public List> getConsumers() { return consumers; } + public void markDataProduced() { + hasDataProduced = true; + } + public boolean isConsumable() { - return totalResult.isConsumable(); + if (getResultType().isPipelined()) { + return hasDataProduced; + } else { + return totalResult.areAllPartitionsFinished(); + } + } + + void resetForNewExecution() { + hasDataProduced = false; } int addConsumerGroup() { @@ -94,6 +111,8 @@ boolean markFinished() { throw new IllegalStateException("Tried to mark a non-blocking result partition as finished"); } + hasDataProduced = true; + final int refCnt = totalResult.decrementNumberOfRunningProducersAndGetRemaining(); if (refCnt == 0) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobVertex.java index 1fe95ebf010542..c78d707029b5a9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobVertex.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobVertex.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.jobgraph; import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.InputDependencyConstraint; import org.apache.flink.api.common.operators.ResourceSpec; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.io.InputSplitSource; @@ -112,6 +113,9 @@ public class JobVertex implements java.io.Serializable { * to be included in the JSON plan */ private String resultOptimizerProperties; + /** The input dependency constraint to schedule this vertex. */ + private InputDependencyConstraint inputDependencyConstraint = InputDependencyConstraint.ANY; + // -------------------------------------------------------------------------------------------- /** @@ -557,6 +561,14 @@ public void setResultOptimizerProperties(String resultOptimizerProperties) { this.resultOptimizerProperties = resultOptimizerProperties; } + public InputDependencyConstraint getInputDependencyConstraint() { + return inputDependencyConstraint; + } + + public void setInputDependencyConstraint(InputDependencyConstraint inputDependencyConstraint) { + this.inputDependencyConstraint = inputDependencyConstraint; + } + // -------------------------------------------------------------------------------------------- @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/InputChannelDeploymentDescriptorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/InputChannelDeploymentDescriptorTest.java index 6aa36b70ab8893..4f7417f27fd1f8 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/InputChannelDeploymentDescriptorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/InputChannelDeploymentDescriptorTest.java @@ -29,6 +29,7 @@ import org.apache.flink.runtime.executiongraph.IntermediateResultPartition; import org.apache.flink.runtime.io.network.ConnectionID; import org.apache.flink.runtime.io.network.partition.ResultPartitionID; +import org.apache.flink.runtime.io.network.partition.ResultPartitionType; import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID; import org.apache.flink.runtime.jobmaster.LogicalSlot; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; @@ -192,6 +193,7 @@ private static ExecutionVertex mockExecutionVertex(ExecutionState state, Resourc private static IntermediateResultPartition mockPartition(ExecutionVertex producer) { IntermediateResultPartition partition = mock(IntermediateResultPartition.class); + when(partition.getResultType()).thenReturn(ResultPartitionType.PIPELINED); when(partition.isConsumable()).thenReturn(true); IntermediateResult result = mock(IntermediateResult.class); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexInputConstraintTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexInputConstraintTest.java new file mode 100644 index 00000000000000..3c7354adc2f8ba --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexInputConstraintTest.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.executiongraph; + +import org.apache.flink.api.common.InputDependencyConstraint; +import org.apache.flink.api.common.JobID; +import org.apache.flink.runtime.akka.AkkaUtils; +import org.apache.flink.runtime.execution.ExecutionState; +import org.apache.flink.runtime.executiongraph.failover.RestartAllStrategy; +import org.apache.flink.runtime.executiongraph.restart.FixedDelayRestartStrategy; +import org.apache.flink.runtime.executiongraph.utils.SimpleSlotProvider; +import org.apache.flink.runtime.io.network.partition.ResultPartitionID; +import org.apache.flink.runtime.io.network.partition.ResultPartitionType; +import org.apache.flink.runtime.jobgraph.DistributionPattern; +import org.apache.flink.runtime.jobgraph.JobStatus; +import org.apache.flink.runtime.jobgraph.JobVertex; +import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable; +import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; +import org.apache.flink.runtime.testingUtils.TestingUtils; +import org.apache.flink.util.TestLogger; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.isInExecutionState; +import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitForAllExecutionsPredicate; +import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitUntilExecutionVertexState; +import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitUntilJobStatus; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests for the inputs constraint for {@link ExecutionVertex}. + */ +public class ExecutionVertexInputConstraintTest extends TestLogger { + + @Test + public void testInputConsumable() throws Exception { + List vertices = createOrderedVertices(); + ExecutionGraph eg = createExecutionGraph(vertices, InputDependencyConstraint.ALL); + ExecutionVertex ev11 = eg.getJobVertex(vertices.get(0).getID()).getTaskVertices()[0]; + ExecutionVertex ev21 = eg.getJobVertex(vertices.get(1).getID()).getTaskVertices()[0]; + ExecutionVertex ev22 = eg.getJobVertex(vertices.get(1).getID()).getTaskVertices()[1]; + ExecutionVertex ev31 = eg.getJobVertex(vertices.get(2).getID()).getTaskVertices()[0]; + ExecutionVertex ev32 = eg.getJobVertex(vertices.get(2).getID()).getTaskVertices()[1]; + + eg.scheduleForExecution(); + + // Inputs not consumable on init + assertFalse(ev31.isInputConsumable(0)); + assertFalse(ev31.isInputConsumable(1)); + + // One pipelined input consumable on data produced + IntermediateResultPartition partition11 = ev11.getProducedPartitions().values().iterator().next(); + ev11.scheduleOrUpdateConsumers(new ResultPartitionID(partition11.getPartitionId(), + ev11.getCurrentExecutionAttempt().getAttemptId())); + assertTrue(ev31.isInputConsumable(0)); + // Input0 of ev32 is not consumable. It consumes the same PIPELINED result with ev31 but not the same partition + assertFalse(ev32.isInputConsumable(0)); + + // The blocking input not consumable if only one partition is FINISHED + ev21.getCurrentExecutionAttempt().markFinished(); + assertFalse(ev31.isInputConsumable(1)); + + // The blocking input consumable if all partitions are FINISHED + ev22.getCurrentExecutionAttempt().markFinished(); + assertTrue(ev31.isInputConsumable(1)); + + // Inputs not consumable after failover + ev11.fail(new Exception()); + waitUntilJobRestarted(eg); + assertFalse(ev31.isInputConsumable(0)); + assertFalse(ev31.isInputConsumable(1)); + } + + @Test + public void testInputConstraintANY() throws Exception { + List vertices = createOrderedVertices(); + ExecutionGraph eg = createExecutionGraph(vertices, InputDependencyConstraint.ANY); + ExecutionVertex ev11 = eg.getJobVertex(vertices.get(0).getID()).getTaskVertices()[0]; + ExecutionVertex ev21 = eg.getJobVertex(vertices.get(1).getID()).getTaskVertices()[0]; + ExecutionVertex ev22 = eg.getJobVertex(vertices.get(1).getID()).getTaskVertices()[1]; + ExecutionVertex ev31 = eg.getJobVertex(vertices.get(2).getID()).getTaskVertices()[0]; + + eg.scheduleForExecution(); + + // Inputs constraint not satisfied on init + assertFalse(ev31.checkInputDependencyConstraints()); + + // Input1 consumable satisfies the constraint + IntermediateResultPartition partition11 = ev11.getProducedPartitions().values().iterator().next(); + ev11.scheduleOrUpdateConsumers(new ResultPartitionID(partition11.getPartitionId(), + ev11.getCurrentExecutionAttempt().getAttemptId())); + assertTrue(ev31.checkInputDependencyConstraints()); + + // Inputs constraint not satisfied after failover + ev11.fail(new Exception()); + waitUntilJobRestarted(eg); + assertFalse(ev31.checkInputDependencyConstraints()); + + // Input2 consumable satisfies the constraint + waitUntilExecutionVertexState(ev21, ExecutionState.DEPLOYING, 2000L); + waitUntilExecutionVertexState(ev22, ExecutionState.DEPLOYING, 2000L); + ev21.getCurrentExecutionAttempt().markFinished(); + ev22.getCurrentExecutionAttempt().markFinished(); + assertTrue(ev31.checkInputDependencyConstraints()); + } + + @Test + public void testInputConstraintALL() throws Exception { + List vertices = createOrderedVertices(); + ExecutionGraph eg = createExecutionGraph(vertices, InputDependencyConstraint.ALL); + ExecutionVertex ev11 = eg.getJobVertex(vertices.get(0).getID()).getTaskVertices()[0]; + ExecutionVertex ev21 = eg.getJobVertex(vertices.get(1).getID()).getTaskVertices()[0]; + ExecutionVertex ev22 = eg.getJobVertex(vertices.get(1).getID()).getTaskVertices()[1]; + ExecutionVertex ev31 = eg.getJobVertex(vertices.get(2).getID()).getTaskVertices()[0]; + + eg.scheduleForExecution(); + + // Inputs constraint not satisfied on init + assertFalse(ev31.checkInputDependencyConstraints()); + + // Input1 consumable does not satisfy the constraint + IntermediateResultPartition partition11 = ev11.getProducedPartitions().values().iterator().next(); + ev11.scheduleOrUpdateConsumers(new ResultPartitionID(partition11.getPartitionId(), + ev11.getCurrentExecutionAttempt().getAttemptId())); + assertFalse(ev31.checkInputDependencyConstraints()); + + // Input2 consumable satisfies the constraint + ev21.getCurrentExecutionAttempt().markFinished(); + ev22.getCurrentExecutionAttempt().markFinished(); + assertTrue(ev31.checkInputDependencyConstraints()); + + // Inputs constraint not satisfied after failover + ev11.fail(new Exception()); + waitUntilJobRestarted(eg); + assertFalse(ev31.checkInputDependencyConstraints()); + } + + private static List createOrderedVertices() { + JobVertex v1 = new JobVertex("vertex1"); + JobVertex v2 = new JobVertex("vertex2"); + JobVertex v3 = new JobVertex("vertex3"); + v1.setParallelism(2); + v2.setParallelism(2); + v3.setParallelism(2); + v1.setInvokableClass(AbstractInvokable.class); + v2.setInvokableClass(AbstractInvokable.class); + v3.setInvokableClass(AbstractInvokable.class); + v3.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); + v3.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); + return Arrays.asList(v1, v2, v3); + } + + private static ExecutionGraph createExecutionGraph( + List orderedVertices, + InputDependencyConstraint inputDependencyConstraint) throws Exception { + + final JobID jobId = new JobID(); + final String jobName = "Test Job Sample Name"; + final SlotProvider slotProvider = new SimpleSlotProvider(jobId, 20); + + for (JobVertex vertex : orderedVertices) { + vertex.setInputDependencyConstraint(inputDependencyConstraint); + } + + ExecutionGraph eg = new ExecutionGraph( + new DummyJobInformation( + jobId, + jobName), + TestingUtils.defaultExecutor(), + TestingUtils.defaultExecutor(), + AkkaUtils.getDefaultTimeout(), + new FixedDelayRestartStrategy(1, 0), + new RestartAllStrategy.Factory(), + slotProvider); + eg.attachJobGraph(orderedVertices); + + return eg; + } + + private void waitUntilJobRestarted(ExecutionGraph eg) throws Exception { + waitForAllExecutionsPredicate(eg, + isInExecutionState(ExecutionState.CANCELING) + .or(isInExecutionState(ExecutionState.CANCELED)) + .or(isInExecutionState(ExecutionState.FAILED)) + .or(isInExecutionState(ExecutionState.FINISHED)), + 2000L); + + for (ExecutionVertex ev : eg.getAllExecutionVertices()) { + if (ev.getCurrentExecutionAttempt().getState() == ExecutionState.CANCELING) { + ev.getCurrentExecutionAttempt().cancelingComplete(); + } + } + waitUntilJobStatus(eg, JobStatus.RUNNING, 2000L); + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/IntermediateResultPartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/IntermediateResultPartitionTest.java new file mode 100644 index 00000000000000..e20c0bd7efac2a --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/IntermediateResultPartitionTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.executiongraph; + +import org.apache.flink.api.common.time.Time; +import org.apache.flink.runtime.io.network.partition.ResultPartitionType; +import org.apache.flink.runtime.jobgraph.IntermediateDataSetID; +import org.apache.flink.runtime.jobgraph.JobVertexID; +import org.apache.flink.runtime.testutils.DirectScheduledExecutorService; +import org.apache.flink.util.TestLogger; +import org.junit.Test; + +import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.getExecutionVertex; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests for {@link IntermediateResultPartition}. + */ +public class IntermediateResultPartitionTest extends TestLogger { + + @Test + public void testPipelinedPartitionConsumable() throws Exception { + IntermediateResult result = createResult(ResultPartitionType.PIPELINED, 2); + IntermediateResultPartition partition1 = result.getPartitions()[0]; + IntermediateResultPartition partition2 = result.getPartitions()[1]; + + // Not consumable on init + assertFalse(partition1.isConsumable()); + assertFalse(partition2.isConsumable()); + + // Partition 1 consumable after data are produced + partition1.markDataProduced(); + assertTrue(partition1.isConsumable()); + assertFalse(partition2.isConsumable()); + + // Not consumable if failover happens + result.resetForNewExecution(); + assertFalse(partition1.isConsumable()); + assertFalse(partition2.isConsumable()); + } + + @Test + public void testBlockingPartitionConsumable() throws Exception { + IntermediateResult result = createResult(ResultPartitionType.BLOCKING, 2); + IntermediateResultPartition partition1 = result.getPartitions()[0]; + IntermediateResultPartition partition2 = result.getPartitions()[1]; + + // Not consumable on init + assertFalse(partition1.isConsumable()); + assertFalse(partition2.isConsumable()); + + // Not consumable if only one partition is FINISHED + partition1.markFinished(); + assertFalse(partition1.isConsumable()); + assertFalse(partition2.isConsumable()); + + // Consumable after all partitions are FINISHED + partition2.markFinished(); + assertTrue(partition1.isConsumable()); + assertTrue(partition2.isConsumable()); + + // Not consumable if failover happens + result.resetForNewExecution(); + assertFalse(partition1.isConsumable()); + assertFalse(partition2.isConsumable()); + } + + private static IntermediateResult createResult( + ResultPartitionType resultPartitionType, + int producerCount) throws Exception { + + ExecutionJobVertex jobVertex = getExecutionVertex(new JobVertexID(), new DirectScheduledExecutorService()); + IntermediateResult result = + new IntermediateResult(new IntermediateDataSetID(), jobVertex, producerCount, resultPartitionType); + for (int i = 0; i < producerCount; i++) { + // Generate result partition in the result + new ExecutionVertex(jobVertex, i, new IntermediateResult[]{result}, Time.minutes(1)); + } + + return result; + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java index 69213024975e5c..bc0377b79320fe 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java @@ -404,6 +404,9 @@ private StreamConfig createJobVertex( LOG.debug("Parallelism set: {} for {}", parallelism, streamNodeId); } + // TODO: inherit InputDependencyConstraint from the head operator + jobVertex.setInputDependencyConstraint(streamGraph.getExecutionConfig().getDefaultInputDependencyConstraint()); + jobVertices.put(streamNodeId, jobVertex); builtVertices.add(streamNodeId); jobGraph.addVertex(jobVertex); From 94647f1ca02ec9614941e40f62ed9aca0871b59d Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 16 Jan 2019 14:16:27 +0100 Subject: [PATCH 143/147] [FLINK-10945] Mark InputDependencyConstraint as PublicEvolving --- .../java/org/apache/flink/api/common/ExecutionConfig.java | 6 ++++-- .../apache/flink/api/common/InputDependencyConstraint.java | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java index bee67cb51fdef5..d288810c094d22 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java @@ -525,10 +525,11 @@ public ExecutionMode getExecutionMode() { * Sets the default input dependency constraint for vertex scheduling. It indicates when a task * should be scheduled considering its inputs status. * - * The default constraint is {@link InputDependencyConstraint#ANY}. + *

The default constraint is {@link InputDependencyConstraint#ANY}. * * @param inputDependencyConstraint The input dependency constraint. */ + @PublicEvolving public void setDefaultInputDependencyConstraint(InputDependencyConstraint inputDependencyConstraint) { this.defaultInputDependencyConstraint = inputDependencyConstraint; } @@ -537,10 +538,11 @@ public void setDefaultInputDependencyConstraint(InputDependencyConstraint inputD * Gets the default input dependency constraint for vertex scheduling. It indicates when a task * should be scheduled considering its inputs status. * - * The default constraint is {@link InputDependencyConstraint#ANY}. + *

The default constraint is {@link InputDependencyConstraint#ANY}. * * @return The input dependency constraint of this job. */ + @PublicEvolving public InputDependencyConstraint getDefaultInputDependencyConstraint() { return defaultInputDependencyConstraint; } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/InputDependencyConstraint.java b/flink-core/src/main/java/org/apache/flink/api/common/InputDependencyConstraint.java index c7a1e6a3519812..6738ebacb73423 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/InputDependencyConstraint.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/InputDependencyConstraint.java @@ -18,9 +18,12 @@ package org.apache.flink.api.common; +import org.apache.flink.annotation.PublicEvolving; + /** * This constraint indicates when a task should be scheduled considering its inputs status. */ +@PublicEvolving public enum InputDependencyConstraint { /** From f61d1129e277a23fcf6113211c344e8b89da49e8 Mon Sep 17 00:00:00 2001 From: eaglewatcherwb Date: Sun, 13 Jan 2019 10:23:08 +0800 Subject: [PATCH 144/147] [FLINK-11295][conf] Rename configuration options of queryable-state from query.x to queryable-state.x Change-Id: Idb561da3982de07e77a05ffef9ad227094f527b0 --- .../queryable_state_configuration.html | 24 ++++++------- docs/dev/stream/state/queryable_state.md | 12 +++---- .../kubernetes/job-cluster-job.yaml.template | 2 +- .../configuration/QueryableStateOptions.java | 35 +++++++++++-------- flink-end-to-end-tests/test-scripts/common.sh | 4 +-- .../job-cluster-job.yaml.template | 2 +- 6 files changed, 43 insertions(+), 36 deletions(-) diff --git a/docs/_includes/generated/queryable_state_configuration.html b/docs/_includes/generated/queryable_state_configuration.html index 5940fdc822991d..91eaf912a0cee1 100644 --- a/docs/_includes/generated/queryable_state_configuration.html +++ b/docs/_includes/generated/queryable_state_configuration.html @@ -8,44 +8,44 @@ -

query.client.network-threads
+
queryable-state.client.network-threads
0 Number of network (Netty's event loop) Threads for queryable state client. -
query.proxy.network-threads
+
queryable-state.enable
+ false + Option whether the queryable state proxy and server should be enabled where possible and configurable. + + +
queryable-state.proxy.network-threads
0 Number of network (Netty's event loop) Threads for queryable state proxy. -
query.proxy.ports
+
queryable-state.proxy.ports
"9069" The port range of the queryable state proxy. The specified range can be a single port: "9123", a range of ports: "50100-50200", or a list of ranges and ports: "50100-50200,50300-50400,51234". -
query.proxy.query-threads
+
queryable-state.proxy.query-threads
0 Number of query Threads for queryable state proxy. Uses the number of slots if set to 0. -
query.server.network-threads
+
queryable-state.server.network-threads
0 Number of network (Netty's event loop) Threads for queryable state server. -
query.server.ports
+
queryable-state.server.ports
"9067" The port range of the queryable state server. The specified range can be a single port: "9123", a range of ports: "50100-50200", or a list of ranges and ports: "50100-50200,50300-50400,51234". -
query.server.query-threads
+
queryable-state.server.query-threads
0 Number of query Threads for queryable state server. Uses the number of slots if set to 0. - -
queryable-state.enable
- false - Option whether the queryable state proxy and server should be enabled where possible and configurable. - diff --git a/docs/dev/stream/state/queryable_state.md b/docs/dev/stream/state/queryable_state.md index 4fd2186626dd31..7963487e6cd83d 100644 --- a/docs/dev/stream/state/queryable_state.md +++ b/docs/dev/stream/state/queryable_state.md @@ -290,19 +290,19 @@ The following configuration parameters influence the behaviour of the queryable They are defined in `QueryableStateOptions`. ### State Server -* `query.server.ports`: the server port range of the queryable state server. This is useful to avoid port clashes if more +* `queryable-state.server.ports`: the server port range of the queryable state server. This is useful to avoid port clashes if more than 1 task managers run on the same machine. The specified range can be: a port: "9123", a range of ports: "50100-50200", or a list of ranges and or points: "50100-50200,50300-50400,51234". The default port is 9067. -* `query.server.network-threads`: number of network (event loop) threads receiving incoming requests for the state server (0 => #slots) -* `query.server.query-threads`: number of threads handling/serving incoming requests for the state server (0 => #slots). +* `queryable-state.server.network-threads`: number of network (event loop) threads receiving incoming requests for the state server (0 => #slots) +* `queryable-state.server.query-threads`: number of threads handling/serving incoming requests for the state server (0 => #slots). ### Proxy -* `query.proxy.ports`: the server port range of the queryable state proxy. This is useful to avoid port clashes if more +* `queryable-state.proxy.ports`: the server port range of the queryable state proxy. This is useful to avoid port clashes if more than 1 task managers run on the same machine. The specified range can be: a port: "9123", a range of ports: "50100-50200", or a list of ranges and or points: "50100-50200,50300-50400,51234". The default port is 9069. -* `query.proxy.network-threads`: number of network (event loop) threads receiving incoming requests for the client proxy (0 => #slots) -* `query.proxy.query-threads`: number of threads handling/serving incoming requests for the client proxy (0 => #slots). +* `queryable-state.proxy.network-threads`: number of network (event loop) threads receiving incoming requests for the client proxy (0 => #slots) +* `queryable-state.proxy.query-threads`: number of threads handling/serving incoming requests for the client proxy (0 => #slots). ## Limitations diff --git a/flink-container/kubernetes/job-cluster-job.yaml.template b/flink-container/kubernetes/job-cluster-job.yaml.template index 0bbb1a1aa3a0a2..86db3c7794ce5e 100644 --- a/flink-container/kubernetes/job-cluster-job.yaml.template +++ b/flink-container/kubernetes/job-cluster-job.yaml.template @@ -32,7 +32,7 @@ spec: - name: flink-job-cluster image: ${FLINK_IMAGE_NAME} args: ["job-cluster", "--job-classname", "${FLINK_JOB}", "-Djobmanager.rpc.address=flink-job-cluster", - "-Dparallelism.default=${FLINK_JOB_PARALLELISM}", "-Dblob.server.port=6124", "-Dquery.server.ports=6125"] + "-Dparallelism.default=${FLINK_JOB_PARALLELISM}", "-Dblob.server.port=6124", "-Dqueryable-state.server.ports=6125"] ports: - containerPort: 6123 name: rpc diff --git a/flink-core/src/main/java/org/apache/flink/configuration/QueryableStateOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/QueryableStateOptions.java index 20c6b53a2090e0..253d4f391b1f12 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/QueryableStateOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/QueryableStateOptions.java @@ -51,23 +51,26 @@ public class QueryableStateOptions { *

The default port is 9069. */ public static final ConfigOption PROXY_PORT_RANGE = - key("query.proxy.ports") + key("queryable-state.proxy.ports") .defaultValue("9069") .withDescription("The port range of the queryable state proxy. The specified range can be a single " + "port: \"9123\", a range of ports: \"50100-50200\", " + - "or a list of ranges and ports: \"50100-50200,50300-50400,51234\"."); + "or a list of ranges and ports: \"50100-50200,50300-50400,51234\".") + .withDeprecatedKeys("query.proxy.ports"); /** Number of network (event loop) threads for the client proxy (0 => #slots). */ public static final ConfigOption PROXY_NETWORK_THREADS = - key("query.proxy.network-threads") + key("queryable-state.proxy.network-threads") .defaultValue(0) - .withDescription("Number of network (Netty's event loop) Threads for queryable state proxy."); + .withDescription("Number of network (Netty's event loop) Threads for queryable state proxy.") + .withDeprecatedKeys("query.proxy.network-threads"); /** Number of async query threads for the client proxy (0 => #slots). */ public static final ConfigOption PROXY_ASYNC_QUERY_THREADS = - key("query.proxy.query-threads") + key("queryable-state.proxy.query-threads") .defaultValue(0) - .withDescription("Number of query Threads for queryable state proxy. Uses the number of slots if set to 0."); + .withDescription("Number of query Threads for queryable state proxy. Uses the number of slots if set to 0.") + .withDeprecatedKeys("query.proxy.query-threads"); /** * The config parameter defining the server port range of the queryable state server. @@ -88,23 +91,26 @@ public class QueryableStateOptions { *

The default port is 9067. */ public static final ConfigOption SERVER_PORT_RANGE = - key("query.server.ports") + key("queryable-state.server.ports") .defaultValue("9067") .withDescription("The port range of the queryable state server. The specified range can be a single " + "port: \"9123\", a range of ports: \"50100-50200\", " + - "or a list of ranges and ports: \"50100-50200,50300-50400,51234\"."); + "or a list of ranges and ports: \"50100-50200,50300-50400,51234\".") + .withDeprecatedKeys("query.server.ports"); /** Number of network (event loop) threads for the KvState server (0 => #slots). */ public static final ConfigOption SERVER_NETWORK_THREADS = - key("query.server.network-threads") + key("queryable-state.server.network-threads") .defaultValue(0) - .withDescription("Number of network (Netty's event loop) Threads for queryable state server."); + .withDescription("Number of network (Netty's event loop) Threads for queryable state server.") + .withDeprecatedKeys("query.server.network-threads"); /** Number of async query threads for the KvStateServerHandler (0 => #slots). */ public static final ConfigOption SERVER_ASYNC_QUERY_THREADS = - key("query.server.query-threads") + key("queryable-state.server.query-threads") .defaultValue(0) - .withDescription("Number of query Threads for queryable state server. Uses the number of slots if set to 0."); + .withDescription("Number of query Threads for queryable state server. Uses the number of slots if set to 0.") + .withDeprecatedKeys("query.server.query-threads"); /** Option whether the queryable state proxy and server should be enabled where possible and configurable. * @@ -122,9 +128,10 @@ public class QueryableStateOptions { /** Number of network (event loop) threads for the KvState client (0 => Use number of available cores). */ public static final ConfigOption CLIENT_NETWORK_THREADS = - key("query.client.network-threads") + key("queryable-state.client.network-threads") .defaultValue(0) - .withDescription("Number of network (Netty's event loop) Threads for queryable state client."); + .withDescription("Number of network (Netty's event loop) Threads for queryable state client.") + .withDeprecatedKeys("query.client.network-threads"); // ------------------------------------------------------------------------ diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh index 645cf2a2f1e746..cb2f5512876ac4 100644 --- a/flink-end-to-end-tests/test-scripts/common.sh +++ b/flink-end-to-end-tests/test-scripts/common.sh @@ -152,8 +152,8 @@ function create_ha_config() { rest.port: 8081 - query.server.ports: 9000-9009 - query.proxy.ports: 9010-9019 + queryable-state.server.ports: 9000-9009 + queryable-state.proxy.ports: 9010-9019 EOL } diff --git a/flink-end-to-end-tests/test-scripts/container-scripts/job-cluster-job.yaml.template b/flink-end-to-end-tests/test-scripts/container-scripts/job-cluster-job.yaml.template index 3785e6f11d0a44..58231913546f66 100644 --- a/flink-end-to-end-tests/test-scripts/container-scripts/job-cluster-job.yaml.template +++ b/flink-end-to-end-tests/test-scripts/container-scripts/job-cluster-job.yaml.template @@ -33,7 +33,7 @@ spec: image: ${FLINK_IMAGE_NAME} imagePullPolicy: Never args: ["job-cluster", "--job-classname", "${FLINK_JOB}", "-Djobmanager.rpc.address=flink-job-cluster", - "-Dparallelism.default=${FLINK_JOB_PARALLELISM}", "-Dblob.server.port=6124", "-Dquery.server.ports=6125", + "-Dparallelism.default=${FLINK_JOB_PARALLELISM}", "-Dblob.server.port=6124", "-Dqueryable-state.server.ports=6125", ${FLINK_JOB_ARGUMENTS}] ports: - containerPort: 6123 From 291373a0b61f4651ddbbaf0dc3f9dd9fd68db611 Mon Sep 17 00:00:00 2001 From: Artsem Semianenka Date: Thu, 10 Jan 2019 17:06:00 +0100 Subject: [PATCH 145/147] [FLINK-11302][fs-connector] Correctly parse tmp dirs in FlinkS3FileSystem. This closes #7458. --- .../s3/common/AbstractS3FileSystemFactory.java | 7 +++++-- .../fs/s3/common/S3EntropyFsFactoryTest.java | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/AbstractS3FileSystemFactory.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/AbstractS3FileSystemFactory.java index 6ccdeae7df8356..ff575be6f55c81 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/AbstractS3FileSystemFactory.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/AbstractS3FileSystemFactory.java @@ -21,11 +21,12 @@ import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.CoreOptions; +import org.apache.flink.configuration.ConfigurationUtils; import org.apache.flink.configuration.IllegalConfigurationException; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.FileSystemFactory; import org.apache.flink.fs.s3.common.writer.S3AccessHelper; +import org.apache.flink.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -138,7 +139,9 @@ public FileSystem create(URI fsUri) throws IOException { } } - final String localTmpDirectory = flinkConfig.getString(CoreOptions.TMP_DIRS); + final String[] localTmpDirectories = ConfigurationUtils.parseTempDirectories(flinkConfig); + Preconditions.checkArgument(localTmpDirectories.length > 0); + final String localTmpDirectory = localTmpDirectories[0]; final long s3minPartSize = flinkConfig.getLong(PART_UPLOAD_MIN_SIZE); final int maxConcurrentUploads = flinkConfig.getInteger(MAX_CONCURRENT_UPLOADS); final S3AccessHelper s3AccessHelper = getS3AccessHelper(fs); diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/S3EntropyFsFactoryTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/S3EntropyFsFactoryTest.java index 5b15652db62f9c..943de1d8897274 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/S3EntropyFsFactoryTest.java +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/S3EntropyFsFactoryTest.java @@ -52,6 +52,24 @@ public void testEntropyInjectionConfig() throws Exception { assertEquals(7, fs.generateEntropy().length()); } + /** + * Test validates that the produced by AbstractS3FileSystemFactory object will contains + * only first path from multiple paths in config. + */ + @Test + public void testMultipleTempDirsConfig() throws Exception { + final Configuration conf = new Configuration(); + String dir1 = "/tmp/dir1"; + String dir2 = "/tmp/dir2"; + conf.setString("io.tmp.dirs", dir1 + "," + dir2); + + TestFsFactory factory = new TestFsFactory(); + factory.configure(conf); + + FlinkS3FileSystem fs = (FlinkS3FileSystem) factory.create(new URI("s3://test")); + assertEquals(fs.getLocalTmpDir(), dir1); + } + // ------------------------------------------------------------------------ private static final class TestFsFactory extends AbstractS3FileSystemFactory { From 86e06bc9914768f70556b0796fb5eaee01aef5fd Mon Sep 17 00:00:00 2001 From: leesf <490081539@qq.com> Date: Wed, 16 Jan 2019 22:15:48 +0800 Subject: [PATCH 146/147] [hotfix][docs] Remove stray character --- docs/ops/state/large_state_tuning.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ops/state/large_state_tuning.md b/docs/ops/state/large_state_tuning.md index 4bf890d622aa3e..81d360660754cd 100644 --- a/docs/ops/state/large_state_tuning.md +++ b/docs/ops/state/large_state_tuning.md @@ -132,7 +132,7 @@ Incremental checkpoints can dramatically reduce the checkpointing time in compar recovery time. The core idea is that incremental checkpoints only record all changes to the previous completed checkpoint, instead of producing a full, self-contained backup of the state backend. Like this, incremental checkpoints build upon previous checkpoints. Flink leverages RocksDB's internal backup mechanism in a way that is self-consolidating over time. As a result, the incremental checkpoint history in Flink -does not grow indefinitely, and old checkpoints are eventually subsumed and pruned automatically. ` +does not grow indefinitely, and old checkpoints are eventually subsumed and pruned automatically. While we strongly encourage the use of incremental checkpoints for large state, please note that this is a new feature and currently not enabled by default. To enable this feature, users can instantiate a `RocksDBStateBackend` with the corresponding boolean flag in the constructor set to `true`, e.g.: From e683ba4a1502f23f2445da982deab74987ab34b4 Mon Sep 17 00:00:00 2001 From: lamber-ken <2217232293@qq.com> Date: Wed, 16 Jan 2019 23:41:08 +0800 Subject: [PATCH 147/147] [hotfix][runtime] remove RejectedExecutionException code in executeAsyncCallRunnable method --- .../org/apache/flink/runtime/taskmanager/Task.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java index 9abf92f76fdfa4..0e946c6dfde82f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java @@ -1319,16 +1319,7 @@ private void executeAsyncCallRunnable(Runnable runnable, String callName) { LOG.debug("Invoking async call {} on task {}", callName, taskNameWithSubtask); - try { - executor.submit(runnable); - } - catch (RejectedExecutionException e) { - // may be that we are concurrently finished or canceled. - // if not, report that something is fishy - if (executionState == ExecutionState.RUNNING) { - throw new RuntimeException("Async call was rejected, even though the task is running.", e); - } - } + executor.submit(runnable); } }