BigDataArtisans · lamberken · Jan 2, 2019 · Jan 2, 2019 · Dec 19, 2018 · Jan 2, 2019
diff --git a/NOTICE-binary b/NOTICE-binary
@@ -9,12 +9,8 @@ Copyright 2014-2018 The Apache Software Foundation
 This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
 
 - com.chuusai:shapeless_2.11:2.3.2
-- com.google.guava:guava:14.0.1
-- com.twitter:hbc-core:2.2.0
-- com.twitter:joauth:6.0.2
 - log4j:log4j:1.2.17
 - net.sf.opencsv:opencsv:2.3
-- org.apache.kafka:kafka-clients:jar:0.10.2.1
 - org.scalanlp:breeze_2.11:0.13
 - org.scalanlp:breeze-macros_2.11:0.13
 - org.typelevel:macro-compat_2.11:1.1.1
@@ -64,7 +60,7 @@ This project bundles the following dependencies under the Apache Software Licens
 - commons-collections:commons-collections:3.2.2
 - commons-io:commons-io:2.4
 - org.apache.camel:camel-core:2.17.7
-- org.apache.commons:commons-compress:1.4.1
+- org.apache.commons:commons-compress:1.18
 - org.apache.commons:commons-lang3:3.3.2
 - org.apache.commons:commons-math3:3.5
 - org.javassist:javassist:3.19.0-GA
@@ -344,7 +340,7 @@ This project bundles the following dependencies under the Apache Software Licens
 - org.apache.htrace:htrace-core4:4.0.1-incubating
 - org.apache.httpcomponents:httpclient:4.5.3
 - org.apache.httpcomponents:httpcore:4.4.6
-- org.apache.commons:commons-compress:1.4.1
+- org.apache.commons:commons-compress:1.18
 - org.apache.commons:commons-math3:3.5
 - commons-beanutils:commons-beanutils:1.8.3
 - commons-cli:commons-cli:1.3.1
@@ -2412,7 +2408,7 @@ This project bundles the following dependencies under the Apache Software Licens
 - commons-logging:commons-logging:1.1.3
 - commons-net:commons-net:3.1
 - org.apache.avro:avro:1.8.2
-- org.apache.commons:commons-compress:1.4.1
+- org.apache.commons:commons-compress:1.18
 - org.apache.commons:commons-math3:3.5
 - org.apache.zookeeper:zookeeper:3.4.10
 - org.codehaus.jackson:jackson-core-asl:1.9.13
@@ -5797,7 +5793,7 @@ The Apache Software Foundation (http://www.apache.org/).
 This project bundles the following dependencies under the Python Software Foundation License. (https://opensource.org/licenses/PythonSoftFoundation.php)
 See bundled license files for details.
 
-- org.python:jython-standalone:2.7.0
+- org.python:jython-standalone:2.7.1
 
 flink-metrics-graphite
 Copyright 2014-2018 The Apache Software Foundation
@@ -5806,3 +5802,62 @@ This project bundles the following dependencies under the Apache Software Licens
 
 - io.dropwizard.metrics:metrics-core:3.1.5
 - io.dropwizard.metrics:metrics-graphite:3.1.5
+
+// NOTICE file corresponding to the section 4d of The Apache License,
+// Version 2.0, in this case for Apache Flink
+// ------------------------------------------------------------------
+
+Apache Flink
+Copyright 2006-2019 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+flink-examples-streaming-state-machine
+Copyright 2014-2018 The Apache Software Foundation
+
+This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt)
+
+- org.apache.kafka:kafka-clients:0.10.2.1
+
+
+flink-connector-kafka-0.10
+Copyright 2014-2019 The Apache Software Foundation
+
+flink-connector-kafka-0.9
+Copyright 2014-2019 The Apache Software Foundation
+
+flink-connector-kafka-base
+Copyright 2014-2019 The Apache Software Foundation
+
+// ------------------------------------------------------------------
+// NOTICE file corresponding to the section 4d of The Apache License,
+// Version 2.0, in this case for Apache Flink
+// ------------------------------------------------------------------
+
+Apache Flink
+Copyright 2006-2019 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+
+flink-examples-streaming-twitter
+Copyright 2014-2019 The Apache Software Foundation
+
+Apache HttpCore
+Copyright 2005-2017 The Apache Software Foundation
+
+flink-connector-twitter
+Copyright 2014-2018 The Apache Software Foundation
+
+This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
+
+- com.google.guava:guava:14.0.1
+- com.twitter:hbc-core:2.2.0
+- com.twitter:joauth:6.0.2
+- org.apache.httpcomponents:httpclient:4.5.3
+- org.apache.httpcomponents:httpcore:4.4.6
+
+Apache HttpClient
+Copyright 1999-2017 The Apache Software Foundation
diff --git a/README.md b/README.md
@@ -27,7 +27,7 @@ Learn more about Flink at [http://flink.apache.org/](http://flink.apache.org/)
 
 * Custom memory management for efficient and robust switching between in-memory and out-of-core data processing algorithms
 
-* Compatibility layers for Apache Hadoop MapReduce and Apache Storm
+* Compatibility layers for Apache Hadoop MapReduce
 
 * Integration with YARN, HDFS, HBase, and other components of the Apache Hadoop ecosystem
 

diff --git a/docs/_config.yml b/docs/_config.yml
@@ -34,6 +34,8 @@ version_title: "1.8-SNAPSHOT"
 # Branch on Github for this version
 github_branch: "master"
 
+# Plain Scala version is needed for e.g. the Gradle quickstart.
+scala_version: "2.11"
 # This suffix is appended to the Scala-dependent Maven artifact names
 scala_version_suffix: "_2.11"
 

diff --git a/docs/_includes/generated/common_section.html b/docs/_includes/generated/common_section.html
@@ -20,7 +20,7 @@
         <tr>
             <td><h5>parallelism.default</h5></td>
             <td style="word-wrap: break-word;">1</td>
-            <td></td>
+            <td>Default parallelism for jobs.</td>
         </tr>
         <tr>
             <td><h5>taskmanager.numberOfTaskSlots</h5></td>

diff --git a/docs/_includes/generated/core_configuration.html b/docs/_includes/generated/core_configuration.html
@@ -25,12 +25,12 @@
         <tr>
             <td><h5>io.tmp.dirs</h5></td>
             <td style="word-wrap: break-word;">'LOCAL_DIRS' on Yarn. '_FLINK_TMP_DIR' on Mesos. System.getProperty("java.io.tmpdir") in standalone.</td>
-            <td></td>
+            <td>Directories for temporary files, separated by",", "|", or the system's java.io.File.pathSeparator.</td>
         </tr>
         <tr>
             <td><h5>parallelism.default</h5></td>
             <td style="word-wrap: break-word;">1</td>
-            <td></td>
+            <td>Default parallelism for jobs.</td>
         </tr>
     </tbody>
 </table>
diff --git a/docs/_includes/generated/history_server_configuration.html b/docs/_includes/generated/history_server_configuration.html
@@ -30,7 +30,7 @@
         <tr>
             <td><h5>historyserver.web.refresh-interval</h5></td>
             <td style="word-wrap: break-word;">10000</td>
-            <td></td>
+            <td>The refresh interval for the HistoryServer web-frontend in milliseconds.</td>
         </tr>
         <tr>
             <td><h5>historyserver.web.ssl.enabled</h5></td>

diff --git a/docs/_includes/generated/job_manager_configuration.html b/docs/_includes/generated/job_manager_configuration.html
@@ -10,7 +10,7 @@
         <tr>
             <td><h5>jobmanager.archive.fs.dir</h5></td>
             <td style="word-wrap: break-word;">(none)</td>
-            <td></td>
+            <td>Dictionary for JobManager to store the archives of completed jobs.</td>
         </tr>
         <tr>
             <td><h5>jobmanager.execution.attempts-history-size</h5></td>

diff --git a/docs/_includes/generated/metric_configuration.html b/docs/_includes/generated/metric_configuration.html
@@ -50,12 +50,12 @@
         <tr>
             <td><h5>metrics.reporters</h5></td>
             <td style="word-wrap: break-word;">(none)</td>
-            <td></td>
+            <td>An optional list of reporter names. If configured, only reporters whose name matches any of the names in the list will be started. Otherwise, all reporters that could be found in the configuration will be started.</td>
         </tr>
         <tr>
             <td><h5>metrics.scope.delimiter</h5></td>
             <td style="word-wrap: break-word;">"."</td>
-            <td></td>
+            <td>Delimiter used to assemble the metric identifier.</td>
         </tr>
         <tr>
             <td><h5>metrics.scope.jm</h5></td>
@@ -90,12 +90,12 @@
         <tr>
             <td><h5>metrics.system-resource</h5></td>
             <td style="word-wrap: break-word;">false</td>
-            <td></td>
+            <td>Flag indicating whether Flink should report system resource metrics such as machine's CPU, memory or network usage.</td>
         </tr>
         <tr>
             <td><h5>metrics.system-resource-probing-interval</h5></td>
             <td style="word-wrap: break-word;">5000</td>
-            <td></td>
+            <td>Interval between probing of system resource metrics specified in milliseconds. Has an effect only when 'metrics.system-resource' is enabled.</td>
         </tr>
     </tbody>
 </table>
diff --git a/docs/_includes/generated/prometheus_push_gateway_reporter_configuration.html b/docs/_includes/generated/prometheus_push_gateway_reporter_configuration.html
@@ -12,6 +12,11 @@
             <td style="word-wrap: break-word;">true</td>
             <td>Specifies whether to delete metrics from the PushGateway on shutdown.</td>
         </tr>
+        <tr>
+            <td><h5>filterLabelValueCharacters</h5></td>
+            <td style="word-wrap: break-word;">true</td>
+            <td>Specifies whether to filter label value characters. If enabled, all characters not matching [a-zA-Z0-9:_] will be removed, otherwise no characters will be removed. Before disabling this option please ensure that your label values meet the <a href="https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels">Prometheus requirements</a>.</td>
+        </tr>
         <tr>
             <td><h5>host</h5></td>
             <td style="word-wrap: break-word;">(none)</td>

diff --git a/docs/_includes/generated/queryable_state_configuration.html b/docs/_includes/generated/queryable_state_configuration.html
@@ -8,37 +8,42 @@
     </thead>
     <tbody>
         <tr>
-            <td><h5>query.client.network-threads</h5></td>
+            <td><h5>queryable-state.client.network-threads</h5></td>
             <td style="word-wrap: break-word;">0</td>
             <td>Number of network (Netty's event loop) Threads for queryable state client.</td>
         </tr>
         <tr>
-            <td><h5>query.proxy.network-threads</h5></td>
+            <td><h5>queryable-state.enable</h5></td>
+            <td style="word-wrap: break-word;">false</td>
+            <td>Option whether the queryable state proxy and server should be enabled where possible and configurable.</td>
+        </tr>
+        <tr>
+            <td><h5>queryable-state.proxy.network-threads</h5></td>
             <td style="word-wrap: break-word;">0</td>
             <td>Number of network (Netty's event loop) Threads for queryable state proxy.</td>
         </tr>
         <tr>
-            <td><h5>query.proxy.ports</h5></td>
+            <td><h5>queryable-state.proxy.ports</h5></td>
             <td style="word-wrap: break-word;">"9069"</td>
             <td>The port range of the queryable state proxy. The specified range can be a single port: "9123", a range of ports: "50100-50200", or a list of ranges and ports: "50100-50200,50300-50400,51234".</td>
         </tr>
         <tr>
-            <td><h5>query.proxy.query-threads</h5></td>
+            <td><h5>queryable-state.proxy.query-threads</h5></td>
             <td style="word-wrap: break-word;">0</td>
             <td>Number of query Threads for queryable state proxy. Uses the number of slots if set to 0.</td>
         </tr>
         <tr>
-            <td><h5>query.server.network-threads</h5></td>
+            <td><h5>queryable-state.server.network-threads</h5></td>
             <td style="word-wrap: break-word;">0</td>
             <td>Number of network (Netty's event loop) Threads for queryable state server.</td>
         </tr>
         <tr>
-            <td><h5>query.server.ports</h5></td>
+            <td><h5>queryable-state.server.ports</h5></td>
             <td style="word-wrap: break-word;">"9067"</td>
             <td>The port range of the queryable state server. The specified range can be a single port: "9123", a range of ports: "50100-50200", or a list of ranges and ports: "50100-50200,50300-50400,51234".</td>
         </tr>
         <tr>
-            <td><h5>query.server.query-threads</h5></td>
+            <td><h5>queryable-state.server.query-threads</h5></td>
             <td style="word-wrap: break-word;">0</td>
             <td>Number of query Threads for queryable state server. Uses the number of slots if set to 0.</td>
         </tr>

diff --git a/docs/_includes/generated/resource_manager_configuration.html b/docs/_includes/generated/resource_manager_configuration.html
@@ -20,7 +20,7 @@
         <tr>
             <td><h5>local.number-resourcemanager</h5></td>
             <td style="word-wrap: break-word;">1</td>
-            <td></td>
+            <td>The number of resource managers start.</td>
         </tr>
         <tr>
             <td><h5>resourcemanager.job.timeout</h5></td>

diff --git a/docs/_includes/generated/rest_v1_dispatcher.html b/docs/_includes/generated/rest_v1_dispatcher.html
@@ -3871,3 +3871,5 @@
     </tr>
   </tbody>
 </table>
+
+{% top %}
diff --git a/docs/_includes/generated/task_manager_configuration.html b/docs/_includes/generated/task_manager_configuration.html
@@ -20,7 +20,7 @@
         <tr>
             <td><h5>task.cancellation.timers.timeout</h5></td>
             <td style="word-wrap: break-word;">7500</td>
-            <td></td>
+            <td>Time we wait for the timers in milliseconds to finish all pending timer threads when the stream task is cancelled.</td>
         </tr>
         <tr>
             <td><h5>task.checkpoint.alignment.max-size</h5></td>

diff --git a/docs/_includes/generated/web_configuration.html b/docs/_includes/generated/web_configuration.html
@@ -10,77 +10,77 @@
         <tr>
             <td><h5>web.access-control-allow-origin</h5></td>
             <td style="word-wrap: break-word;">"*"</td>
-            <td></td>
+            <td>Access-Control-Allow-Origin header for all responses from the web-frontend.</td>
         </tr>
         <tr>
             <td><h5>web.address</h5></td>
             <td style="word-wrap: break-word;">(none)</td>
-            <td></td>
+            <td>Address for runtime monitor web-frontend server.</td>
         </tr>
         <tr>
             <td><h5>web.backpressure.cleanup-interval</h5></td>
             <td style="word-wrap: break-word;">600000</td>
-            <td></td>
+            <td>Time, in milliseconds, after which cached stats are cleaned up if not accessed.</td>
         </tr>
         <tr>
             <td><h5>web.backpressure.delay-between-samples</h5></td>
             <td style="word-wrap: break-word;">50</td>
-            <td></td>
+            <td>Delay between stack trace samples to determine back pressure in milliseconds.</td>
         </tr>
         <tr>
             <td><h5>web.backpressure.num-samples</h5></td>
             <td style="word-wrap: break-word;">100</td>
-            <td></td>
+            <td>Number of stack trace samples to take to determine back pressure.</td>
         </tr>
         <tr>
             <td><h5>web.backpressure.refresh-interval</h5></td>
             <td style="word-wrap: break-word;">60000</td>
-            <td></td>
+            <td>Time, in milliseconds, after which available stats are deprecated and need to be refreshed (by resampling).</td>
         </tr>
         <tr>
             <td><h5>web.checkpoints.history</h5></td>
             <td style="word-wrap: break-word;">10</td>
-            <td></td>
+            <td>Number of checkpoints to remember for recent history.</td>
         </tr>
         <tr>
             <td><h5>web.history</h5></td>
             <td style="word-wrap: break-word;">5</td>
-            <td></td>
+            <td>Number of archived jobs for the JobManager.</td>
         </tr>
         <tr>
             <td><h5>web.log.path</h5></td>
             <td style="word-wrap: break-word;">(none)</td>
-            <td></td>
+            <td>Path to the log file (may be in /log for standalone but under log directory when using YARN).</td>
         </tr>
         <tr>
             <td><h5>web.refresh-interval</h5></td>
             <td style="word-wrap: break-word;">3000</td>
-            <td></td>
+            <td>Refresh interval for the web-frontend in milliseconds.</td>
         </tr>
         <tr>
             <td><h5>web.ssl.enabled</h5></td>
             <td style="word-wrap: break-word;">true</td>
-            <td></td>
+            <td>Flag indicating whether to override SSL support for the JobManager Web UI.</td>
         </tr>
         <tr>
             <td><h5>web.submit.enable</h5></td>
             <td style="word-wrap: break-word;">true</td>
-            <td></td>
+            <td>Flag indicating whether jobs can be uploaded and run from the web-frontend.</td>
         </tr>
         <tr>
             <td><h5>web.timeout</h5></td>
             <td style="word-wrap: break-word;">10000</td>
-            <td></td>
+            <td>Timeout for asynchronous operations by the WebRuntimeMonitor in milliseconds.</td>
         </tr>
         <tr>
             <td><h5>web.tmpdir</h5></td>
             <td style="word-wrap: break-word;">System.getProperty("java.io.tmpdir")</td>
-            <td></td>
+            <td>Flink web directory which is used by the webmonitor.</td>
         </tr>
         <tr>
             <td><h5>web.upload.dir</h5></td>
             <td style="word-wrap: break-word;">(none)</td>
-            <td></td>
+            <td>Directory for uploading the job jars. If not specified a dynamic directory will be used under the directory specified by JOB_MANAGER_WEB_TMPDIR_KEY.</td>
         </tr>
     </tbody>
 </table>