From 195a82523a9cece67cb839b306fee5756a6e1b77 Mon Sep 17 00:00:00 2001 From: shangeyao Date: Tue, 14 Jul 2026 21:01:41 +0800 Subject: [PATCH] [Flink] Remove StreamPark dev framework and connector wrappers Drop FlinkStreaming/FlinkTable user traits, all streampark-flink-connector modules, Spark dev traits, and streampark-spark-connector modules. Refactor Flink/Spark SqlClient to use shims-base and SparkSqlExecutor directly. Closes #4431. Co-authored-by: Cursor --- streampark-flink/pom.xml | 10 - .../streampark-flink-connector/pom.xml | 66 ---- .../streampark-flink-connector-base/pom.xml | 104 ------ .../connector/function/RunningFunction.java | 31 -- .../connector/function/SQLQueryFunction.java | 31 -- .../connector/function/SQLResultFunction.java | 32 -- .../function/StreamEnvConfigFunction.java | 35 -- .../function/TableEnvConfigFunction.java | 35 -- .../connector/function/TransformFunction.java | 29 -- .../flink/connector/conf/ThresholdConf.scala | 58 --- .../conf/ThresholdConfigOption.scala | 89 ----- .../connector/failover/FailoverChecker.scala | 56 --- .../connector/failover/FailoverWriter.scala | 147 -------- .../flink/connector/failover/SinkBuffer.scala | 75 ---- .../connector/failover/SinkRequest.scala | 83 ----- .../flink/connector/failover/SinkWriter.scala | 24 -- .../flink/connector/sink/Sink.scala | 41 --- .../connector/failover/SinkRequestTest.scala | 48 --- .../pom.xml | 98 ----- .../conf/ClickHouseHttpConfig.scala | 75 ---- .../conf/ClickHouseJdbcConfig.scala | 47 --- .../conf/ClickHouseSinkConfigOption.scala | 130 ------- .../AsyncClickHouseSinkFunction.scala | 114 ------ .../internal/ClickHouseSinkFunction.scala | 162 --------- .../internal/ClickHouseSinkWriter.scala | 91 ----- .../internal/ClickHouseWriterTask.scala | 143 -------- .../sink/AsyncClickHouseOutputFormat.scala | 57 --- .../sink/ClickHouseOutputFormat.scala | 57 --- .../clickhouse/sink/ClickHouseSink.scala | 166 --------- .../util/ClickhouseConvertUtils.scala | 36 -- .../streampark-flink-connector-doris/pom.xml | 86 ----- .../doris/bean/DorisSinkBufferEntry.java | 130 ------- .../doris/bean/DorisSinkRowDataWithMeta.java | 57 --- .../doris/bean/LoadStatusFailedException.java | 48 --- .../connector/doris/bean/RespContent.java | 275 -------------- .../doris/internal/DorisSinkFunction.java | 138 ------- .../doris/internal/DorisSinkWriter.java | 336 ------------------ .../doris/internal/DorisStreamLoader.java | 299 ---------------- .../flink/connector/doris/sink/DorisSink.java | 55 --- .../doris/util/DorisDelimiterParser.java | 69 ---- .../connector/doris/conf/DorisConfig.scala | 89 ----- .../doris/conf/DorisSinkConfigOption.scala | 148 -------- .../pom.xml | 51 --- .../pom.xml | 94 ----- .../elasticsearch5/conf/ESConfig.scala | 31 -- .../conf/ESSinkConfigOption.scala | 75 ---- .../internal/ESSinkFunction.scala | 74 ---- .../elasticsearch5/sink/ES5Sink.scala | 137 ------- .../util/ElasticsearchUtils.scala | 41 --- .../pom.xml | 92 ----- .../bean/RestClientFactoryImpl.scala | 101 ------ .../elasticsearch6/conf/ES6Config.scala | 62 ---- .../conf/ES6SinkConfigOption.scala | 169 --------- .../internal/ESSinkFunction.scala | 73 ---- .../elasticsearch6/sink/ES6Sink.scala | 191 ---------- .../util/ElasticsearchUtils.scala | 40 --- .../pom.xml | 87 ----- .../bean/RestClientFactoryImpl.scala | 96 ----- .../elasticsearch7/conf/ES7Config.scala | 60 ---- .../conf/ES7SinkConfigOption.scala | 169 --------- .../internal/ESSinkFunction.scala | 69 ---- .../elasticsearch7/sink/ES7Sink.scala | 175 --------- .../util/ElasticsearchUtils.scala | 41 --- .../streampark-flink-connector-hbase/pom.xml | 105 ------ .../connector/hbase/bean/HBaseQuery.java | 72 ---- .../hbase/function/HBaseQueryFunction.java | 33 -- .../hbase/function/HBaseResultFunction.java | 33 -- .../hbase/source/HBaseJavaSource.java | 51 --- .../hbase/internal/HBaseSinkFunction.scala | 160 --------- .../hbase/internal/HBaseSourceFunction.scala | 182 ---------- .../hbase/request/HBaseRequest.scala | 125 ------- .../hbase/sink/HBaseOutputFormat.scala | 63 ---- .../connector/hbase/sink/HBaseSink.scala | 88 ----- .../connector/hbase/source/HBaseSource.scala | 59 --- .../streampark-flink-connector-http/pom.xml | 93 ----- .../http/conf/HttpConfigOption.scala | 41 --- .../http/function/HttpSinkFunction.scala | 94 ----- .../http/internal/HttpSinkWriter.scala | 80 ----- .../http/internal/HttpWriterTask.scala | 171 --------- .../flink/connector/http/sink/HttpSink.scala | 108 ------ .../streampark-flink-connector-influx/pom.xml | 89 ----- .../influx/function/InfluxFieldFunction.java | 25 -- .../influx/function/InfluxTagFunction.java | 25 -- .../connector/influx/bean/InfluxEntity.scala | 72 ---- .../influx/function/InfluxFunction.scala | 78 ---- .../influx/sink/InfluxOutputFormat.scala | 45 --- .../connector/influx/sink/InfluxSink.scala | 62 ---- .../streampark-flink-connector-jdbc/pom.xml | 83 ----- .../connector/jdbc/sink/JdbcJavaSink.java | 64 ---- .../connector/jdbc/source/JdbcJavaSource.java | 65 ---- .../connector/jdbc/bean/Transaction.scala | 34 -- .../jdbc/internal/Jdbc2PCSinkFunction.scala | 139 -------- .../jdbc/internal/JdbcASyncFunction.scala | 86 ----- .../jdbc/internal/JdbcSinkFunction.scala | 122 ------- .../jdbc/internal/JdbcSourceFunction.scala | 147 -------- .../connector/jdbc/request/JdbcRequest.scala | 70 ---- .../flink/connector/jdbc/sink/JdbcSink.scala | 126 ------- .../connector/jdbc/source/JdbcSource.scala | 59 --- .../streampark-flink-connector-kafka/pom.xml | 88 ----- .../connector/kafka/sink/KafkaJavaSink.java | 142 -------- .../kafka/source/KafkaJavaSource.java | 91 ----- .../kafka/bean/KafkaEqualityPartitioner.scala | 78 ---- .../connector/kafka/bean/KafkaRecord.scala | 28 -- .../connector/kafka/sink/KafkaSink.scala | 124 ------- .../connector/kafka/source/KafkaSource.scala | 277 --------------- .../streampark-flink-connector-mongo/pom.xml | 89 ----- .../mongo/function/MongoQueryFunction.java | 35 -- .../mongo/function/MongoResultFunction.java | 33 -- .../mongo/source/MongoJavaSource.java | 54 --- .../mongo/internal/MongoSourceFunction.scala | 177 --------- .../connector/mongo/source/MongoSource.scala | 55 --- .../streampark-flink-connector-redis/pom.xml | 95 ----- .../connector/redis/bean/RedisContainer.scala | 150 -------- .../connector/redis/bean/RedisMapper.scala | 105 ------ .../redis/bean/RedisTransaction.scala | 34 -- .../connector/redis/conf/RedisConfig.scala | 48 --- .../redis/conf/RedisSinkConfigOption.scala | 89 ----- .../redis/internal/Redis2PCSinkFunction.scala | 84 ----- .../redis/internal/RedisSinkFunction.scala | 50 --- .../connector/redis/sink/RedisSink.scala | 197 ---------- .../streampark-flink-core/pom.xml | 111 ------ .../flink/core/scala/DataStreamExt.scala | 140 -------- .../flink/core/scala/FlinkStreamTable.scala | 70 ---- .../flink/core/scala/FlinkStreaming.scala | 97 ----- .../flink/core/scala/FlinkTable.scala | 57 --- .../streampark-flink-shims-test/pom.xml | 7 - .../streampark-flink-sqlclient/pom.xml | 6 - .../streampark/flink/cli/SqlClient.scala | 25 +- streampark-spark/pom.xml | 1 - .../streampark-spark-connector/pom.xml | 42 --- .../streampark-spark-connector-base/pom.xml | 41 --- .../spark/connector/sink/ShowSink.scala | 53 --- .../spark/connector/sink/Sink.scala | 81 ----- .../spark/connector/source/Source.scala | 51 --- .../streampark-spark-connector-kafka/pom.xml | 81 ----- .../kafka/offset/DefaultOffset.scala | 52 --- .../connector/kafka/offset/HBaseOffset.scala | 163 --------- .../connector/kafka/offset/KafkaClient.scala | 125 ------- .../connector/kafka/offset/MySQLOffset.scala | 138 ------- .../spark/connector/kafka/offset/Offset.scala | 154 -------- .../connector/kafka/offset/RedisOffset.scala | 89 ----- .../connector/kafka/sink/KafkaSink.scala | 49 --- .../connector/kafka/source/KafkaSource.scala | 106 ------ .../kafka/writer/DStreamKafkaWriter.scala | 50 --- .../kafka/writer/IterKafkaWriter.scala | 46 --- .../connector/kafka/writer/KafkaWriter.scala | 139 -------- .../kafka/writer/RDDKafkaWriter.scala | 49 --- .../kafka/writer/SimpleKafkaWriter.scala | 46 --- .../streampark/spark/core/SparkBatch.scala | 37 -- .../spark/core/SparkStreaming.scala | 74 ---- .../MultipleAvroOutputsFormat.scala | 36 -- .../core/serializable/MultipleOutputer.scala | 67 ---- .../serializable/MultipleOutputsFormat.scala | 167 --------- .../MultipleTextOutputsFormat.scala | 23 -- .../SerializationExtensions.scala | 118 ------ .../core/serializable/TextOutputFormat.scala | 31 -- .../SparkSqlExecutor.scala} | 211 +++++------ .../streampark/spark/cli/SqlClient.scala | 37 +- 158 files changed, 117 insertions(+), 13653 deletions(-) delete mode 100644 streampark-flink/streampark-flink-connector/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/RunningFunction.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/SQLQueryFunction.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/SQLResultFunction.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/StreamEnvConfigFunction.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/TableEnvConfigFunction.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/TransformFunction.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConf.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConfigOption.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverChecker.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverWriter.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkBuffer.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkRequest.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkWriter.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/sink/Sink.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/test/scala/org/apache/streampark/flink/connector/failover/SinkRequestTest.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseHttpConfig.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseJdbcConfig.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseSinkConfigOption.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/AsyncClickHouseSinkFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkWriter.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseWriterTask.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/AsyncClickHouseOutputFormat.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseOutputFormat.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseSink.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/util/ClickhouseConvertUtils.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/DorisSinkBufferEntry.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/DorisSinkRowDataWithMeta.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/LoadStatusFailedException.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/RespContent.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/internal/DorisSinkFunction.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/internal/DorisSinkWriter.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/internal/DorisStreamLoader.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/sink/DorisSink.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/util/DorisDelimiterParser.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisConfig.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisSinkConfigOption.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/conf/ESConfig.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/conf/ESSinkConfigOption.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/internal/ESSinkFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/sink/ES5Sink.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/util/ElasticsearchUtils.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/bean/RestClientFactoryImpl.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6Config.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6SinkConfigOption.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/internal/ESSinkFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/sink/ES6Sink.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/util/ElasticsearchUtils.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/bean/RestClientFactoryImpl.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/conf/ES7Config.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/conf/ES7SinkConfigOption.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/internal/ESSinkFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/sink/ES7Sink.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/util/ElasticsearchUtils.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/bean/HBaseQuery.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/function/HBaseQueryFunction.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/function/HBaseResultFunction.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/source/HBaseJavaSource.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/internal/HBaseSinkFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/internal/HBaseSourceFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/request/HBaseRequest.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/sink/HBaseOutputFormat.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/sink/HBaseSink.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/source/HBaseSource.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-http/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/conf/HttpConfigOption.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/function/HttpSinkFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/internal/HttpSinkWriter.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/internal/HttpWriterTask.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/sink/HttpSink.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/java/org/apache/streampark/flink/connector/influx/function/InfluxFieldFunction.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/java/org/apache/streampark/flink/connector/influx/function/InfluxTagFunction.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/bean/InfluxEntity.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/function/InfluxFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/sink/InfluxOutputFormat.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/sink/InfluxSink.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/java/org/apache/streampark/flink/connector/jdbc/sink/JdbcJavaSink.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/java/org/apache/streampark/flink/connector/jdbc/source/JdbcJavaSource.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/bean/Transaction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/Jdbc2PCSinkFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/JdbcASyncFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/JdbcSinkFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/JdbcSourceFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/request/JdbcRequest.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/sink/JdbcSink.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/source/JdbcSource.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/java/org/apache/streampark/flink/connector/kafka/sink/KafkaJavaSink.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/java/org/apache/streampark/flink/connector/kafka/source/KafkaJavaSource.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/bean/KafkaEqualityPartitioner.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/bean/KafkaRecord.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/sink/KafkaSink.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/source/KafkaSource.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/java/org/apache/streampark/flink/connector/mongo/function/MongoQueryFunction.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/java/org/apache/streampark/flink/connector/mongo/function/MongoResultFunction.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/java/org/apache/streampark/flink/connector/mongo/source/MongoJavaSource.java delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/scala/org/apache/streampark/flink/connector/mongo/internal/MongoSourceFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/scala/org/apache/streampark/flink/connector/mongo/source/MongoSource.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/pom.xml delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/bean/RedisContainer.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/bean/RedisMapper.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/bean/RedisTransaction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/conf/RedisConfig.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/conf/RedisSinkConfigOption.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/internal/Redis2PCSinkFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/internal/RedisSinkFunction.scala delete mode 100644 streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/sink/RedisSink.scala delete mode 100644 streampark-flink/streampark-flink-core/pom.xml delete mode 100644 streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/DataStreamExt.scala delete mode 100644 streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/FlinkStreamTable.scala delete mode 100644 streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/FlinkStreaming.scala delete mode 100644 streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/FlinkTable.scala delete mode 100644 streampark-spark/streampark-spark-connector/pom.xml delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-base/pom.xml delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-base/src/main/scala/org/apache/streampark/spark/connector/sink/ShowSink.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-base/src/main/scala/org/apache/streampark/spark/connector/sink/Sink.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-base/src/main/scala/org/apache/streampark/spark/connector/source/Source.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/pom.xml delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/DefaultOffset.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/HBaseOffset.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/KafkaClient.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/MySQLOffset.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/Offset.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/RedisOffset.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/sink/KafkaSink.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/source/KafkaSource.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/DStreamKafkaWriter.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/IterKafkaWriter.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/KafkaWriter.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/RDDKafkaWriter.scala delete mode 100644 streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/SimpleKafkaWriter.scala delete mode 100644 streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/SparkBatch.scala delete mode 100644 streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/SparkStreaming.scala delete mode 100644 streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleAvroOutputsFormat.scala delete mode 100644 streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleOutputer.scala delete mode 100644 streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleOutputsFormat.scala delete mode 100644 streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleTextOutputsFormat.scala delete mode 100644 streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/SerializationExtensions.scala delete mode 100644 streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/TextOutputFormat.scala rename streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/{Spark.scala => util/SparkSqlExecutor.scala} (51%) diff --git a/streampark-flink/pom.xml b/streampark-flink/pom.xml index 41f938c3cd..ce87ce94af 100644 --- a/streampark-flink/pom.xml +++ b/streampark-flink/pom.xml @@ -30,7 +30,6 @@ streampark-flink-shims - streampark-flink-core streampark-flink-sqlclient streampark-flink-udf streampark-flink-client @@ -82,13 +81,4 @@ - - - connector - - streampark-flink-connector - - - - diff --git a/streampark-flink/streampark-flink-connector/pom.xml b/streampark-flink/streampark-flink-connector/pom.xml deleted file mode 100644 index 3808a16f79..0000000000 --- a/streampark-flink/streampark-flink-connector/pom.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink - 2.2.0-SNAPSHOT - - - streampark-flink-connector - pom - StreamPark : Flink Connector - - - streampark-flink-connector-base - streampark-flink-connector-clickhouse - streampark-flink-connector-doris - streampark-flink-connector-hbase - streampark-flink-connector-http - streampark-flink-connector-elasticsearch - streampark-flink-connector-influx - streampark-flink-connector-jdbc - streampark-flink-connector-kafka - streampark-flink-connector-mongo - - - - 2.12.3 - 4.5.13 - - - - - - org.asynchttpclient - async-http-client - ${async.client.version} - - - - org.apache.httpcomponents - httpclient - ${httpclient4.version} - - - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/pom.xml deleted file mode 100644 index 0679820eac..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/pom.xml +++ /dev/null @@ -1,104 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink-connector - 2.2.0-SNAPSHOT - - - streampark-flink-connector-base_${scala.binary.version} - StreamPark : Flink Connector Base - - - - - org.apache.kafka - kafka-clients - 2.5.0 - - - - - org.apache.streampark - streampark-common_${scala.binary.version} - provided - - - - org.apache.streampark - streampark-flink-core_${scala.binary.version} - ${project.version} - - - - org.apache.streampark - streampark-flink-shims-base_${scala.binary.version} - ${project.version} - - - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/RunningFunction.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/RunningFunction.java deleted file mode 100644 index bfc0eb69f9..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/RunningFunction.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.function; - -import java.io.Serializable; - -@FunctionalInterface -public interface RunningFunction extends Serializable { - - /** - * Is it running... - * - * @return Boolean: isRunning - */ - Boolean running(); -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/SQLQueryFunction.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/SQLQueryFunction.java deleted file mode 100644 index fc3c86bec5..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/SQLQueryFunction.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.function; - -import java.io.Serializable; - -@FunctionalInterface -public interface SQLQueryFunction extends Serializable { - /** - * Get the SQL to query - * - * @param last: last one - * @return String: sql - */ - String query(T last) throws Exception; -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/SQLResultFunction.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/SQLResultFunction.java deleted file mode 100644 index 552bd8010e..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/SQLResultFunction.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.function; - -import java.io.Serializable; -import java.util.Map; - -@FunctionalInterface -public interface SQLResultFunction extends Serializable { - /** - * The result of the search is returned as a Map, and the user can convert it into an object. - * - * @param map: sqlQuery result - * @return Iterable: Iterable - */ - Iterable result(Iterable> map); -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/StreamEnvConfigFunction.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/StreamEnvConfigFunction.java deleted file mode 100644 index 30ccfd1e31..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/StreamEnvConfigFunction.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.function; - -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - -import java.io.Serializable; - -@FunctionalInterface -public interface StreamEnvConfigFunction extends Serializable { - /** - * When used to initialize StreamExecutionEnvironment, it is used to implement this function and - * customize the parameters to be set... - * - * @param environment: StreamExecutionEnvironment instance - * @param parameterTool: ParameterTool - */ - void configuration(StreamExecutionEnvironment environment, ParameterTool parameterTool); -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/TableEnvConfigFunction.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/TableEnvConfigFunction.java deleted file mode 100644 index ab2745e6d7..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/TableEnvConfigFunction.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.function; - -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.table.api.TableConfig; - -import java.io.Serializable; - -@FunctionalInterface -public interface TableEnvConfigFunction extends Serializable { - /** - * When used to initialize TableEnvironment, it is used to implement this function and customize - * the parameters to be set... - * - * @param tableConfig: flink tableConfig - * @param parameterTool: parameterTool - */ - void configuration(TableConfig tableConfig, ParameterTool parameterTool); -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/TransformFunction.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/TransformFunction.java deleted file mode 100644 index c3843bf821..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/java/org/apache/streampark/flink/connector/function/TransformFunction.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.function; - -import java.io.Serializable; - -@FunctionalInterface -public interface TransformFunction extends Serializable { - /** - * @param bean: bean - * @return String: - */ - R transform(T bean); -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConf.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConf.scala deleted file mode 100644 index bf0a0bd53d..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConf.scala +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.conf - -import org.apache.streampark.common.util.ConfigUtils -import org.apache.streampark.flink.connector.conf.FailoverStorageType.{Console, FailoverStorageType, Kafka, MySQL, NONE} - -import java.util.Properties - -import org.apache.streampark.common.util.Implicits._ - -case class ThresholdConf(prefixStr: String, parameters: Properties) { - - private val option: ThresholdConfigOption = ThresholdConfigOption(prefixStr, parameters) - - val bufferSize: Int = option.bufferSize.get() - val queueCapacity: Int = option.queueCapacity.get() - val delayTime: Long = option.delayTime.get() - val timeout: Int = option.timeout.get() - val numWriters: Int = option.numWriters.get() - val maxRetries: Int = option.maxRetries.get() - val storageType: FailoverStorageType = option.storageType.get() - val failoverTable: String = option.failoverTable.get() - - def getFailoverConfig: Properties = { - storageType match { - case Console | NONE => null - case Kafka => ConfigUtils.getConf(parameters.toMap, "failover.kafka.") - case MySQL => ConfigUtils.getConf(parameters.toMap, "failover.mysql.") - case _ => - throw new IllegalArgumentException( - s"[StreamPark] usage error! failover.storage must not be null! ") - } - } -} - -object FailoverStorageType extends Enumeration { - type FailoverStorageType = Value - val Console, MySQL, Kafka, NONE = Value - - def get(key: String): Value = values.find(_.toString.equalsIgnoreCase(key)).get - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConfigOption.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConfigOption.scala deleted file mode 100644 index b2924b1e76..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/conf/ThresholdConfigOption.scala +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.conf - -import org.apache.streampark.common.conf.ConfigOption -import org.apache.streampark.flink.connector.conf -import org.apache.streampark.flink.connector.conf.FailoverStorageType.FailoverStorageType - -import java.util.Properties - -object ThresholdConfigOption { - def apply(prefixStr: String, properties: Properties = new Properties): ThresholdConfigOption = - new ThresholdConfigOption(prefixStr, properties) -} - -class ThresholdConfigOption(prefixStr: String, properties: Properties) { - - implicit val (prefix, prop) = (prefixStr, properties) - - val SIGN_COMMA = "," - - val bufferSize: ConfigOption[Int] = ConfigOption( - key = "threshold.bufferSize", - required = false, - defaultValue = 1000, - classType = classOf[Int]) - - val queueCapacity: ConfigOption[Int] = ConfigOption( - key = "threshold.queueCapacity", - required = false, - defaultValue = 10000, - classType = classOf[Int]) - - val delayTime: ConfigOption[Long] = ConfigOption( - key = "threshold.delayTime", - required = false, - defaultValue = 1000L, - classType = classOf[Long]) - - val timeout: ConfigOption[Int] = ConfigOption( - key = "threshold.requestTimeout", - required = false, - defaultValue = 2000, - classType = classOf[Int]) - - val numWriters: ConfigOption[Int] = ConfigOption( - key = "threshold.numWriters", - required = false, - defaultValue = Runtime.getRuntime.availableProcessors(), - classType = classOf[Int]) - - val maxRetries: ConfigOption[Int] = ConfigOption( - key = "threshold.retries", - required = false, - defaultValue = 3, - classType = classOf[Int]) - - val storageType: ConfigOption[conf.FailoverStorageType.Value] = ConfigOption( - key = "failover.storage", - required = false, - classType = classOf[FailoverStorageType], - defaultValue = FailoverStorageType.NONE, - handle = k => { - FailoverStorageType.get(properties.getProperty(k)) - } - ) - - val failoverTable: ConfigOption[String] = ConfigOption( - key = "failover.table", - required = false, - defaultValue = "", - classType = classOf[String]) - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverChecker.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverChecker.scala deleted file mode 100644 index 85b67f9f12..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverChecker.scala +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.failover - -import org.apache.streampark.common.util.{Logger, ThreadUtils} - -import java.util.concurrent.{Executors, ScheduledExecutorService, ThreadFactory, TimeUnit} - -import org.apache.streampark.common.util.Implicits._ -import scala.collection.mutable.ListBuffer - -case class FailoverChecker(delayTime: Long) extends AutoCloseable with Logger { - - val sinkBuffers: ListBuffer[SinkBuffer] = ListBuffer[SinkBuffer]() - val factory: ThreadFactory = ThreadUtils.threadFactory("FailoverChecker") - val scheduledExecutorService: ScheduledExecutorService = - Executors.newSingleThreadScheduledExecutor(factory) - scheduledExecutorService.scheduleWithFixedDelay( - getTask, - delayTime, - delayTime, - TimeUnit.MILLISECONDS) - logInfo(s"Build Sink scheduled checker, timeout (microSeconds) = $delayTime") - - def addSinkBuffer(buffer: SinkBuffer): Unit = { - this.synchronized(sinkBuffers.add(buffer)) - logDebug(s"Add SinkBuffer, size: ${buffer.bufferSize}") - } - - def getTask(): Runnable = new Runnable { - override def run(): Unit = { - this synchronized { - logDebug(s"Start checking buffers. Current count of buffers = ${sinkBuffers.size}") - sinkBuffers.foreach(_.tryAddToQueue()) - } - } - } - - override def close(): Unit = ThreadUtils.shutdownExecutorService(scheduledExecutorService) - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverWriter.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverWriter.scala deleted file mode 100644 index 390a90125e..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/FailoverWriter.scala +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.failover - -import org.apache.streampark.common.conf.ConfigConst._ -import org.apache.streampark.common.util._ -import org.apache.streampark.flink.connector.conf.FailoverStorageType._ - -import org.apache.kafka.clients.producer.{Callback, KafkaProducer, ProducerRecord, RecordMetadata} - -import java.util._ -import java.util.concurrent.locks.ReentrantLock - -import org.apache.streampark.common.util.Implicits._ - -class FailoverWriter(failoverStorage: FailoverStorageType, properties: Properties) - extends AutoCloseable - with Logger { - - private[this] object Lock { - @volatile var initialized = false - val lock = new ReentrantLock() - } - - private var kafkaProducer: KafkaProducer[String, String] = _ - - def write(request: SinkRequest): Unit = { - this.synchronized { - val table = request.table.split("\\.").last - failoverStorage match { - case NONE => - case Console => - val records = request.records.map(x => s"(${cleanUp(x)})") - logInfo(s"failover body: [ ${records.mkString(",")} ]") - case Kafka => - if (!Lock.initialized) { - try { - Lock.lock.lock() - if (!Lock.initialized) { - Lock.initialized = true - properties.put( - "key.serializer", - "org.apache.kafka.common.serialization.StringSerializer") - properties.put( - "value.serializer", - "org.apache.kafka.common.serialization.StringSerializer") - kafkaProducer = new KafkaProducer[String, String](properties) - } - } catch { - case exception: Exception => { - logError( - s"build Failover storageType:KAFKA failed exception ${exception.getStackTrace - .mkString("Array(", ", ", ")")}") - throw exception - } - } finally { - Lock.lock.unlock() - } - } - val topic = properties.getProperty(KEY_KAFKA_TOPIC) - val timestamp = System.currentTimeMillis() - val records = request.records.map(cleanUp) - val sendData = - s""" - |{ - |"values":[${records.mkString(",")}], - |"timestamp":$timestamp - |} - |""".stripMargin - val record = new ProducerRecord[String, String](topic, sendData) - kafkaProducer - .send( - record, - new Callback() { - override def onCompletion(recordMetadata: RecordMetadata, e: Exception): Unit = { - logInfo( - s"Failover successful!! storageType:Kafka,table: $table,size:${request.size}") - } - } - ) - .get() - case MySQL => - if (!Lock.initialized) { - try { - Lock.lock.lock() - if (!Lock.initialized) { - Lock.initialized = true - properties.put(KEY_ALIAS, s"failover-$table") - val mysqlConnect = JdbcUtils.getConnection(properties) - val mysqlTable = - mysqlConnect.getMetaData.getTables(null, null, table, Array("TABLE", "VIEW")) - if (!mysqlTable.next()) { - JdbcUtils.execute( - mysqlConnect, - s"create table $table (`values` text, `timestamp` bigint)") - logWarn(s"Failover storageType:MySQL,table: $table is not exist,auto created...") - } - } - } catch { - case exception: Exception => - logError( - s"build Failover storageType:MySQL failed exception ${exception.getStackTrace}") - throw exception - } finally { - Lock.lock.unlock() - } - } - val timestamp = System.currentTimeMillis() - val records = request.records.map( - x => { - val v = cleanUp(x) - s""" ($v,$timestamp) """.stripMargin - }) - val sql = s"INSERT INTO $table(`values`,`timestamp`) VALUES ${records.mkString(",")} " - JdbcUtils.update(sql)(properties) - logInfo(s"Failover successful!! storageType:MySQL,table: $table,size:${request.size}") - case _ => - throw new UnsupportedOperationException( - s"[StreamPark] unsupported failover storageType:$failoverStorage") - } - } - } - - private[this] def cleanUp(record: String) = { - s""" "${record.replace("\"", "\\\"")}" """.stripMargin - } - - override def close(): Unit = { - if (kafkaProducer != null) kafkaProducer.close() - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkBuffer.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkBuffer.scala deleted file mode 100644 index 27b7634568..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkBuffer.scala +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.failover - -import org.apache.streampark.common.util.Logger - -import java.util -import java.util.Collections -import java.util.concurrent.CopyOnWriteArrayList - -import org.apache.streampark.common.util.Implicits._ - -case class SinkBuffer(writer: SinkWriter, flushInterval: Long, bufferSize: Int) - extends AutoCloseable - with Logger { - - private var timestamp = 0L - - var localValues = new CopyOnWriteArrayList[String]() - - def put(value: String): Unit = { - tryAddToQueue() - localValues.add(value) - timestamp = System.currentTimeMillis - } - - def tryAddToQueue(): Unit = { - this.synchronized { - if (flush) { - addToQueue() - } - } - } - - private[this] def addToQueue(): Unit = { - val deepCopy = buildDeepCopy(localValues) - val params = SinkRequest(deepCopy) - logDebug(s"Build blank with params: buffer size = ${params.size}") - writer.write(params) - localValues.clear() - } - - private[this] def flush: Boolean = { - if (localValues.nonEmpty) { - localValues.size >= bufferSize || { - if (timestamp == 0) false - else { - val current = System.currentTimeMillis - current - timestamp > flushInterval - } - } - } else false - } - - private[this] def buildDeepCopy(original: util.List[String]): util.List[String] = - Collections.unmodifiableList(new util.ArrayList[String](original)) - - override def close(): Unit = if (localValues.nonEmpty) addToQueue() - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkRequest.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkRequest.scala deleted file mode 100644 index f1b71fba66..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkRequest.scala +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.failover - -import org.apache.streampark.common.util.Logger - -import java.util -import java.util.regex.Pattern - -import org.apache.streampark.common.Constant - -import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` -import scala.collection.mutable -import scala.collection.mutable.ListBuffer - -case class SinkRequest(records: util.List[String], var attemptCounter: Int = 0) extends Logger { - def incrementCounter(): Unit = attemptCounter += 1 - - def size: Int = records.size() - - private[this] lazy val TABLE_REGEXP = - Pattern.compile("(insert\\s+into|update|delete)\\s+(.*?)(\\(|\\s+)", Pattern.CASE_INSENSITIVE) - - private[this] lazy val INSERT_REGEXP = - Pattern.compile("^(.*?)\\s+(values|value)(.*)", Pattern.CASE_INSENSITIVE) - - lazy val sqlStatement: List[String] = { - var result: List[String] = List.empty[String] - val prefixMap: mutable.Map[String, ListBuffer[String]] = - mutable.Map.empty[String, ListBuffer[String]] - - records.forEach( - x => { - // group statements by the part before 'value(s)' in insert statements. - val valueMatcher = INSERT_REGEXP.matcher(x) - if (valueMatcher.find()) { - val prefix = valueMatcher.group(1) - prefixMap.get(prefix) match { - case Some(value) => value += valueMatcher.group(3) - case None => prefixMap(prefix) = ListBuffer(valueMatcher.group(3)) - } - } else { - // other statements will be ignored. - logWarn(s"ignore record: $x") - } - }) - if (prefixMap.nonEmpty) { - // combine statements by the part before 'value(s)' in insert statements. - result = prefixMap.map(m => s"""${m._1} VALUES ${m._2.mkString(",")}""").toList - } - - logDebug(s"script to commit: ${result.mkString(Constant.SEMICOLON)}") - - result - } - - lazy val table: String = { - // 1) insert into default.table(c1,c2) values ... - // 2) insert into default.table values ..." - // 3) update default.table where ..." - // 4) delete default.table where ..." - val matcher = TABLE_REGEXP.matcher(records.head) - if (matcher.find()) { - matcher.group(2) - } else null - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkWriter.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkWriter.scala deleted file mode 100644 index 2ac72fe35a..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/failover/SinkWriter.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.failover - -import org.apache.streampark.common.util.Logger - -trait SinkWriter extends AutoCloseable with Logger { - def write(request: SinkRequest): Unit -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/sink/Sink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/sink/Sink.scala deleted file mode 100644 index a830cc0114..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/main/scala/org/apache/streampark/flink/connector/sink/Sink.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.sink - -import org.apache.flink.streaming.api.datastream.DataStreamSink - -trait Sink extends Serializable { - - def afterSink[T]( - sink: DataStreamSink[T], - parallelism: Int, - name: String, - uid: String): DataStreamSink[T] = { - if (parallelism > 0) { - sink.setParallelism(parallelism) - } - if (name != null && name.nonEmpty) { - sink.name(name) - } - if (uid != null && uid.nonEmpty) { - sink.uid(uid) - } - sink - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/test/scala/org/apache/streampark/flink/connector/failover/SinkRequestTest.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/test/scala/org/apache/streampark/flink/connector/failover/SinkRequestTest.scala deleted file mode 100644 index 634a8cc616..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-base/src/test/scala/org/apache/streampark/flink/connector/failover/SinkRequestTest.scala +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.failover - -import org.junit.jupiter.api.{Assertions, Test} - -import org.apache.streampark.common.util.Implicits._ - -class SinkRequestTest { - @Test - def sqlStatement(): Unit = { - // input statements - val statementsList = List( - "insert into table_1(col1, col2) values(1, 2)", - "insert into table_1(col1, col2) values(11, 22)", - "insert into table_1(col1, col2, col3) values(11, 22, 33)", - "insert into table_2(col1, col2, col3) values(11, 22, 33)", - ) - - val sinkRequest = SinkRequest(statementsList) - - // expected result - val expectedSqlStatement = List( - "insert into table_2(col1, col2, col3) VALUES (11, 22, 33)", - "insert into table_1(col1, col2) VALUES (1, 2),(11, 22)", - "insert into table_1(col1, col2, col3) VALUES (11, 22, 33)", - ) - - // comparison of result should be based on Set, that is, there is no need to care about the order of elements - Assertions.assertTrue(sinkRequest.sqlStatement.toSet == expectedSqlStatement.toSet) - - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/pom.xml deleted file mode 100644 index d344058e1a..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/pom.xml +++ /dev/null @@ -1,98 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink-connector - 2.2.0-SNAPSHOT - - - streampark-flink-connector-clickhouse_${scala.binary.version} - StreamPark : Flink Connector Clickhouse - - - - - org.apache.streampark - streampark-flink-connector-base_${scala.binary.version} - ${project.version} - - - - - ru.yandex.clickhouse - clickhouse-jdbc - 0.3.2 - - - - - org.asynchttpclient - async-http-client - - - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseHttpConfig.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseHttpConfig.scala deleted file mode 100644 index 83b92516a5..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseHttpConfig.scala +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.clickhouse.conf - -import org.apache.streampark.flink.connector.conf.ThresholdConf - -import org.apache.streampark.common.Constant -import java.util.{Base64, Properties} -import java.util.concurrent.ThreadLocalRandom - -import org.apache.streampark.common.util.Implicits._ - -/** - * Flink sink for Clickhouse database. Powered by Async Http Client. - * - * High-performance library for loading data to Clickhouse. - * - * It has two triggers for loading data: by timeout and by buffer size. - */ -//--------------------------------------------------------------------------------------- - -class ClickHouseHttpConfig(parameters: Properties) - extends ThresholdConf(ClickHouseSinkConfigOption.CLICKHOUSE_SINK_PREFIX, parameters) { - - @transient val sinkOption: ClickHouseSinkConfigOption = - ClickHouseSinkConfigOption(properties = parameters) - - val user: String = sinkOption.user.get() - - val password: String = sinkOption.password.get() - - val hosts: List[String] = sinkOption.hosts.get() - - var currentHostId: Int = 0 - - val credentials: String = (user, password) match { - case (null, null) => null - case (u, p) => new String(Base64.getEncoder.encode(s"$u:$p".getBytes)) - } - - def getRandomHostUrl: String = { - currentHostId = ThreadLocalRandom.current.nextInt(hosts.size) - hosts.get(currentHostId) - } - - def nextHost: String = { - if (currentHostId >= hosts.size - 1) { - currentHostId = 0 - } else { - currentHostId += 1 - } - hosts.get(currentHostId) - } - - override def toString: String = { - s""" - |{ user: $user, password: ${Constant.DEFAULT_DATAMASK_STRING}, hosts: ${hosts.mkString(",")} } - |""".stripMargin - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseJdbcConfig.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseJdbcConfig.scala deleted file mode 100644 index 2c645fb213..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseJdbcConfig.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.clickhouse.conf - -import java.util.Properties - -/** - * Flink sink for Clickhouse database. Powered by Async Http Client. - * - * High-performance library for loading data to Clickhouse. - * - * It has two triggers for loading data: by timeout and by buffer size. - */ -//--------------------------------------------------------------------------------------- - -class ClickHouseJdbcConfig(parameters: Properties) extends Serializable { - - val sinkOption: ClickHouseSinkConfigOption = ClickHouseSinkConfigOption(properties = parameters) - - val user: String = sinkOption.user.get() - - val password: String = sinkOption.password.get() - - val jdbcUrl: String = sinkOption.jdbcUrl.get() - - val driverClassName: String = sinkOption.driverClassName.get() - - val batchSize: Int = sinkOption.batchSize.get() - - val flushInterval: Long = sinkOption.flushInterval.get() - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseSinkConfigOption.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseSinkConfigOption.scala deleted file mode 100644 index 4b054fd0c9..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/conf/ClickHouseSinkConfigOption.scala +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.clickhouse.conf - -import org.apache.streampark.common.Constant -import org.apache.streampark.common.conf.ConfigOption -import org.apache.streampark.common.util.ConfigUtils - -import org.asynchttpclient.config.AsyncHttpClientConfigDefaults - -import java.util.Properties - -import org.apache.streampark.common.util.Implicits._ - -object ClickHouseSinkConfigOption { - - val CLICKHOUSE_SINK_PREFIX = "clickhouse.sink" - - /** - * @param properties - * @return - */ - def apply( - prefixStr: String = CLICKHOUSE_SINK_PREFIX, - properties: Properties = new Properties): ClickHouseSinkConfigOption = - new ClickHouseSinkConfigOption(prefixStr, properties) - -} - -class ClickHouseSinkConfigOption(prefixStr: String, properties: Properties) extends Serializable { - - implicit val (prefix, prop) = (prefixStr, properties) - - val SIGN_COMMA = "," - - val hosts: ConfigOption[List[String]] = ConfigOption[List[String]]( - key = "hosts", - required = false, - defaultValue = List(), - classType = classOf[List[String]], - handle = k => { - properties - .getProperty(k) - .split(SIGN_COMMA) - .filter(_.nonEmpty) - .map(_.replaceAll("\\s+", "").replaceFirst("^http://|^", Constant.HTTP_SCHEMA)) - .toList - } - ) - - val user: ConfigOption[String] = - ConfigOption(key = "user", required = true, classType = classOf[String]) - - val password: ConfigOption[String] = - ConfigOption(key = "password", required = false, defaultValue = "", classType = classOf[String]) - - val database: ConfigOption[String] = ConfigOption( - key = "database", - required = true, - defaultValue = Constant.DEFAULT, - classType = classOf[String]) - - val requestTimeout: ConfigOption[Int] = ConfigOption( - key = "requestTimeout", - required = false, - defaultValue = AsyncHttpClientConfigDefaults.defaultRequestTimeout, - classType = classOf[Int]) - - val connectTimeout: ConfigOption[Int] = ConfigOption( - key = "connectTimeout", - required = false, - defaultValue = AsyncHttpClientConfigDefaults.defaultConnectTimeout(), - classType = classOf[Long]) - - val maxRequestRetry: ConfigOption[Int] = ConfigOption( - key = "maxRequestRetry", - required = false, - defaultValue = AsyncHttpClientConfigDefaults.defaultMaxRequestRetry(), - classType = classOf[Long]) - - val maxConnections: ConfigOption[Int] = ConfigOption( - key = "maxConnections", - required = false, - defaultValue = AsyncHttpClientConfigDefaults.defaultMaxConnections(), - classType = classOf[Int]) - - val failoverTable: ConfigOption[String] = - ConfigOption(key = "failover.table", required = false, classType = classOf[String]) - - val jdbcUrl: ConfigOption[String] = - ConfigOption(key = "jdbcUrl", required = false, classType = classOf[String]) - - val driverClassName: ConfigOption[String] = ConfigOption( - key = "driverClassName", - required = false, - defaultValue = null, - classType = classOf[String]) - - val batchSize: ConfigOption[Int] = - ConfigOption(key = "batchSize", required = false, defaultValue = 1, classType = classOf[Int]) - - val flushInterval: ConfigOption[Long] = ConfigOption( - key = "flushInterval", - required = false, - defaultValue = 1000L, - classType = classOf[Long], - handle = k => { - properties.remove(k).toString.toLong - }) - - def getInternalConfig(): Properties = { - ConfigUtils.getConf(prop, prefix)("") - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/AsyncClickHouseSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/AsyncClickHouseSinkFunction.scala deleted file mode 100644 index 14c65d3fd1..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/AsyncClickHouseSinkFunction.scala +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.clickhouse.internal - -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.enums.ApiType.ApiType -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.clickhouse.conf.ClickHouseHttpConfig -import org.apache.streampark.flink.connector.clickhouse.internal -import org.apache.streampark.flink.connector.clickhouse.util.ClickhouseConvertUtils.convert -import org.apache.streampark.flink.connector.failover.{FailoverChecker, SinkBuffer} -import org.apache.streampark.flink.connector.function.TransformFunction - -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction - -import java.util.Properties - -import scala.util.{Failure, Try} - -class AsyncClickHouseSinkFunction[T](apiType: ApiType = ApiType.scala, properties: Properties) - extends RichSinkFunction[T] - with Logger { - - private[this] object Lock { - @volatile var initialized = false - val lock = new Object() - } - - private[this] var scalaSqlFunc: T => String = _ - private[this] var javaSqlFunc: TransformFunction[T, String] = _ - - // for Scala - def this(properties: Properties, scalaSqlFunc: T => String) = { - this(ApiType.scala, properties) - this.scalaSqlFunc = scalaSqlFunc - } - - // for JAVA - def this(properties: Properties, javaSqlFunc: TransformFunction[T, String]) = { - this(ApiType.java, properties) - this.javaSqlFunc = javaSqlFunc - } - - @transient var clickHouseConf: ClickHouseHttpConfig = _ - @transient var sinkBuffer: SinkBuffer = _ - @transient var clickHouseWriter: ClickHouseSinkWriter = _ - @transient var failoverChecker: FailoverChecker = _ - @volatile var isClosed: Boolean = false - - override def open(config: Configuration): Unit = { - if (!Lock.initialized) { - Lock.lock.synchronized { - if (!Lock.initialized) { - Lock.initialized = true - clickHouseConf = new ClickHouseHttpConfig(properties) - clickHouseWriter = internal.ClickHouseSinkWriter(clickHouseConf) - failoverChecker = FailoverChecker(clickHouseConf.delayTime) - sinkBuffer = - SinkBuffer(clickHouseWriter, clickHouseConf.delayTime, clickHouseConf.bufferSize) - failoverChecker.addSinkBuffer(sinkBuffer) - logInfo("AsyncClickHouseSink initialize... ") - } - } - } - } - - override def invoke(value: T): Unit = { - val sql = (javaSqlFunc, scalaSqlFunc) match { - case (null, null) => convert[T](value) - case _ => - apiType match { - case ApiType.java => javaSqlFunc.transform(value) - case ApiType.scala => scalaSqlFunc(value) - } - } - - Try(sinkBuffer.put(sql)) match { - case Failure(e) => - logError(s"""Error while sending data to Clickhouse, record = $sql,error:$e""") - throw e - case _ => - } - } - - override def close(): Unit = { - if (!isClosed) { - Lock.lock.synchronized { - if (!isClosed) { - if (sinkBuffer != null) sinkBuffer.close() - if (clickHouseWriter != null) clickHouseWriter.close() - if (failoverChecker != null) failoverChecker.close() - isClosed = true - super.close() - } - } - } - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkFunction.scala deleted file mode 100644 index c13db6b3d5..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkFunction.scala +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.clickhouse.internal - -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.enums.ApiType.ApiType -import org.apache.streampark.common.util.{JdbcUtils, Logger} -import org.apache.streampark.flink.connector.clickhouse.conf.ClickHouseJdbcConfig -import org.apache.streampark.flink.connector.clickhouse.util.ClickhouseConvertUtils.convert -import org.apache.streampark.flink.connector.function.TransformFunction - -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction} -import ru.yandex.clickhouse.ClickHouseDataSource -import ru.yandex.clickhouse.settings.ClickHouseProperties - -import java.sql.{Connection, Statement} -import java.util -import java.util.Properties -import java.util.concurrent.atomic.AtomicLong - -import org.apache.streampark.common.util.Implicits._ -import scala.util.Try - -class ClickHouseSinkFunction[T](apiType: ApiType = ApiType.scala, config: Properties) - extends RichSinkFunction[T] - with Logger { - private var connection: Connection = _ - private var statement: Statement = _ - var clickHouseConf: ClickHouseJdbcConfig = new ClickHouseJdbcConfig(config) - - private val batchSize = clickHouseConf.batchSize - private val offset: AtomicLong = new AtomicLong(0L) - private var timestamp = 0L - private val flushInterval = clickHouseConf.flushInterval - private val sqlValues = new util.ArrayList[String](batchSize) - private var insertSqlPrefixes: String = _ - - private[this] var scalaSqlFunc: T => String = _ - private[this] var javaSqlFunc: TransformFunction[T, String] = _ - - // for Scala - def this(properties: Properties, scalaSqlFunc: T => String) = { - this(ApiType.scala, properties) - this.scalaSqlFunc = scalaSqlFunc - } - - // for JAVA - def this(properties: Properties, javaSqlFunc: TransformFunction[T, String]) = { - - this(ApiType.java, properties) - this.javaSqlFunc = javaSqlFunc - } - - override def open(parameters: Configuration): Unit = { - val user: String = clickHouseConf.user - val driver: String = clickHouseConf.driverClassName - val properties = new ClickHouseProperties() - (user, driver) match { - case (u, d) if u != null && d != null => - Class.forName(d) - properties.setUser(u) - case (null, null) => - case (_, d) if d != null => Class.forName(d) - case _ => properties.setUser(user) - } - // reflect set all properties... - clickHouseConf.sinkOption - .getInternalConfig() - .foreach( - x => { - Try(Option(properties.getClass.getDeclaredField(x._1))).getOrElse(None) match { - case Some(field) => - field.setAccessible(true) - field.getType.getSimpleName match { - case "String" => field.set(properties, x._2) - case "int" | "Integer" => field.set(properties, x._2.toInt) - case "long" | "Long" => field.set(properties, x._2.toLong) - case "boolean" | "Boolean" => field.set(properties, x._2.toBoolean) - case _ => - } - case None => - logWarn( - s"ClickHouseProperties config error,property:${x._1} invalid,please see ru.yandex.clickhouse.settings.ClickHouseProperties") - } - }) - val dataSource = new ClickHouseDataSource(clickHouseConf.jdbcUrl, properties) - connection = dataSource.getConnection - } - - override def invoke(value: T, context: SinkFunction.Context): Unit = { - require(connection != null) - val sql = (javaSqlFunc, scalaSqlFunc) match { - case (null, null) => convert[T](value) - case _ => - apiType match { - case ApiType.java => javaSqlFunc.transform(value) - case ApiType.scala => scalaSqlFunc(value) - } - } - - batchSize match { - case 1 => - Try(connection.prepareStatement(sql).executeUpdate).recover { - case e => - logError(s"ClickHouseSink invoke error: $e") - throw e - }.get - case batch => - Try { - sqlValues.add(sql) - (offset.incrementAndGet() % batch, System.currentTimeMillis()) match { - case (0, _) => execBatch() - case (_, current) if current - timestamp > flushInterval => execBatch() - case _ => - } - }.recover { - case e => - logError(s"""ClickHouseSink batch invoke error:$sqlValues""") - throw e - }.get - } - } - - override def close(): Unit = { - execBatch() - JdbcUtils.close(statement, connection) - } - - private[this] def execBatch(): Unit = { - if (offset.get() > 0) { - try { - logInfo(s"ClickHouseSink batch ${offset.get()} insert begin..") - offset.set(0) - val valuesStr: String = sqlValues.mkString(",") - val sql = s"$insertSqlPrefixes $valuesStr" - // clickhouse batch insert return num always 1 - val insertNum: Int = connection.prepareStatement(sql).executeUpdate() - logInfo(s"ClickHouseSink batch successful, execute size : $insertNum") - timestamp = System.currentTimeMillis() - } finally { - sqlValues.clear() - } - } - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkWriter.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkWriter.scala deleted file mode 100644 index 2b7c219955..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkWriter.scala +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.clickhouse.internal - -import org.apache.streampark.common.util.{Logger, ThreadUtils} -import org.apache.streampark.flink.connector.clickhouse.conf.ClickHouseHttpConfig -import org.apache.streampark.flink.connector.clickhouse.internal -import org.apache.streampark.flink.connector.failover.{SinkRequest, SinkWriter} - -import org.asynchttpclient.{AsyncHttpClient, DefaultAsyncHttpClientConfig, Dsl} - -import java.util.concurrent._ - -import org.apache.streampark.common.util.Implicits._ -import scala.collection.mutable.ListBuffer - -case class ClickHouseSinkWriter(clickHouseConfig: ClickHouseHttpConfig) - extends SinkWriter - with Logger { - private val callbackServiceFactory = - ThreadUtils.threadFactory("ClickHouse-writer-callback-executor") - private val threadFactory: ThreadFactory = ThreadUtils.threadFactory("ClickHouse-writer") - - var callbackService: ExecutorService = new ThreadPoolExecutor( - math.max(Runtime.getRuntime.availableProcessors / 4, 2), - Integer.MAX_VALUE, - 60L, - TimeUnit.SECONDS, - new LinkedBlockingQueue[Runnable], - callbackServiceFactory) - - var tasks: ListBuffer[ClickHouseWriterTask] = ListBuffer[ClickHouseWriterTask]() - var recordQueue: BlockingQueue[SinkRequest] = - new LinkedBlockingQueue[SinkRequest](clickHouseConfig.queueCapacity) - var asyncHttpClient: AsyncHttpClient = Dsl.asyncHttpClient( - new DefaultAsyncHttpClientConfig.Builder() - .setRequestTimeout(clickHouseConfig.sinkOption.requestTimeout.get()) - .setConnectTimeout(clickHouseConfig.sinkOption.connectTimeout.get()) - .setMaxRequestRetry(clickHouseConfig.sinkOption.maxRequestRetry.get()) - .setMaxConnections(clickHouseConfig.sinkOption.maxConnections.get()) - .build()) - var service: ExecutorService = - Executors.newFixedThreadPool(clickHouseConfig.numWriters, threadFactory) - - for (i <- 0 until clickHouseConfig.numWriters) { - val task = internal.ClickHouseWriterTask( - i, - clickHouseConfig, - asyncHttpClient, - recordQueue, - callbackService) - tasks.add(task) - service.submit(task) - } - - def write(request: SinkRequest): Unit = { - try { - recordQueue.put(request) - } catch { - case e: InterruptedException => - logError(s"Interrupted error while putting data to queue,error:$e") - Thread.currentThread.interrupt() - throw new RuntimeException(e) - } - } - - override def close(): Unit = { - logInfo("Closing ClickHouse-writer...") - tasks.foreach(_.close()) - ThreadUtils.shutdownExecutorService(service) - ThreadUtils.shutdownExecutorService(callbackService) - asyncHttpClient.close() - logInfo(s"${classOf[ClickHouseSinkWriter].getSimpleName} is closed") - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseWriterTask.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseWriterTask.scala deleted file mode 100644 index 3aaeabbaf8..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseWriterTask.scala +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.clickhouse.internal - -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.clickhouse.conf.ClickHouseHttpConfig -import org.apache.streampark.flink.connector.failover.{FailoverWriter, SinkRequest} - -import io.netty.handler.codec.http.HttpHeaderNames -import org.asynchttpclient.{AsyncHttpClient, ListenableFuture, Request, Response} - -import java.util.concurrent.{BlockingQueue, ExecutorService, TimeUnit} - -import org.apache.streampark.common.util.Implicits._ -import scala.util.Try - -case class ClickHouseWriterTask( - id: Int, - clickHouseConf: ClickHouseHttpConfig, - asyncHttpClient: AsyncHttpClient, - queue: BlockingQueue[SinkRequest], - callbackService: ExecutorService) - extends Runnable - with AutoCloseable - with Logger { - - @volatile var isWorking = false - - val failoverWriter: FailoverWriter = - new FailoverWriter(clickHouseConf.storageType, clickHouseConf.getFailoverConfig) - - override def run(): Unit = - try { - isWorking = true - logInfo(s"Start writer task, id = $id") - while (isWorking || queue.nonEmpty) { - val req = queue.poll(300, TimeUnit.MILLISECONDS) - if (req != null) { - send(req) - } - } - } catch { - case e: Exception => - logError("Error while inserting data", e) - throw new RuntimeException(e) - } finally { - logInfo(s"Task id = $id is finished") - } - - def send(sinkRequest: SinkRequest): Unit = { - // ClickHouse's http API does not accept EMPTY request body - if (sinkRequest.sqlStatement == null || sinkRequest.sqlStatement.isEmpty) { - logWarn(s"Skip empty sql statement") - return - } - - val requests = buildRequest(sinkRequest) - requests.foreach( - request => { - logDebug(s"Ready to fire request: $request") - val whenResponse = asyncHttpClient.executeRequest(request) - val callback = respCallback(whenResponse, sinkRequest) - whenResponse.addListener(callback, callbackService) - }) - } - - private def buildRequest(sinkRequest: SinkRequest): List[Request] = { - logDebug(s"There is [${sinkRequest.sqlStatement.size}] statement(s) in SinkRequest ") - // ClickHouse's http API does not accept multiple statements, so requests should be built by splitting statements - sinkRequest.sqlStatement.filter(_.nonEmpty).map( - statement => { - val host = clickHouseConf.getRandomHostUrl - val builder = asyncHttpClient - .preparePost(host) - .setRequestTimeout(clickHouseConf.timeout) - .setHeader(HttpHeaderNames.CONTENT_TYPE, "text/plain; charset=utf-8") - .setBody(statement) - if (clickHouseConf.credentials != null) { - builder.setHeader(HttpHeaderNames.AUTHORIZATION, "Basic " + clickHouseConf.credentials) - } - builder.build - }) - } - - def respCallback(whenResponse: ListenableFuture[Response], sinkRequest: SinkRequest): Runnable = - new Runnable { - override def run(): Unit = { - Try(whenResponse.get()).getOrElse(null) match { - case null => - logError( - s"""Error ClickHouseSink executing callback, params = $clickHouseConf,can not get Response. """) - handleFailedResponse(null, sinkRequest) - case resp if resp.getStatusCode != 200 => - logError( - s"Error ClickHouseSink executing callback, params = $clickHouseConf, StatusCode = ${resp.getStatusCode} ") - handleFailedResponse(resp, sinkRequest) - case _ => - } - } - } - - /** - * if send data to ClickHouse Failed, retry maxRetries, if still failed,flush data to - * failoverStorage - * - * @param response - * @param sinkRequest - */ - def handleFailedResponse(response: Response, sinkRequest: SinkRequest): Unit = { - if (sinkRequest.attemptCounter > clickHouseConf.maxRetries) { - logWarn( - s"""Failed to send data to ClickHouse, cause: limit of attempts is exceeded. ClickHouse response = $response. Ready to flush data to ${clickHouseConf.storageType}""") - failoverWriter.write(sinkRequest) - logInfo( - s"Failover Successful, StorageType = ${clickHouseConf.storageType}, size = ${sinkRequest.size}") - } else { - sinkRequest.incrementCounter() - logWarn( - s"Next attempt to send data to ClickHouse, table = ${sinkRequest.table}, buffer size = ${sinkRequest.size}, current attempt num = ${sinkRequest.attemptCounter}, max attempt num = ${clickHouseConf.maxRetries}, response = $response") - queue.put(sinkRequest) - } - } - - override def close(): Unit = { - isWorking = false - failoverWriter.close() - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/AsyncClickHouseOutputFormat.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/AsyncClickHouseOutputFormat.scala deleted file mode 100644 index 350b1eb7fa..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/AsyncClickHouseOutputFormat.scala +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.clickhouse.sink - -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.clickhouse.internal.AsyncClickHouseSinkFunction -import org.apache.streampark.flink.connector.function.TransformFunction - -import org.apache.flink.api.common.io.RichOutputFormat -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration - -import java.util.Properties - -class AsyncClickHouseOutputFormat[T: TypeInformation](prop: Properties) - extends RichOutputFormat[T] - with Logger { - - var sinkFunction: AsyncClickHouseSinkFunction[T] = _ - - var configuration: Configuration = _ - - // for Scala - def this(properties: Properties, scalaSqlFunc: T => String) = { - this(properties) - sinkFunction = new AsyncClickHouseSinkFunction[T](properties, scalaSqlFunc) - } - - // for JAVA - def this(properties: Properties, javaSqlFunc: TransformFunction[T, String]) = { - this(properties) - sinkFunction = new AsyncClickHouseSinkFunction[T](properties, javaSqlFunc) - } - - override def configure(configuration: Configuration): Unit = this.configuration = configuration - - override def open(taskNumber: Int, numTasks: Int): Unit = sinkFunction.open(this.configuration) - - override def writeRecord(record: T): Unit = sinkFunction.invoke(record, null) - - override def close(): Unit = sinkFunction.close() -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseOutputFormat.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseOutputFormat.scala deleted file mode 100644 index 2a89d5e2fd..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseOutputFormat.scala +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.clickhouse.sink - -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.clickhouse.internal.ClickHouseSinkFunction -import org.apache.streampark.flink.connector.function.TransformFunction - -import org.apache.flink.api.common.io.RichOutputFormat -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration - -import java.util.Properties - -class ClickHouseOutputFormat[T: TypeInformation](prop: Properties) - extends RichOutputFormat[T] - with Logger { - - var sinkFunction: ClickHouseSinkFunction[T] = _ - - var configuration: Configuration = _ - - // for Scala - def this(properties: Properties, scalaSqlFunc: T => String) = { - this(properties) - sinkFunction = new ClickHouseSinkFunction[T](properties, scalaSqlFunc) - } - - // for JAVA - def this(properties: Properties, javaSqlFunc: TransformFunction[T, String]) = { - this(properties) - sinkFunction = new ClickHouseSinkFunction[T](properties, javaSqlFunc) - } - - override def configure(configuration: Configuration): Unit = this.configuration = configuration - - override def open(taskNumber: Int, numTasks: Int): Unit = sinkFunction.open(this.configuration) - - override def writeRecord(record: T): Unit = sinkFunction.invoke(record, null) - - override def close(): Unit = sinkFunction.close() -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseSink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseSink.scala deleted file mode 100644 index 65c70d98f3..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/sink/ClickHouseSink.scala +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.clickhouse.sink - -import org.apache.streampark.common.util._ -import org.apache.streampark.flink.connector.clickhouse.internal.{AsyncClickHouseSinkFunction, ClickHouseSinkFunction} -import org.apache.streampark.flink.connector.function.TransformFunction -import org.apache.streampark.flink.connector.sink.Sink -import org.apache.streampark.flink.core.scala.StreamingContext - -import org.apache.flink.streaming.api.datastream.{DataStream => JavaDataStream} -import org.apache.flink.streaming.api.datastream.DataStreamSink -import org.apache.flink.streaming.api.scala.DataStream - -import java.util.Properties - -import scala.annotation.meta.param - -object ClickHouseSink { - - val sinkNullHintMsg = "Sink Stream must not null" - - /** - * @param property - * @param parallelism - * @param name - * @param uid - * @param ctx - * @return - */ - - def apply( - @(transient @param) - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null)(implicit ctx: StreamingContext): ClickHouseSink = - new ClickHouseSink(ctx, property, parallelism, name, uid) - -} - -class ClickHouseSink( - @(transient @param) ctx: StreamingContext, - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null)(implicit alias: String = "") - extends Sink - with Logger { - - val prop = ctx.parameter.getProperties - - Utils.copyProperties(property, prop) - - def this(ctx: StreamingContext, alias: String) { - this(ctx, new Properties, 0, null, null)(alias) - } - - def this(ctx: StreamingContext) { - this(ctx, new Properties, 0, null, null)("") - } - - /** - * asynchronous Write - * - * @param scala - * stream - * @param toSQLFn - * @tparam T - * @return - */ - def asyncSink[T](stream: DataStream[T])(implicit - toSQLFn: T => String = null): DataStreamSink[T] = { - require(stream != null, () => sinkNullHintMsg) - val sinkFun = new AsyncClickHouseSinkFunction[T](prop, toSQLFn) - val sink = stream.addSink(sinkFun) - afterSink(sink, parallelism, name, uid) - } - - /** - * asynchronous Write - * - * @param java - * stream - * @param toSQLFn - * @tparam T - * @return - */ - def asyncSink[T]( - stream: JavaDataStream[T], - toSQLFn: TransformFunction[T, String]): DataStreamSink[T] = { - require(stream != null, () => sinkNullHintMsg) - val sinkFun = new AsyncClickHouseSinkFunction[T](prop, toSQLFn) - val sink = stream.addSink(sinkFun) - afterSink(sink, parallelism, name, uid) - } - - /** - * asynchronous Write with all Feild - * - * @param stream - * @tparam T - * @return - */ - def asyncSink[T](stream: JavaDataStream[T]): DataStreamSink[T] = asyncSink(stream, null) - - /** - * synchronous Write - * - * @param scala - * stream - * @param toSQLFn - * @tparam T - * @return - */ - def jdbcSink[T](stream: DataStream[T])(implicit - toSQLFn: T => String = null): DataStreamSink[T] = { - require(stream != null, () => sinkNullHintMsg) - val sinkFun = new ClickHouseSinkFunction[T](prop, toSQLFn) - val sink = stream.addSink(sinkFun) - afterSink(sink, parallelism, name, uid) - } - - /** - * synchronous Write - * - * @param java - * stream - * @param sqlFromFn - * @tparam T - * @return - */ - def jdbcSink[T]( - stream: JavaDataStream[T], - sqlFromFn: TransformFunction[T, String]): DataStreamSink[T] = { - require(stream != null, () => sinkNullHintMsg) - val sinkFun = new ClickHouseSinkFunction[T](prop, sqlFromFn) - val sink = stream.addSink(sinkFun) - afterSink(sink, parallelism, name, uid) - } - - /** - * synchronous Write with all Feild - * - * @param stream - * @tparam T - * @return - */ - def jdbcSink[T](stream: JavaDataStream[T]): DataStreamSink[T] = jdbcSink(stream, null) - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/util/ClickhouseConvertUtils.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/util/ClickhouseConvertUtils.scala deleted file mode 100644 index f4ef102c64..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/scala/org/apache/streampark/flink/connector/clickhouse/util/ClickhouseConvertUtils.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.clickhouse.util - -object ClickhouseConvertUtils { - - def convert[T](value: T): String = { - val buffer = new StringBuilder("(") - val fields = value.getClass.getDeclaredFields - fields.foreach( - f => { - f.setAccessible(true) - val v = f.get(value) - f.getType.getSimpleName match { - case "String" => buffer.append(s""""$v",""".stripMargin) - case _ => buffer.append(s"""$v,""".stripMargin) - } - }) - buffer.toString().replaceFirst(",$", ")") - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/pom.xml deleted file mode 100644 index 1fedfb502d..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink-connector - 2.2.0-SNAPSHOT - - - streampark-flink-connector-doris_${scala.binary.version} - StreamPark : Flink Connector Doris - - - - org.apache.streampark - streampark-flink-connector-base_${scala.binary.version} - ${project.version} - - - - org.apache.httpcomponents - httpclient - - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/DorisSinkBufferEntry.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/DorisSinkBufferEntry.java deleted file mode 100644 index 97ce5c8f59..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/DorisSinkBufferEntry.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.doris.bean; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.UUID; - -/** doris sink buffer */ -public class DorisSinkBufferEntry implements Serializable { - - private static final long serialVersionUID = 1L; - - public List getBuffer() { - return buffer; - } - - private List buffer = new ArrayList<>(); - private int batchCount = 0; - private long batchSize = 0; - private String label; - private String database; - private String table; - private String labelPrefix; - - public DorisSinkBufferEntry(String database, String table, String labelPrefix) { - this.database = database; - this.table = table; - this.labelPrefix = labelPrefix; - label = createLabel(); - } - - public void setBuffer(ArrayList buffer) { - this.buffer = buffer; - } - - public int getBatchCount() { - return batchCount; - } - - public void setBatchCount(int batchCount) { - this.batchCount = batchCount; - } - - public long getBatchSize() { - return batchSize; - } - - public void setBatchSize(long batchSize) { - this.batchSize = batchSize; - } - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public String getDatabase() { - return database; - } - - public void setDatabase(String database) { - this.database = database; - } - - public String getTable() { - return table; - } - - public void setTable(String table) { - this.table = table; - } - - public String getLabelPrefix() { - return labelPrefix; - } - - public void setLabelPrefix(String labelPrefix) { - this.labelPrefix = labelPrefix; - } - - public void addToBuffer(byte[] bytes) { - incBatchCount(); - incBatchSize(bytes.length); - buffer.add(bytes); - } - - private void incBatchSize(long batchSize) { - this.batchSize += batchSize; - } - - private void incBatchCount() { - this.batchCount += 1; - } - - public synchronized void clear() { - buffer.clear(); - batchCount = 0; - batchSize = 0; - label = createLabel(); - } - - public String reGenerateLabel() { - return label = createLabel(); - } - - public String createLabel() { - return String.format( - "%s_%s_%s", - labelPrefix, System.currentTimeMillis(), UUID.randomUUID().toString().replaceAll("-", "")); - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/DorisSinkRowDataWithMeta.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/DorisSinkRowDataWithMeta.java deleted file mode 100644 index 5b4effa7d8..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/DorisSinkRowDataWithMeta.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.doris.bean; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -public class DorisSinkRowDataWithMeta implements Serializable { - - private static final long serialVersionUID = 1L; - - private String table; - private String database; - private final List dataRows = new ArrayList<>(); - - public DorisSinkRowDataWithMeta database(String database) { - this.database = database; - return this; - } - - public DorisSinkRowDataWithMeta table(String table) { - this.table = table; - return this; - } - - public String[] getDataRows() { - return dataRows.toArray(new String[] {}); - } - - public void addDataRow(String dataRow) { - this.dataRows.add(dataRow); - } - - public String getTable() { - return table; - } - - public String getDatabase() { - return database; - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/LoadStatusFailedException.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/LoadStatusFailedException.java deleted file mode 100644 index afb6b49abb..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/LoadStatusFailedException.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.doris.bean; - -import java.io.IOException; -import java.util.Map; - -public class LoadStatusFailedException extends IOException { - - private static final long serialVersionUID = 1L; - private final Map response; - private boolean reCreateLabel; - - public LoadStatusFailedException(String message, Map response) { - super(message); - this.response = response; - } - - public LoadStatusFailedException( - String message, Map response, boolean reCreateLabel) { - super(message); - this.response = response; - this.reCreateLabel = reCreateLabel; - } - - public Map getFailedResponse() { - return response; - } - - public boolean needReCreateLabel() { - return reCreateLabel; - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/RespContent.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/RespContent.java deleted file mode 100644 index 87f20dab2c..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/bean/RespContent.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.doris.bean; - -import org.apache.streampark.shaded.com.fasterxml.jackson.annotation.JsonProperty; - -import java.io.Serializable; - -/** Entity returned by streamLoad */ -public class RespContent implements Serializable { - - private static final long serialVersionUID = 1L; - - /** Imported transaction ID. The user may not perceive it. */ - @JsonProperty(value = "TxnId") - private int txnId; - /** Import Label. Specified by the user or automatically generated by the system. */ - @JsonProperty(value = "Label") - private String label; - /** - * Import complete status. "Success": Indicates that the import was successful. "Publish Timeout": - * This status also indicates that the import has been completed, but the data may be visible with - * a delay, and there is no need to retry. "Label Already Exists": The Label is duplicated, and - * the Label needs to be replaced. - */ - @JsonProperty(value = "Status") - private String status; - /** - * The status of the import job corresponding to the existing Label. This field will only be - * displayed when the Status is "Label Already Exists". The user can know the status of the import - * job corresponding to the existing Label through this status. "RUNNING" means that the job is - * still executing, and "FINISHED" means that the job is successful. - */ - @JsonProperty(value = "ExistingJobStatus") - private String existingJobStatus; - /** Import error information */ - @JsonProperty(value = "Message") - private String message; - /** Import the total number of processed rows */ - @JsonProperty(value = "NumberTotalRows") - private long numberTotalRows; - /** The number of rows successfully imported. */ - @JsonProperty(value = "NumberLoadedRows") - private long numberLoadedRows; - /** Number of rows with unqualified data quality */ - @JsonProperty(value = "NumberFilteredRows") - private int numberFilteredRows; - /** The number of rows filtered by the where condition */ - @JsonProperty(value = "NumberUnselectedRows") - private int numberUnselectedRows; - /** Number of bytes imported */ - @JsonProperty(value = "LoadBytes") - private long loadBytes; - /** Import completion time. The unit is milliseconds. */ - @JsonProperty(value = "LoadTimeMs") - private int loadTimeMs; - /** The time it takes to request Fe to start a transaction, in milliseconds */ - @JsonProperty(value = "BeginTxnTimeMs") - private int beginTxnTimeMs; - /** The time it takes to request Fe to obtain the import data execution plan, in milliseconds */ - @JsonProperty(value = "StreamLoadPutTimeMs") - private int streamLoadPutTimeMs; - /** The time spent reading data, in milliseconds */ - @JsonProperty(value = "ReadDataTimeMs") - private int readDataTimeMs; - /** Time to perform a data write operation takes milliseconds */ - @JsonProperty(value = "WriteDataTimeMs") - private int writeDataTimeMs; - /** The time taken to submit and publish the transaction request to Fe, in milliseconds */ - @JsonProperty(value = "CommitAndPublishTimeMs") - private int commitAndPublishTimeMs; - /** If there is a data quality problem, check the specific error line by visiting this URL */ - @JsonProperty(value = "ErrorURL") - private String errorURL; - - public static long getSerialVersionUID() { - return serialVersionUID; - } - - public int getTxnId() { - return txnId; - } - - public void setTxnId(int txnId) { - this.txnId = txnId; - } - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } - - public String getExistingJobStatus() { - return existingJobStatus; - } - - public void setExistingJobStatus(String existingJobStatus) { - this.existingJobStatus = existingJobStatus; - } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - - public long getNumberTotalRows() { - return numberTotalRows; - } - - public void setNumberTotalRows(long numberTotalRows) { - this.numberTotalRows = numberTotalRows; - } - - public long getNumberLoadedRows() { - return numberLoadedRows; - } - - public void setNumberLoadedRows(long numberLoadedRows) { - this.numberLoadedRows = numberLoadedRows; - } - - public int getNumberFilteredRows() { - return numberFilteredRows; - } - - public void setNumberFilteredRows(int numberFilteredRows) { - this.numberFilteredRows = numberFilteredRows; - } - - public int getNumberUnselectedRows() { - return numberUnselectedRows; - } - - public void setNumberUnselectedRows(int numberUnselectedRows) { - this.numberUnselectedRows = numberUnselectedRows; - } - - public long getLoadBytes() { - return loadBytes; - } - - public void setLoadBytes(long loadBytes) { - this.loadBytes = loadBytes; - } - - public int getLoadTimeMs() { - return loadTimeMs; - } - - public void setLoadTimeMs(int loadTimeMs) { - this.loadTimeMs = loadTimeMs; - } - - public int getBeginTxnTimeMs() { - return beginTxnTimeMs; - } - - public void setBeginTxnTimeMs(int beginTxnTimeMs) { - this.beginTxnTimeMs = beginTxnTimeMs; - } - - public int getStreamLoadPutTimeMs() { - return streamLoadPutTimeMs; - } - - public void setStreamLoadPutTimeMs(int streamLoadPutTimeMs) { - this.streamLoadPutTimeMs = streamLoadPutTimeMs; - } - - public int getReadDataTimeMs() { - return readDataTimeMs; - } - - public void setReadDataTimeMs(int readDataTimeMs) { - this.readDataTimeMs = readDataTimeMs; - } - - public int getWriteDataTimeMs() { - return writeDataTimeMs; - } - - public void setWriteDataTimeMs(int writeDataTimeMs) { - this.writeDataTimeMs = writeDataTimeMs; - } - - public int getCommitAndPublishTimeMs() { - return commitAndPublishTimeMs; - } - - public void setCommitAndPublishTimeMs(int commitAndPublishTimeMs) { - this.commitAndPublishTimeMs = commitAndPublishTimeMs; - } - - public String getErrorURL() { - return errorURL; - } - - public void setErrorURL(String errorURL) { - this.errorURL = errorURL; - } - - @Override - public String toString() { - return "RespContent{" - + "txnId=" - + txnId - + ", label='" - + label - + '\'' - + ", status='" - + status - + '\'' - + ", existingJobStatus='" - + existingJobStatus - + '\'' - + ", message='" - + message - + '\'' - + ", numberTotalRows=" - + numberTotalRows - + ", numberLoadedRows=" - + numberLoadedRows - + ", numberFilteredRows=" - + numberFilteredRows - + ", numberUnselectedRows=" - + numberUnselectedRows - + ", loadBytes=" - + loadBytes - + ", loadTimeMs=" - + loadTimeMs - + ", beginTxnTimeMs=" - + beginTxnTimeMs - + ", streamLoadPutTimeMs=" - + streamLoadPutTimeMs - + ", readDataTimeMs=" - + readDataTimeMs - + ", writeDataTimeMs=" - + writeDataTimeMs - + ", commitAndPublishTimeMs=" - + commitAndPublishTimeMs - + ", errorURL='" - + errorURL - + '\'' - + '}'; - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/internal/DorisSinkFunction.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/internal/DorisSinkFunction.java deleted file mode 100644 index c0ee6ee673..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/internal/DorisSinkFunction.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.doris.internal; - -import org.apache.streampark.common.enums.Semantic; -import org.apache.streampark.connector.doris.conf.DorisConfig; -import org.apache.streampark.flink.connector.doris.bean.DorisSinkBufferEntry; -import org.apache.streampark.flink.connector.doris.bean.DorisSinkRowDataWithMeta; -import org.apache.streampark.flink.core.scala.StreamingContext; - -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.state.ListState; -import org.apache.flink.api.common.state.ListStateDescriptor; -import org.apache.flink.api.common.typeinfo.TypeHint; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.metrics.Counter; -import org.apache.flink.runtime.state.FunctionInitializationContext; -import org.apache.flink.runtime.state.FunctionSnapshotContext; -import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.streaming.api.functions.sink.SinkFunction; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; -import java.util.Map; -import java.util.Properties; - -/** DorisSinkFunction */ -public class DorisSinkFunction extends RichSinkFunction implements CheckpointedFunction { - - private static final Logger LOGGER = LoggerFactory.getLogger(DorisSinkFunction.class); - private final Properties properties; - private final DorisSinkWriter dorisSinkWriter; - private final DorisConfig dorisConfig; - // state only works with `EXACTLY_ONCE` - private transient ListState> checkpointedState; - private transient Counter totalInvokeRowsTime; - private transient Counter totalInvokeRows; - private static final String COUNTER_INVOKE_ROWS_COST_TIME = "totalInvokeRowsTimeNs"; - private static final String COUNTER_INVOKE_ROWS = "totalInvokeRows"; - - public DorisSinkFunction(StreamingContext context) { - this.properties = context.parameter().getProperties(); - this.dorisConfig = new DorisConfig(properties); - this.dorisSinkWriter = new DorisSinkWriter(dorisConfig); - } - - @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - dorisSinkWriter.setRuntimeContext(getRuntimeContext()); - totalInvokeRows = getRuntimeContext().getMetricGroup().counter(COUNTER_INVOKE_ROWS); - totalInvokeRowsTime = - getRuntimeContext().getMetricGroup().counter(COUNTER_INVOKE_ROWS_COST_TIME); - dorisSinkWriter.startScheduler(); - dorisSinkWriter.startAsyncFlushing(); - } - - @Override - public void invoke(T value, SinkFunction.Context context) throws Exception { - long start = System.nanoTime(); - if (value instanceof DorisSinkRowDataWithMeta) { - DorisSinkRowDataWithMeta data = (DorisSinkRowDataWithMeta) value; - if (StringUtils.isBlank(data.getDatabase()) - || StringUtils.isBlank(data.getTable()) - || null == data.getDataRows()) { - LOGGER.warn( - String.format( - " row data not fulfilled. {database: %s, table: %s, dataRows: %s}", - data.getDatabase(), data.getTable(), Arrays.toString(data.getDataRows()))); - return; - } - dorisSinkWriter.writeRecords(data.getDatabase(), data.getTable(), data.getDataRows()); - } else { - if (StringUtils.isBlank(dorisConfig.database()) || StringUtils.isBlank(dorisConfig.table())) { - throw new RuntimeException( - " database|table is empty ,please check your config or create DorisSinkRowDataWithMeta instance"); - } - dorisSinkWriter.writeRecords(dorisConfig.database(), dorisConfig.table(), (String) value); - } - // raw data sink - totalInvokeRows.inc(1); - totalInvokeRowsTime.inc(System.nanoTime() - start); - } - - @Override - public void close() throws Exception { - super.close(); - dorisSinkWriter.close(); - } - - @Override - public void snapshotState(FunctionSnapshotContext context) throws Exception { - if (Semantic.EXACTLY_ONCE.equals(Semantic.of(dorisConfig.semantic()))) { - // save state - checkpointedState.add(dorisSinkWriter.getBufferedBatchMap()); - flushPreviousState(); - } - } - - @Override - public void initializeState(FunctionInitializationContext context) throws Exception { - if (Semantic.EXACTLY_ONCE.equals(Semantic.of(dorisConfig.semantic()))) { - ListStateDescriptor> descriptor = - new ListStateDescriptor<>( - "buffered-rows", - TypeInformation.of(new TypeHint>() {})); - checkpointedState = context.getOperatorStateStore().getListState(descriptor); - } - } - - private void flushPreviousState() throws Exception { - // flush the batch saved at the previous checkpoint - for (Map state : checkpointedState.get()) { - dorisSinkWriter.setBufferedBatchMap(state); - dorisSinkWriter.flush(null, true); - } - checkpointedState.clear(); - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/internal/DorisSinkWriter.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/internal/DorisSinkWriter.java deleted file mode 100644 index 160b320478..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/internal/DorisSinkWriter.java +++ /dev/null @@ -1,336 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.doris.internal; - -import org.apache.streampark.common.enums.Semantic; -import org.apache.streampark.common.util.ThreadUtils; -import org.apache.streampark.connector.doris.conf.DorisConfig; -import org.apache.streampark.flink.connector.doris.bean.DorisSinkBufferEntry; -import org.apache.streampark.flink.connector.doris.bean.LoadStatusFailedException; - -import org.apache.flink.api.common.functions.RuntimeContext; -import org.apache.flink.metrics.Counter; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.Serializable; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -public class DorisSinkWriter implements Serializable { - private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(DorisSinkWriter.class); - - private final DorisConfig dorisConfig; - private final DorisStreamLoader streamLoader; - - private ScheduledExecutorService schedule; - private ScheduledFuture scheduledFuture; - - private transient Counter totalFlushBytes; - private transient Counter totalFlushRows; - private transient Counter totalFlushTime; - private transient Counter totalFlushTimeWithoutRetries; - private transient Counter totalFlushSucceededTimes; - private transient Counter totalFlushFailedTimes; - - private final Map bufferMap = new ConcurrentHashMap<>(); - private final Long timeout = 3000L; - private volatile boolean closed = false; - private volatile boolean flushThreadAlive = false; - private volatile Throwable flushException; - - final LinkedBlockingDeque flushQueue = new LinkedBlockingDeque<>(10); - - private static final String COUNTER_TOTAL_FLUSH_BYTES = "totalFlushBytes"; - private static final String COUNTER_TOTAL_FLUSH_ROWS = "totalFlushRows"; - private static final String COUNTER_TOTAL_FLUSH_COST_TIME_WITHOUT_RETRIES = - "totalFlushTimeNsWithoutRetries"; - private static final String COUNTER_TOTAL_FLUSH_COST_TIME = "totalFlushTimeNs"; - private static final String COUNTER_TOTAL_FLUSH_SUCCEEDED_TIMES = "totalFlushSucceededTimes"; - private static final String COUNTER_TOTAL_FLUSH_FAILED_TIMES = "totalFlushFailedTimes"; - - private final Semantic semantic; - - public DorisSinkWriter(DorisConfig dorisConfig) { - this.streamLoader = new DorisStreamLoader(dorisConfig); - this.dorisConfig = dorisConfig; - semantic = Semantic.of(dorisConfig.semantic()); - } - - public void setRuntimeContext(RuntimeContext runtimeCtx) { - totalFlushBytes = runtimeCtx.getMetricGroup().counter(COUNTER_TOTAL_FLUSH_BYTES); - totalFlushRows = runtimeCtx.getMetricGroup().counter(COUNTER_TOTAL_FLUSH_ROWS); - totalFlushTime = runtimeCtx.getMetricGroup().counter(COUNTER_TOTAL_FLUSH_COST_TIME); - totalFlushTimeWithoutRetries = - runtimeCtx.getMetricGroup().counter(COUNTER_TOTAL_FLUSH_COST_TIME_WITHOUT_RETRIES); - totalFlushSucceededTimes = - runtimeCtx.getMetricGroup().counter(COUNTER_TOTAL_FLUSH_SUCCEEDED_TIMES); - totalFlushFailedTimes = runtimeCtx.getMetricGroup().counter(COUNTER_TOTAL_FLUSH_FAILED_TIMES); - } - - public void startAsyncFlushing() { - final Thread flushThread = - new Thread( - () -> { - while (true) { - try { - if (!asyncFlush()) { - LOG.info("doris flush thread is about to exit."); - flushThreadAlive = false; - break; - } - } catch (Exception e) { - flushException = e; - } - } - }); - flushThread.setUncaughtExceptionHandler( - (t, e) -> { - LOG.error("dorics flush thread uncaught exception occurred:" + e.getMessage(), e); - flushException = e; - flushThreadAlive = false; - }); - flushThread.setName("doris-flush"); - flushThread.setDaemon(true); - flushThread.start(); - flushThreadAlive = true; - } - - public void startScheduler() { - if (semantic.equals(Semantic.EXACTLY_ONCE)) { - return; - } - stopSchedule(); - this.schedule = - Executors.newScheduledThreadPool(1, ThreadUtils.threadFactory("doris-interval-sink")); - this.scheduledFuture = - this.schedule.schedule( - () -> { - synchronized (DorisSinkWriter.this) { - if (!closed) { - try { - LOG.info("doris interval sinking trigger"); - if (bufferMap.isEmpty()) { - startScheduler(); - } - flush(null, false); - } catch (Exception e) { - flushException = e; - } - } - } - }, - dorisConfig.flushInterval(), - TimeUnit.MILLISECONDS); - } - - private void stopSchedule() { - if (this.scheduledFuture != null) { - scheduledFuture.cancel(false); - this.schedule.shutdown(); - } - } - - public final synchronized void writeRecords(String database, String table, String... records) - throws IOException { - checkFlushException(); - try { - if (records.length == 0) { - return; - } - final String bufferKey = String.format("%s.%s", database, table); - final DorisSinkBufferEntry bufferEntity = - bufferMap.computeIfAbsent( - bufferKey, k -> new DorisSinkBufferEntry(database, table, dorisConfig.labelPrefix())); - for (String record : records) { - byte[] bts = record.getBytes(StandardCharsets.UTF_8); - bufferEntity.addToBuffer(bts); - } - if (Semantic.EXACTLY_ONCE.equals(semantic)) { - return; - } - if (bufferEntity.getBatchCount() >= dorisConfig.sinkMaxRow() - || bufferEntity.getBatchSize() >= dorisConfig.sinkMaxBytes()) { - LOG.info( - String.format( - "doris buffer Sinking triggered: db: [%s] table: [%s] rows[%d] label[%s].", - database, table, bufferEntity.getBatchCount(), bufferEntity.getLabel())); - flush(bufferKey, false); - } - } catch (Exception e) { - throw new IOException("Writing records to doris failed.", e); - } - } - - public synchronized void flush(String bufferKey, boolean waitUntilDone) throws Exception { - if (bufferMap.isEmpty()) { - flushInternal(null, waitUntilDone); - return; - } - if (null == bufferKey) { - for (String key : bufferMap.keySet()) { - flushInternal(key, waitUntilDone); - } - return; - } - flushInternal(bufferKey, waitUntilDone); - } - - private synchronized void flushInternal(String bufferKey, boolean waitUntilDone) - throws Exception { - checkFlushException(); - if (null == bufferKey || bufferMap.isEmpty() || !bufferMap.containsKey(bufferKey)) { - if (waitUntilDone) { - waitAsyncFlushingDone(); - } - return; - } - offer(bufferMap.get(bufferKey)); - bufferMap.remove(bufferKey); - if (waitUntilDone) { - // wait the last flush - waitAsyncFlushingDone(); - } - } - - private void waitAsyncFlushingDone() throws InterruptedException { - // wait for previous flushings - offer(new DorisSinkBufferEntry(null, null, null)); - checkFlushException(); - } - - private void offer(DorisSinkBufferEntry bufferEntity) throws InterruptedException { - if (!flushThreadAlive) { - throw new RuntimeException( - "Flush thread already exit or not start ,please exec startAsyncFlushing() , ignore offer request for label[%s] "); - } - if (!flushQueue.offer(bufferEntity, dorisConfig.sinkOfferTimeout(), TimeUnit.MILLISECONDS)) { - throw new RuntimeException( - "Timeout while offering data to flushQueue, exceed " - + dorisConfig.sinkOfferTimeout() - + " ms, see " - + dorisConfig.sinkOption().sinkOfferTimeout().key()); - } - } - - private boolean asyncFlush() throws Exception { - final DorisSinkBufferEntry flushData = flushQueue.poll(timeout, TimeUnit.MILLISECONDS); - if (flushData == null || flushData.getBatchCount() == 0) { - return true; - } - stopSchedule(); - LOG.info( - String.format( - "Async stream load: db[%s] table[%s] rows[%d] bytes[%d] label[%s].", - flushData.getDatabase(), - flushData.getTable(), - flushData.getBatchCount(), - flushData.getBatchSize(), - flushData.getLabel())); - long startWithRetries = System.nanoTime(); - for (int i = 0; i < dorisConfig.sinkMaxRetries(); i++) { - try { - long start = System.nanoTime(); - streamLoader.doStreamLoad(flushData); - LOG.info(String.format("Async stream load finished: label[%s].", flushData.getLabel())); - if (null != totalFlushBytes) { - totalFlushBytes.inc(flushData.getBatchSize()); - totalFlushRows.inc(flushData.getBatchCount()); - totalFlushTime.inc(System.nanoTime() - startWithRetries); - totalFlushTimeWithoutRetries.inc(System.nanoTime() - start); - totalFlushSucceededTimes.inc(); - } - startScheduler(); - break; - } catch (Exception e) { - if (totalFlushFailedTimes != null) { - totalFlushFailedTimes.inc(); - } - LOG.warn("Failed to flush batch data to doris, retry times = {}", i, e); - if (i >= dorisConfig.sinkMaxRetries()) { - throw e; - } - if (e instanceof LoadStatusFailedException - && ((LoadStatusFailedException) e).needReCreateLabel()) { - String oldLabel = flushData.getLabel(); - flushData.reGenerateLabel(); - LOG.warn( - String.format( - "Batch label changed from [%s] to [%s]", oldLabel, flushData.getLabel())); - } - } - try { - Thread.sleep(1000L * (i + 1)); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - throw new IOException("Unable to flush, interrupted while doing another attempt", ex); - } - } - return true; - } - - public synchronized void close() throws Exception { - if (!closed) { - closed = true; - LOG.info("Sink is about to close."); - flush(null, false); - this.bufferMap.clear(); - if (scheduledFuture != null) { - scheduledFuture.cancel(false); - schedule.shutdown(); - } - } - checkFlushException(); - } - - private void checkFlushException() { - if (flushException != null) { - StackTraceElement[] stack = Thread.currentThread().getStackTrace(); - for (int i = 0; i < stack.length; i++) { - LOG.info( - stack[i].getClassName() - + "." - + stack[i].getMethodName() - + " line:" - + stack[i].getLineNumber()); - } - throw new RuntimeException("Writing records to doris failed.", flushException); - } - } - - public Map getBufferedBatchMap() { - return new HashMap<>(bufferMap); - } - - public void setBufferedBatchMap(Map newBufferMap) { - if (Semantic.EXACTLY_ONCE.equals(semantic)) { - bufferMap.clear(); - bufferMap.putAll(newBufferMap); - } - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/internal/DorisStreamLoader.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/internal/DorisStreamLoader.java deleted file mode 100644 index e8a850bd36..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/internal/DorisStreamLoader.java +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.doris.internal; - -import org.apache.streampark.connector.doris.conf.DorisConfig; -import org.apache.streampark.flink.connector.doris.bean.DorisSinkBufferEntry; -import org.apache.streampark.flink.connector.doris.bean.LoadStatusFailedException; -import org.apache.streampark.flink.connector.doris.bean.RespContent; -import org.apache.streampark.flink.connector.doris.util.DorisDelimiterParser; - -import org.apache.streampark.shaded.com.fasterxml.jackson.databind.ObjectMapper; - -import org.apache.commons.codec.binary.Base64; -import org.apache.http.HttpHeaders; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.DefaultRedirectStrategy; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.Serializable; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -public class DorisStreamLoader implements Serializable { - - private static final long serialVersionUID = 1L; - - private static final String LOAD_URL_PATTERN = "%s/api/%s/%s/_stream_load?"; - private static final String GET_LOAD_STATUS_URL = "%s/api/%s/get_load_state?label=%s"; - private static final Logger LOG = LoggerFactory.getLogger(DorisStreamLoader.class); - - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private final DorisConfig dorisConfig; - private static final String RESULT_FAILED = "Fail"; - private static final String RESULT_LABEL_EXISTED = "Label Already Exists"; - private static final String LAEBL_STATE_VISIBLE = "VISIBLE"; - private static final String LAEBL_STATE_COMMITTED = "COMMITTED"; - private static final String RESULT_LABEL_PREPARE = "PREPARE"; - private static final String RESULT_LABEL_ABORTED = "ABORTED"; - private static final String RESULT_LABEL_UNKNOWN = "UNKNOWN"; - - public DorisStreamLoader(DorisConfig dorisConfig) { - this.dorisConfig = dorisConfig; - } - - public RespContent doStreamLoad(DorisSinkBufferEntry bufferEntity) throws IOException { - String host = getWorkerHost(); - if (null == host) { - throw new IOException("None of the hosts in `load_url` could be connected."); - } - - LOG.info(String.format("Start to join batch data: label[%s].", bufferEntity.getLabel())); - String loadUrl = - String.format(LOAD_URL_PATTERN, host, bufferEntity.getDatabase(), bufferEntity.getTable()); - LoadResponse loadResponse = - doHttpPut( - loadUrl, - bufferEntity.getLabel(), - joinRows(bufferEntity.getBuffer(), (int) bufferEntity.getBatchSize())); - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("Stream Load response: \n%s\n", loadResponse.respContent)); - } - if (loadResponse.status != 200) { - throw new RuntimeException("stream load error: " + loadResponse.respContent); - } else { - try { - RespContent respContent = - OBJECT_MAPPER.readValue(loadResponse.respContent, RespContent.class); - if (RESULT_FAILED.equals(respContent.getStatus())) { - String errMsg = - String.format( - "stream load error: %s, see more in %s", - respContent.getMessage(), respContent.getErrorURL()); - throw new RuntimeException(errMsg); - } else if (RESULT_LABEL_EXISTED.equals(respContent.getStatus())) { - LOG.error(String.format("Stream Load response: \n%s\n", loadResponse.respContent)); - checkLableState(host, bufferEntity.getDatabase(), bufferEntity.getLabel()); - } - return respContent; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - - private void checkLableState(String host, String database, String label) throws IOException { - int tries = 0; - while (tries < 10) { - try { - TimeUnit.SECONDS.sleep(Math.min(++tries, 5)); - } catch (InterruptedException e) { - return; - } - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - HttpGet httpGet = new HttpGet(String.format(GET_LOAD_STATUS_URL, host, database, label)); - httpGet.setHeader( - HttpHeaders.AUTHORIZATION, - getBasicAuthHeader(dorisConfig.user(), dorisConfig.password())); - httpGet.setHeader("Connection", "close"); - try (CloseableHttpResponse response = httpclient.execute(httpGet)) { - final int statusCode = response.getStatusLine().getStatusCode(); - String loadResult = "{}"; - if (response.getEntity() != null) { - loadResult = EntityUtils.toString(response.getEntity()); - } - if (statusCode != 200) { - throw new LoadStatusFailedException( - String.format( - "Failed to flush data to doris, Error " - + "could not get the final state of label[%s].%n", - label), - null); - } - Map result = OBJECT_MAPPER.readValue(loadResult, HashMap.class); - String labelState = (String) result.get("state"); - if (null == labelState) { - throw new LoadStatusFailedException( - String.format( - "Failed to flush data to doris, Error " - + "could not get the final state of label[%s]. response[%s]%n", - label, loadResult), - null); - } - LOG.info(String.format("Checking label[%s] state[%s]%n", label, labelState)); - switch (labelState) { - case LAEBL_STATE_VISIBLE: - return; - case LAEBL_STATE_COMMITTED: - return; - case RESULT_LABEL_PREPARE: - continue; - case RESULT_LABEL_ABORTED: - throw new LoadStatusFailedException( - String.format( - "Failed to flush data to doris, Error " + "label[%s] state[%s]%n", - label, labelState), - null, - true); - case RESULT_LABEL_UNKNOWN: - default: - throw new LoadStatusFailedException( - String.format( - "Failed to flush data to doris, Error " + "label[%s] state[%s]%n", - label, labelState), - null); - } - } - } - } - } - - private String getBasicAuthHeader(String username, String password) { - String auth = username + ":" + password; - byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); - return "Basic " + new String(encodedAuth); - } - - private LoadResponse doHttpPut(String loadUrl, String label, byte[] data) throws IOException { - LOG.info( - String.format( - "Executing stream load to: '%s', size: '%s', thread: %d", - loadUrl, data.length, Thread.currentThread().getId())); - final HttpClientBuilder httpClientBuilder = - HttpClients.custom() - .setRedirectStrategy( - new DefaultRedirectStrategy() { - @Override - protected boolean isRedirectable(String method) { - return true; - } - }); - try (CloseableHttpClient httpclient = httpClientBuilder.build()) { - final HttpPut put = new HttpPut(loadUrl); - final Properties properties = dorisConfig.loadProperties(); - properties.forEach((k, v) -> put.setHeader(k.toString(), v.toString())); - if (properties.containsKey("columns")) { - put.setHeader("timeout", String.valueOf(dorisConfig.timeout())); - } - put.setHeader(HttpHeaders.EXPECT, "100-continue"); - put.setHeader( - HttpHeaders.AUTHORIZATION, - getBasicAuthHeader(dorisConfig.user(), dorisConfig.password())); - put.setHeader("label", label); - put.setEntity(new ByteArrayEntity(data)); - try (CloseableHttpResponse response = httpclient.execute(put)) { - final int statusCode = response.getStatusLine().getStatusCode(); - final String reasonPhrase = response.getStatusLine().getReasonPhrase(); - String loadResult = ""; - if (response.getEntity() != null) { - loadResult = EntityUtils.toString(response.getEntity()); - } - return new LoadResponse(statusCode, reasonPhrase, loadResult); - } - } - } - - private byte[] joinRows(List rows, int totalBytes) { - if (DorisConfig.CSV().equalsIgnoreCase(dorisConfig.loadFormat())) { - byte[] lineDelimiter = - DorisDelimiterParser.parse(dorisConfig.rowDelimiter()).getBytes(StandardCharsets.UTF_8); - ByteBuffer bos = ByteBuffer.allocate(totalBytes + rows.size() * lineDelimiter.length); - for (byte[] row : rows) { - bos.put(row); - bos.put(lineDelimiter); - } - return bos.array(); - } - - if (DorisConfig.JSON().equalsIgnoreCase(dorisConfig.loadFormat())) { - ByteBuffer bos = ByteBuffer.allocate(totalBytes + (rows.isEmpty() ? 2 : rows.size() + 1)); - bos.put("[".getBytes(StandardCharsets.UTF_8)); - byte[] jsonDelimiter = ",".getBytes(StandardCharsets.UTF_8); - boolean isFirstElement = true; - for (byte[] row : rows) { - if (!isFirstElement) { - bos.put(jsonDelimiter); - } - bos.put(row); - isFirstElement = false; - } - bos.put("]".getBytes(StandardCharsets.UTF_8)); - return bos.array(); - } - throw new RuntimeException( - "Failed to join rows data, unsupported `format` from stream load properties:"); - } - - private String getWorkerHost() { - for (int pos = 0; pos < dorisConfig.getLoadUrlSize(); pos++) { - String host = dorisConfig.getHostUrl(); - if (tryHttpConnection(host)) { - return host; - } - } - return null; - } - - private boolean tryHttpConnection(String host) { - try { - URL url = new URL(host); - HttpURLConnection co = (HttpURLConnection) url.openConnection(); - co.setConnectTimeout(dorisConfig.timeout()); - co.connect(); - co.disconnect(); - return true; - } catch (Exception e1) { - LOG.warn("Failed to connect to address:{}", host, e1); - return false; - } - } - - public static class LoadResponse { - public int status; - public String respMsg; - public String respContent; - - public LoadResponse(int status, String respMsg, String respContent) { - this.status = status; - this.respMsg = respMsg; - this.respContent = respContent; - } - - @Override - public String toString() { - return "status: " + status + ", resp msg: " + respMsg + ", resp content: " + respContent; - } - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/sink/DorisSink.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/sink/DorisSink.java deleted file mode 100644 index e9e6505378..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/sink/DorisSink.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.doris.sink; - -import org.apache.streampark.flink.connector.doris.internal.DorisSinkFunction; -import org.apache.streampark.flink.core.scala.StreamingContext; - -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamSink; - -public class DorisSink { - - private final StreamingContext context; - - public DorisSink(StreamingContext context) { - this.context = context; - } - - /** - * java stream - * - * @param source - * @return - */ - public DataStreamSink sink(DataStream source) { - DorisSinkFunction sinkFunction = new DorisSinkFunction<>(context); - return source.addSink(sinkFunction); - } - - /** - * scala stream - * - * @param source - * @return - */ - public DataStreamSink sink(org.apache.flink.streaming.api.scala.DataStream source) { - DorisSinkFunction sinkFunction = new DorisSinkFunction<>(context); - return source.addSink(sinkFunction); - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/util/DorisDelimiterParser.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/util/DorisDelimiterParser.java deleted file mode 100644 index 886c85246f..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/java/org/apache/streampark/flink/connector/doris/util/DorisDelimiterParser.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.doris.util; - -import java.io.StringWriter; - -public class DorisDelimiterParser { - - private static final String HEX_STRING = "0123456789ABCDEF"; - - public static String parse(String sp) throws RuntimeException { - if (sp == null || sp.isBlank()) { - throw new RuntimeException("Delimiter can't be empty"); - } - if (!sp.toUpperCase().startsWith("\\X")) { - return sp; - } - String hexStr = sp.substring(2); - // check hex str - if (hexStr.isEmpty()) { - throw new RuntimeException("Failed to parse delimiter: `Hex str is empty`"); - } - if (hexStr.length() % 2 != 0) { - throw new RuntimeException("Failed to parse delimiter: `Hex str length error`"); - } - for (char hexChar : hexStr.toUpperCase().toCharArray()) { - if (HEX_STRING.indexOf(hexChar) == -1) { - throw new RuntimeException("Failed to parse delimiter: `Hex str format error`"); - } - } - // transform to separator - StringWriter writer = new StringWriter(); - for (byte b : hexStrToBytes(hexStr)) { - writer.append((char) b); - } - return writer.toString(); - } - - private static byte[] hexStrToBytes(String hexStr) { - String upperHexStr = hexStr.toUpperCase(); - int length = upperHexStr.length() / 2; - char[] hexChars = upperHexStr.toCharArray(); - byte[] bytes = new byte[length]; - for (int i = 0; i < length; i++) { - int pos = i * 2; - bytes[i] = (byte) (charToByte(hexChars[pos]) << 4 | charToByte(hexChars[pos + 1])); - } - return bytes; - } - - private static byte charToByte(char c) { - return (byte) HEX_STRING.indexOf(c); - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisConfig.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisConfig.scala deleted file mode 100644 index 2046a68177..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisConfig.scala +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.streampark.connector.doris.conf - -import org.apache.streampark.common.Constant -import org.apache.streampark.common.conf.ConfigKeys - -import java.util.Properties -import org.apache.streampark.common.util.Implicits._ - -object DorisConfig { - - val CSV = "csv" - - val JSON = "json" - - def apply(properties: Properties = new Properties): DorisConfig = new DorisConfig(properties) - -} - -class DorisConfig(parameters: Properties) { - - val sinkOption: DorisSinkConfigOption = DorisSinkConfigOption(properties = parameters) - - val user: String = sinkOption.user.get() - - val password: String = sinkOption.password.get() - - val loadUrl: List[String] = sinkOption.loadUrl.get() - - val loadFormat: String = sinkOption.loadFormat.get() - - val rowDelimiter: String = sinkOption.rowDelimiter.get() - - val timeout: Int = sinkOption.connectTimeout.get() - - val sinkMaxRow: Int = sinkOption.maxRow.get() - - val sinkMaxBytes: Int = sinkOption.maxBytes.get() - - val sinkMaxRetries: Int = sinkOption.maxRetries.get() - - val flushInterval: Long = sinkOption.flushInterval.get() - - val sinkOfferTimeout: Long = sinkOption.sinkOfferTimeout.get() - - val labelPrefix: String = sinkOption.labelPrefix.get() - - val semantic: String = sinkOption.semantic.get() - - val database: String = sinkOption.database.get() - - val table: String = sinkOption.table.get() - - def loadProperties: Properties = { - sinkOption.getInternalProperties() - } - - def getLoadUrlSize(): Int = { - loadUrl.size - } - - var currentHostId: Long = 0 - - def getHostUrl: String = { - currentHostId += 1 - loadUrl.get((currentHostId % loadUrl.size).toInt) - } - - override def toString: String = { - s""" - |{ doris user: $user, password: ${Constant.DEFAULT_DATAMASK_STRING}, hosts: ${loadUrl.mkString(",")} } - |""".stripMargin - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisSinkConfigOption.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisSinkConfigOption.scala deleted file mode 100644 index 91149d6a64..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-doris/src/main/scala/org/apache/streampark/connector/doris/conf/DorisSinkConfigOption.scala +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.connector.doris.conf - -import org.apache.streampark.common.conf.ConfigOption -import org.apache.streampark.common.util.ConfigUtils - -import java.util.Properties - -import org.apache.streampark.common.util.Implicits._ - -object DorisSinkConfigOption { - val DORIS_SINK_PREFIX = "doris.sink" - - def apply( - prefixStr: String = DORIS_SINK_PREFIX, - properties: Properties = new Properties): DorisSinkConfigOption = - new DorisSinkConfigOption(prefixStr, properties) -} - -class DorisSinkConfigOption(prefixStr: String, properties: Properties) extends Serializable { - - implicit val (prefix, prop) = (prefixStr, properties) - - val SIGN_COMMA = "," - - val loadUrl: ConfigOption[List[String]] = ConfigOption[List[String]]( - key = "load_url", - required = true, - defaultValue = List(), - classType = classOf[List[String]], - handle = k => { - properties - .getProperty(k) - .split(SIGN_COMMA) - .filter(_.nonEmpty) - .map(_.replaceAll("\\s+", "").replaceFirst("^http://|^", Constant.HTTP_SCHEMA)) - .toList - } - ) - - val user: ConfigOption[String] = - ConfigOption(key = "user", required = true, classType = classOf[String]) - - val loadFormat: ConfigOption[String] = ConfigOption( - key = "loadFormat", - required = false, - defaultValue = "csv", - classType = classOf[String]) - - val password: ConfigOption[String] = - ConfigOption(key = "password", required = true, classType = classOf[String]) - - val database: ConfigOption[String] = - ConfigOption(key = "database", required = false, defaultValue = "", classType = classOf[String]) - - val table: ConfigOption[String] = - ConfigOption(key = "table", required = false, defaultValue = "", classType = classOf[String]) - - val sinkOfferTimeout: ConfigOption[Long] = ConfigOption( - key = "sinkOfferTimeout", - required = false, - defaultValue = 3000L, - classType = classOf[Long], - handle = k => { - properties.remove(k).toString.toLong - }) - - val rowDelimiter: ConfigOption[String] = ConfigOption( - key = "properties.row_delimiter", - required = false, - defaultValue = "\n", - classType = classOf[String]) - - val flushInterval: ConfigOption[Long] = ConfigOption( - key = "flushInterval", - required = false, - defaultValue = 300000L, - classType = classOf[Long], - handle = k => { - properties.remove(k).toString.toLong - }) - - val connectTimeout: ConfigOption[Int] = ConfigOption( - key = "connectTimeout", - required = false, - defaultValue = 5000, - classType = classOf[Long]) - - val maxRequestRetry: ConfigOption[Int] = ConfigOption( - key = "maxRequestRetry", - required = false, - defaultValue = 1, - classType = classOf[Long]) - - val maxConnections: ConfigOption[Int] = ConfigOption( - key = "maxConnections", - required = false, - defaultValue = -1, - classType = classOf[Int]) - - val maxRow: ConfigOption[Int] = - ConfigOption(key = "maxRow", required = false, defaultValue = 100000, classType = classOf[Int]) - - val maxBytes: ConfigOption[Int] = ConfigOption( - key = "maxRow", - required = false, - defaultValue = 94371840, - classType = classOf[Int]) - - val maxRetries: ConfigOption[Int] = - ConfigOption(key = "maxRetries", required = false, defaultValue = 1, classType = classOf[Int]) - - val labelPrefix: ConfigOption[String] = ConfigOption( - key = "labelPrefix", - required = false, - defaultValue = "doris", - classType = classOf[String]) - - val semantic: ConfigOption[String] = ConfigOption( - key = "semantic", - required = false, - defaultValue = "AT_LEAST_ONCE", - classType = classOf[String]) - - def getInternalConfig(): Properties = { - ConfigUtils.getConf(prop, prefix)("") - } - def getInternalProperties(): Properties = { - ConfigUtils.getConf(prop, prefix)(".properties") - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/pom.xml deleted file mode 100644 index 0c1767b0cd..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/pom.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink-connector - 2.2.0-SNAPSHOT - - - streampark-flink-connector-elasticsearch - StreamPark : Flink Connector Elasticsearch - pom - - - streampark-flink-connector-elasticsearch5 - streampark-flink-connector-elasticsearch6 - streampark-flink-connector-elasticsearch7 - - - - - org.apache.streampark - streampark-flink-connector-base_${scala.binary.version} - ${project.version} - - - - org.apache.httpcomponents - httpclient - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/pom.xml deleted file mode 100644 index 23a84158a3..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/pom.xml +++ /dev/null @@ -1,94 +0,0 @@ - - - - 4.0.0 - - - org.apache.streampark - streampark-flink-connector-elasticsearch - 2.2.0-SNAPSHOT - - - streampark-flink-connector-elasticsearch5_${scala.binary.version} - StreamPark : Flink Connector Elasticsearch5 - - - 8 - 8 - 1.14.0 - - - - - - org.apache.flink - flink-connector-elasticsearch5_${scala.binary.version} - ${flink.version} - - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/conf/ESConfig.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/conf/ESConfig.scala deleted file mode 100644 index beaabcc546..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/conf/ESConfig.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch5.conf - -import java.net.InetSocketAddress -import java.util.Properties - -class ESConfig(parameters: Properties) extends Serializable { - - val sinkOption: ESSinkConfigOption = ESSinkConfigOption(properties = parameters) - - val disableFlushOnCheckpoint: Boolean = sinkOption.disableFlushOnCheckpoint.get() - - val host: List[InetSocketAddress] = sinkOption.host.get().toList - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/conf/ESSinkConfigOption.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/conf/ESSinkConfigOption.scala deleted file mode 100644 index b64645cea6..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/conf/ESSinkConfigOption.scala +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch5.conf - -import org.apache.streampark.common.conf.ConfigOption -import org.apache.streampark.common.util.ConfigUtils -import org.apache.streampark.common.util.Implicits._ - -import java.net.InetSocketAddress -import java.util.Properties - -object ESSinkConfigOption { - val ES_SINK_PREFIX = "es.sink" - - /** - * @param properties - * @return - */ - def apply( - prefixStr: String = ES_SINK_PREFIX, - properties: Properties = new Properties): ESSinkConfigOption = - new ESSinkConfigOption(prefixStr, properties) - -} - -class ESSinkConfigOption(prefixStr: String, properties: Properties) extends Serializable { - - implicit val (prefix, prop) = (prefixStr, properties) - - val SIGN_COMMA = "," - - val SIGN_COLON = ":" - - val disableFlushOnCheckpoint: ConfigOption[Boolean] = ConfigOption( - key = "es.disableFlushOnCheckpoint", - required = false, - classType = classOf[Boolean], - defaultValue = false) - - val host: ConfigOption[Array[InetSocketAddress]] = ConfigOption( - key = "host", - required = true, - classType = classOf[Array[InetSocketAddress]], - handle = key => - properties - .getProperty(key) - .split(SIGN_COMMA) - .map( - x => { - x.split(SIGN_COLON) match { - case Array(host, port) => new InetSocketAddress(host, port.toInt) - } - }) - ) - - def getInternalConfig(): JavaMap[String, String] = { - ConfigUtils.getConf(prop, prefix)(alias = "") - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/internal/ESSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/internal/ESSinkFunction.scala deleted file mode 100644 index 4b23a9cd6c..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/internal/ESSinkFunction.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch5.internal - -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.enums.ApiType.ApiType -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.function.TransformFunction - -import org.apache.flink.api.common.functions.RuntimeContext -import org.apache.flink.streaming.connectors.elasticsearch.{ElasticsearchSinkFunction, RequestIndexer} -import org.elasticsearch.action.ActionRequest -import org.elasticsearch.action.delete.DeleteRequest -import org.elasticsearch.action.index.IndexRequest -import org.elasticsearch.action.update.UpdateRequest - -class ESSinkFunction[T](apiType: ApiType = ApiType.scala) - extends ElasticsearchSinkFunction[T] - with Logger { - - private[this] var scalaFunc: T => ActionRequest = _ - private[this] var javaFunc: TransformFunction[T, ActionRequest] = _ - - // for Scala - def this(scalaFunc: T => ActionRequest) = { - this(ApiType.scala) - this.scalaFunc = scalaFunc - } - - // for JAVA - def this(javaFunc: TransformFunction[T, ActionRequest]) = { - this(ApiType.java) - this.javaFunc = javaFunc - } - - def createIndexRequest(element: T): ActionRequest = apiType match { - case ApiType.java => javaFunc.transform(element) - case ApiType.scala => scalaFunc(element) - } - - override def process( - element: T, - runtimeContext: RuntimeContext, - requestIndexer: RequestIndexer): Unit = { - val request: ActionRequest = createIndexRequest(element) - request match { - case indexRequest if indexRequest.isInstanceOf[IndexRequest] => - requestIndexer.add(indexRequest.asInstanceOf[IndexRequest]) - case deleteRequest if deleteRequest.isInstanceOf[DeleteRequest] => - requestIndexer.add(deleteRequest.asInstanceOf[DeleteRequest]) - case updateRequest if updateRequest.isInstanceOf[UpdateRequest] => - requestIndexer.add(updateRequest.asInstanceOf[UpdateRequest]) - case _ => - logError( - "ElasticsearchSinkFunction add ActionRequest is deprecated, please use IndexRequest|DeleteRequest|UpdateRequest ") - requestIndexer.add(request) - } - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/sink/ES5Sink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/sink/ES5Sink.scala deleted file mode 100644 index 6f999cbc0d..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/sink/ES5Sink.scala +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch5.sink - -import org.apache.streampark.common.util.{Logger, Utils} -import org.apache.streampark.flink.connector.elasticsearch5.conf.ESConfig -import org.apache.streampark.flink.connector.elasticsearch5.internal.ESSinkFunction -import org.apache.streampark.flink.connector.function.TransformFunction -import org.apache.streampark.flink.connector.sink.Sink -import org.apache.streampark.flink.core.scala.StreamingContext - -import org.apache.flink.streaming.api.datastream.{DataStream => JavaDataStream, DataStreamSink} -import org.apache.flink.streaming.api.scala.DataStream -import org.apache.flink.streaming.connectors.elasticsearch.ActionRequestFailureHandler -import org.apache.flink.streaming.connectors.elasticsearch.util.RetryRejectedExecutionFailureHandler -import org.apache.flink.streaming.connectors.elasticsearch5.ElasticsearchSink -import org.elasticsearch.action.ActionRequest - -import java.util.Properties - -import scala.annotation.meta.param -import org.apache.streampark.common.util.Implicits._ - -object ES5Sink { - - val functionNullHintMsg = "ES pocess element func must not null" - val sinkNullHintMsg = "Sink Stream must not null" - - def apply( - @(transient @param) - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null)(implicit ctx: StreamingContext): ES5Sink = - new ES5Sink(ctx, property, parallelism, name, uid) - -} - -class ES5Sink( - @(transient @param) ctx: StreamingContext, - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null, - alias: String = "") - extends Sink - with Logger { - - val prop: Properties = ctx.parameter.getProperties - - Utils.copyProperties(property, prop) - - def this(ctx: StreamingContext) { - this(ctx, new Properties(), 0, null, null, "") - } - - private val config: ESConfig = new ESConfig(prop) - - private def process[T]( - userConfig: JavaMap[String, String], - stream: JavaDataStream[T], - failureHandler: ActionRequestFailureHandler, - f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = { - require(stream != null, () => sinkNullHintMsg) - require(f != null, () => functionNullHintMsg) - val esSink: ElasticsearchSink[T] = - new ElasticsearchSink(userConfig, config.host, new ESSinkFunction(f), failureHandler) - if (config.disableFlushOnCheckpoint) { - esSink.disableFlushOnCheckpoint() - } - val sink = stream.addSink(esSink) - afterSink(sink, parallelism, name, uid) - } - - private def process[T]( - userConfig: JavaMap[String, String], - stream: DataStream[T], - failureHandler: ActionRequestFailureHandler, - f: T => ActionRequest): DataStreamSink[T] = { - require(stream != null, () => sinkNullHintMsg) - require(f != null, () => functionNullHintMsg) - val esSink: ElasticsearchSink[T] = - new ElasticsearchSink(userConfig, config.host, new ESSinkFunction(f), failureHandler) - if (config.disableFlushOnCheckpoint) { - esSink.disableFlushOnCheckpoint() - } - val sink = stream.addSink(esSink) - afterSink(sink, parallelism, name, uid) - } - - /** - * @param stream - * @param suffix - * @param restClientFactory - * @param failureHandler - * @param f - * @tparam T - * @return - */ - def sink[T]( - userConfig: JavaMap[String, String], - stream: DataStream[T], - failureHandler: ActionRequestFailureHandler = new RetryRejectedExecutionFailureHandler)( - implicit f: T => ActionRequest): DataStreamSink[T] = { - process(userConfig, stream, failureHandler, f) - } - - def sink[T]( - userConfig: JavaMap[String, String], - stream: JavaDataStream[T], - failureHandler: ActionRequestFailureHandler, - f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = { - process(userConfig, stream, failureHandler, f) - } - - def sink[T]( - userConfig: JavaMap[String, String], - stream: JavaDataStream[T], - f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = { - process(userConfig, stream, new RetryRejectedExecutionFailureHandler, f) - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/util/ElasticsearchUtils.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/util/ElasticsearchUtils.scala deleted file mode 100644 index 519b4239f0..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch5/src/main/scala/org/apache/streampark/flink/connector/elasticsearch5/util/ElasticsearchUtils.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch5.util - -import org.elasticsearch.action.index.IndexRequest -import org.elasticsearch.common.bytes.BytesArray -import org.elasticsearch.common.xcontent.XContentType - -object ElasticsearchUtils { - - def indexRequest(index: String, indexType: String, id: String, source: String)(implicit - xContentType: XContentType = XContentType.JSON): IndexRequest = { - require(source != null, "IndexRequest error:source can not be null...") - require(xContentType != null, "IndexRequest error:xContentType can not be null...") - val indexReq = new IndexRequest(index, indexType, id) - val mapping = List("source" -> new BytesArray(source), "contentType" -> xContentType) - mapping.foreach { - x => - val field = indexReq.getClass.getDeclaredField(x._1) - field.setAccessible(true) - field.set(indexReq, x._2) - } - indexReq - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/pom.xml deleted file mode 100644 index 0d944d1e62..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/pom.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - - 4.0.0 - - - org.apache.streampark - streampark-flink-connector-elasticsearch - 2.2.0-SNAPSHOT - - - streampark-flink-connector-elasticsearch6_${scala.binary.version} - StreamPark : Flink Connector Elasticsearch6 - - - 8 - 8 - - - - - org.apache.flink - flink-connector-elasticsearch6 - ${flink.elasticserch.connector.version} - - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/bean/RestClientFactoryImpl.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/bean/RestClientFactoryImpl.scala deleted file mode 100644 index a5e23c89c8..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/bean/RestClientFactoryImpl.scala +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch6.bean - -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.elasticsearch6.conf.ES6Config - -import org.apache.flink.streaming.connectors.elasticsearch6.RestClientFactory -import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} -import org.apache.http.client.CredentialsProvider -import org.apache.http.client.config.RequestConfig -import org.apache.http.impl.client.BasicCredentialsProvider -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder -import org.apache.http.message.BasicHeader -import org.elasticsearch.client.RestClientBuilder - -class RestClientFactoryImpl(val config: ES6Config) extends RestClientFactory with Logger { - override def configureRestClientBuilder(restClientBuilder: RestClientBuilder): Unit = { - // httpClientConfigCallback and requestConfigCallback........ - def configCallback(): RestClientBuilder = { - val userName = config.userName - val password = config.password - // userName,password must be all set,or all not set.. - require( - (userName != null && password != null) || (userName == null && password == null), - "[StreamPark] elasticsearch auth info error,userName,password must be all set,or all not set." - ) - val credentialsProvider = (userName, password) match { - case (null, null) => null - case _ => - val credentialsProvider: CredentialsProvider = new BasicCredentialsProvider() - credentialsProvider.setCredentials( - AuthScope.ANY, - new UsernamePasswordCredentials(userName, password)) - credentialsProvider - } - - val httpClientConfigCallback = new RestClientBuilder.HttpClientConfigCallback { - override def customizeHttpClient( - httpClientBuilder: HttpAsyncClientBuilder): HttpAsyncClientBuilder = { - if (credentialsProvider != null) { - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) - logInfo("elasticsearch auth by userName,password...") - } - // other config.... - httpClientBuilder - } - } - - val requestConfigCallback = new RestClientBuilder.RequestConfigCallback { - override def customizeRequestConfig( - requestConfigBuilder: RequestConfig.Builder): RequestConfig.Builder = { - if (credentialsProvider != null) { - requestConfigBuilder.setAuthenticationEnabled(true) - } - requestConfigBuilder.setConnectionRequestTimeout(config.connectRequestTimeout) - requestConfigBuilder.setConnectTimeout(config.connectTimeout) - requestConfigBuilder.setMaxRedirects(config.maxRedirects) - requestConfigBuilder.setRedirectsEnabled(config.redirectsEnabled) - requestConfigBuilder.setConnectTimeout(config.socketTimeout) - requestConfigBuilder.setRelativeRedirectsAllowed(config.relativeRedirectsAllowed) - requestConfigBuilder.setContentCompressionEnabled(config.contentCompressionEnabled) - requestConfigBuilder.setNormalizeUri(config.normalizeUri) - requestConfigBuilder - } - } - restClientBuilder.setHttpClientConfigCallback(httpClientConfigCallback) - restClientBuilder.setRequestConfigCallback(requestConfigCallback) - } - - def setHeader(): RestClientBuilder = { - val headers = new BasicHeader("Content-Type", config.contentType) - restClientBuilder.setDefaultHeaders(Array(headers)) - restClientBuilder.setMaxRetryTimeoutMillis(config.maxRetry) - config.pathPrefix match { - case null => null - case path => restClientBuilder.setPathPrefix(path) - } - - } - - configCallback() - - setHeader() - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6Config.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6Config.scala deleted file mode 100644 index 3d9192190c..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6Config.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch6.conf - -import org.apache.http.HttpHost - -import java.util.Properties - -class ES6Config(parameters: Properties) extends Serializable { - - val sinkOption: ESSinkConfigOption = ESSinkConfigOption(properties = parameters) - - val disableFlushOnCheckpoint: Boolean = sinkOption.disableFlushOnCheckpoint.get() - - val host: List[HttpHost] = sinkOption.host.get().toList - - val userName: String = sinkOption.userName.get() - - val password: String = sinkOption.password.get() - - val connectRequestTimeout: Int = sinkOption.connectRequestTimeout.get() - - val connectTimeout: Int = sinkOption.connectTimeout.get() - - val maxRetry: Int = sinkOption.maxRetry.get() - - val contentType: String = sinkOption.contentType.get() - - val pathPrefix: String = sinkOption.pathPrefix.get() - - val staleConnectionCheckEnabled: Boolean = sinkOption.staleConnectionCheckEnabled.get() - - val redirectsEnabled: Boolean = sinkOption.redirectsEnabled.get() - - val maxRedirects: Int = sinkOption.maxRedirects.get() - - val relativeRedirectsAllowed: Boolean = sinkOption.relativeRedirectsAllowed.get() - - val authenticationEnabled: Boolean = sinkOption.authenticationEnabled.get() - - val socketTimeout: Int = sinkOption.socketTimeout.get() - - val contentCompressionEnabled: Boolean = sinkOption.contentCompressionEnabled.get() - - val normalizeUri: Boolean = sinkOption.normalizeUri.get() - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6SinkConfigOption.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6SinkConfigOption.scala deleted file mode 100644 index 3e3629d24b..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/conf/ES6SinkConfigOption.scala +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch6.conf - -import org.apache.streampark.common.conf.ConfigOption -import org.apache.streampark.common.util.ConfigUtils - -import org.apache.http.HttpHost - -import java.net.InetSocketAddress -import java.util.Properties - -import org.apache.streampark.common.util.Implicits._ - -object ESSinkConfigOption { - - val ES_SINK_PREFIX = "es.sink" - - /** - * @param properties - * @return - */ - def apply( - prefixStr: String = ES_SINK_PREFIX, - properties: Properties = new Properties): ESSinkConfigOption = - new ESSinkConfigOption(prefixStr, properties) - -} - -class ESSinkConfigOption(prefixStr: String, properties: Properties) extends Serializable { - - implicit val (prefix, prop) = (prefixStr, properties) - - val SIGN_COMMA = "," - - val SIGN_COLON = ":" - - val disableFlushOnCheckpoint: ConfigOption[Boolean] = ConfigOption( - key = "es.disableFlushOnCheckpoint", - required = false, - classType = classOf[Boolean], - defaultValue = false) - - val host: ConfigOption[Array[HttpHost]] = ConfigOption( - key = "host", - required = true, - classType = classOf[Array[InetSocketAddress]], - handle = key => - properties - .getProperty(key) - .split(SIGN_COMMA) - .map( - x => { - x.split(SIGN_COLON) match { - case Array(host, port) => new HttpHost(host, port.toInt) - } - }) - ) - - val userName: ConfigOption[String] = ConfigOption( - key = "es.auth.user", - required = false, - classType = classOf[String], - defaultValue = null) - - val password: ConfigOption[String] = ConfigOption( - key = "es.auth.password", - required = false, - classType = classOf[String], - defaultValue = null) - - val connectRequestTimeout: ConfigOption[Int] = ConfigOption( - key = "es.connect.request.timeout", - required = false, - classType = classOf[Int], - defaultValue = -1) - - val connectTimeout: ConfigOption[Int] = ConfigOption( - key = "es.connect.timeout", - required = false, - classType = classOf[Int], - defaultValue = -1) - - val maxRetry: ConfigOption[Int] = ConfigOption( - key = "es.rest.max.retry.timeout", - required = false, - classType = classOf[Int], - defaultValue = 10000) - - val contentType: ConfigOption[String] = ConfigOption( - key = "es.rest.content.type", - required = false, - classType = classOf[String], - defaultValue = "application/json") - - val pathPrefix: ConfigOption[String] = ConfigOption( - key = "es.rest.path.prefix", - required = false, - classType = classOf[String], - defaultValue = null) - - val staleConnectionCheckEnabled: ConfigOption[Boolean] = ConfigOption( - key = "es.connect.check.enable", - required = false, - classType = classOf[Boolean], - defaultValue = false) - - val redirectsEnabled: ConfigOption[Boolean] = ConfigOption( - key = "es.redirects.enable", - required = false, - classType = classOf[Boolean], - defaultValue = false) - - val maxRedirects: ConfigOption[Int] = ConfigOption( - key = "es.max.redirects", - required = false, - classType = classOf[Int], - defaultValue = 50) - - val relativeRedirectsAllowed: ConfigOption[Boolean] = ConfigOption( - key = "es.relative.redirects.allowed", - required = false, - classType = classOf[Boolean], - defaultValue = true) - - val authenticationEnabled: ConfigOption[Boolean] = ConfigOption( - key = "es.authentication.enable", - required = false, - classType = classOf[Boolean], - defaultValue = true) - - val socketTimeout: ConfigOption[Int] = ConfigOption( - key = "es.socket.timeout", - required = false, - classType = classOf[Int], - defaultValue = -1) - - val contentCompressionEnabled: ConfigOption[Boolean] = ConfigOption( - key = "es.content.compression.enable", - required = false, - classType = classOf[Boolean], - defaultValue = true) - - val normalizeUri: ConfigOption[Boolean] = ConfigOption( - key = "es.normalize.uri", - required = false, - classType = classOf[Boolean], - defaultValue = true) - - def getInternalConfig(): JavaMap[String, String] = { - ConfigUtils.getConf(prop, prefix)(alias = "") - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/internal/ESSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/internal/ESSinkFunction.scala deleted file mode 100644 index af2c7afa2c..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/internal/ESSinkFunction.scala +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch6.internal - -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.enums.ApiType.ApiType -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.function.TransformFunction - -import org.apache.flink.api.common.functions.RuntimeContext -import org.apache.flink.streaming.connectors.elasticsearch.{ElasticsearchSinkFunction, RequestIndexer} -import org.elasticsearch.action.ActionRequest -import org.elasticsearch.action.delete.DeleteRequest -import org.elasticsearch.action.index.IndexRequest -import org.elasticsearch.action.update.UpdateRequest - -class ESSinkFunction[T](apiType: ApiType = ApiType.scala) - extends ElasticsearchSinkFunction[T] - with Logger { - private[this] var scalaFunc: T => ActionRequest = _ - private[this] var javaFunc: TransformFunction[T, ActionRequest] = _ - - // for Scala - def this(scalaFunc: T => ActionRequest) = { - this(ApiType.scala) - this.scalaFunc = scalaFunc - } - - // for JAVA - def this(javaFunc: TransformFunction[T, ActionRequest]) = { - this(ApiType.java) - this.javaFunc = javaFunc - } - - def createIndexRequest(element: T): ActionRequest = apiType match { - case ApiType.java => javaFunc.transform(element) - case ApiType.scala => scalaFunc(element) - } - - override def process( - element: T, - runtimeContext: RuntimeContext, - requestIndexer: RequestIndexer): Unit = { - val request: ActionRequest = createIndexRequest(element) - request match { - case indexRequest if indexRequest.isInstanceOf[IndexRequest] => - requestIndexer.add(indexRequest.asInstanceOf[IndexRequest]) - case deleteRequest if deleteRequest.isInstanceOf[DeleteRequest] => - requestIndexer.add(deleteRequest.asInstanceOf[DeleteRequest]) - case updateRequest if updateRequest.isInstanceOf[UpdateRequest] => - requestIndexer.add(updateRequest.asInstanceOf[UpdateRequest]) - case _ => - logError( - "ElasticsearchSinkFunction add ActionRequest is deprecated, please use IndexRequest|DeleteRequest|UpdateRequest ") - requestIndexer.add(request) - } - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/sink/ES6Sink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/sink/ES6Sink.scala deleted file mode 100644 index ff1067f937..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/sink/ES6Sink.scala +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch6.sink - -import org.apache.streampark.common.util.{Logger, Utils} -import org.apache.streampark.flink.connector.elasticsearch6.bean.RestClientFactoryImpl -import org.apache.streampark.flink.connector.elasticsearch6.conf.ES6Config -import org.apache.streampark.flink.connector.elasticsearch6.internal.ESSinkFunction -import org.apache.streampark.flink.connector.function.TransformFunction -import org.apache.streampark.flink.connector.sink.Sink -import org.apache.streampark.flink.core.scala.StreamingContext - -import org.apache.flink.streaming.api.datastream.{DataStream => JavaDataStream, DataStreamSink} -import org.apache.flink.streaming.api.scala.DataStream -import org.apache.flink.streaming.connectors.elasticsearch.ActionRequestFailureHandler -import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase._ -import org.apache.flink.streaming.connectors.elasticsearch.util.RetryRejectedExecutionFailureHandler -import org.apache.flink.streaming.connectors.elasticsearch6.{ElasticsearchSink, RestClientFactory} -import org.elasticsearch.action.ActionRequest - -import java.util.Properties - -import scala.annotation.meta.param -import org.apache.streampark.common.util.Implicits._ - -object ES6Sink { - - val functionNullHintMsg = "ES pocess element func must not null" - val sinkNullHintMsg = "Sink Stream must not null" - - def apply( - @(transient @param) - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null)(implicit ctx: StreamingContext): ES6Sink = - new ES6Sink(ctx, property, parallelism, name, uid) - -} - -class ES6Sink( - @(transient @param) ctx: StreamingContext, - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null) - extends Sink - with Logger { - - val prop: Properties = ctx.parameter.getProperties - - Utils.copyProperties(property, prop) - - def this(ctx: StreamingContext) { - this(ctx, new Properties(), 0, null, null) - } - - private val config: ES6Config = new ES6Config(prop) - - private def process[T]( - stream: DataStream[T], - restClientFactory: Option[RestClientFactory], - failureHandler: ActionRequestFailureHandler, - f: T => ActionRequest): DataStreamSink[T] = { - require(stream != null, sinkNullHintMsg) - require(f != null, functionNullHintMsg) - - val sinkFunc: ESSinkFunction[T] = new ESSinkFunction(f) - - val esSink: ElasticsearchSink[T] = buildESSink(restClientFactory, failureHandler, sinkFunc) - if (config.disableFlushOnCheckpoint) { - esSink.disableFlushOnCheckpoint() - } - val sink = stream.addSink(esSink) - afterSink(sink, parallelism, name, uid) - } - - private def process[T]( - stream: JavaDataStream[T], - restClientFactory: Option[RestClientFactory], - failureHandler: ActionRequestFailureHandler, - f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = { - require(stream != null, () => sinkNullHintMsg) - require(f != null, () => functionNullHintMsg) - - val sinkFunc: ESSinkFunction[T] = new ESSinkFunction(f) - - val esSink: ElasticsearchSink[T] = buildESSink(restClientFactory, failureHandler, sinkFunc) - if (config.disableFlushOnCheckpoint) { - esSink.disableFlushOnCheckpoint() - } - val sink = stream.addSink(esSink) - afterSink(sink, parallelism, name, uid) - } - - private def buildESSink[T]( - restClientFactory: Option[RestClientFactory], - failureHandler: ActionRequestFailureHandler, - sinkFunc: ESSinkFunction[T]): ElasticsearchSink[T] = { - val sinkBuilder = new ElasticsearchSink.Builder[T](config.host, sinkFunc) - // failureHandler - sinkBuilder.setFailureHandler(failureHandler) - - // restClientFactory - restClientFactory match { - case Some(factory) => - sinkBuilder.setRestClientFactory(factory) - case None => - sinkBuilder.setRestClientFactory(new RestClientFactoryImpl(config)) - } - - def doConfig(param: (String, String)): Unit = param match { - // parameter of sink.es.bulk.flush.max.actions - case (CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, v) => sinkBuilder.setBulkFlushMaxActions(v.toInt) - // parameter of sink.es.bulk.flush.max.size.mb - case (CONFIG_KEY_BULK_FLUSH_MAX_SIZE_MB, v) => sinkBuilder.setBulkFlushMaxSizeMb(v.toInt) - // parameter of sink.es.bulk.flush.interval.ms - case (CONFIG_KEY_BULK_FLUSH_INTERVAL_MS, v) => sinkBuilder.setBulkFlushInterval(v.toInt) - // parameter of sink.es.bulk.flush.backoff.enable - case (CONFIG_KEY_BULK_FLUSH_BACKOFF_ENABLE, v) => sinkBuilder.setBulkFlushBackoff(v.toBoolean) - // parameter of sink.es.bulk.flush.backoff.type value of [ CONSTANT or EXPONENTIAL ] - case (CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE, v) => - sinkBuilder.setBulkFlushBackoffType(FlushBackoffType.valueOf(v)) - // parameter of sink.es.bulk.flush.backoff.retries - case (CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES, v) => - sinkBuilder.setBulkFlushBackoffRetries(v.toInt) - // parameter of sink.es.bulk.flush.backoff.delay - case (CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY, v) => - sinkBuilder.setBulkFlushBackoffDelay(v.toLong) - // other... - case _ => - } - // set value from properties - config.sinkOption.getInternalConfig().foreach(doConfig) - sinkBuilder.build() - } - - /** - * @param stream - * @param suffix - * @param restClientFactory - * @param failureHandler - * @param f - * @tparam T - * @return - */ - def sink[T]( - stream: DataStream[T], - restClientFactory: Option[RestClientFactory] = None, - failureHandler: ActionRequestFailureHandler = new RetryRejectedExecutionFailureHandler)( - implicit f: T => ActionRequest): DataStreamSink[T] = { - process(stream, restClientFactory, failureHandler, f) - } - - def sink[T]( - stream: JavaDataStream[T], - restClientFactory: RestClientFactory, - failureHandler: ActionRequestFailureHandler, - f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = { - process(stream, Some(restClientFactory), failureHandler, f) - } - - def sink[T]( - stream: JavaDataStream[T], - restClientFactory: RestClientFactory, - f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = { - process(stream, Some(restClientFactory), new RetryRejectedExecutionFailureHandler, f) - } - - def sink[T]( - stream: JavaDataStream[T], - f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = { - process(stream, None, new RetryRejectedExecutionFailureHandler, f) - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/util/ElasticsearchUtils.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/util/ElasticsearchUtils.scala deleted file mode 100644 index 34cd2d7a8c..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch6/src/main/scala/org/apache/streampark/flink/connector/elasticsearch6/util/ElasticsearchUtils.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch6.util - -import org.elasticsearch.action.index.IndexRequest -import org.elasticsearch.common.bytes.BytesArray -import org.elasticsearch.common.xcontent.XContentType - -object ElasticsearchUtils { - - def indexRequest(index: String, indexType: String, id: String, source: String)(implicit - xContentType: XContentType = XContentType.JSON): IndexRequest = { - require(source != null, "EndexRequest error:source can not be null...") - require(xContentType != null, "IndexRequest error:xContentType can not be null...") - val indexReq = new IndexRequest(index, indexType, id) - val mapping = List("source" -> new BytesArray(source), "contentType" -> xContentType) - mapping.foreach { - x => - val field = indexReq.getClass.getDeclaredField(x._1) - field.setAccessible(true) - field.set(indexReq, x._2) - } - indexReq - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/pom.xml deleted file mode 100644 index 53bde3349c..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/pom.xml +++ /dev/null @@ -1,87 +0,0 @@ - - - - 4.0.0 - - streampark-flink-connector-elasticsearch - org.apache.streampark - 2.2.0-SNAPSHOT - - - streampark-flink-connector-elasticsearch7_${scala.binary.version} - StreamPark : Flink Connector Elasticsearch7 - - - 8 - 8 - - - - - org.apache.flink - flink-connector-elasticsearch7 - ${flink.elasticserch.connector.version} - - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/bean/RestClientFactoryImpl.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/bean/RestClientFactoryImpl.scala deleted file mode 100644 index c30b64e19d..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/bean/RestClientFactoryImpl.scala +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch7.bean - -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.elasticsearch7.conf.ES7Config - -import org.apache.flink.streaming.connectors.elasticsearch7.RestClientFactory -import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} -import org.apache.http.client.CredentialsProvider -import org.apache.http.client.config.RequestConfig -import org.apache.http.impl.client.BasicCredentialsProvider -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder -import org.apache.http.message.BasicHeader -import org.elasticsearch.client.RestClientBuilder - -class RestClientFactoryImpl(val config: ES7Config) extends RestClientFactory with Logger { - override def configureRestClientBuilder(restClientBuilder: RestClientBuilder): Unit = { - // httpClientConfigCallback and requestConfigCallback........ - def configCallback(): RestClientBuilder = { - val userName = config.userName - val password = config.password - // userName,password must be all set,or all not set.. - require( - (userName != null && password != null) || (userName == null && password == null), - "[StreamPark] elasticsearch auth info error,userName,password must be all set,or all not set." - ) - val credentialsProvider = (userName, password) match { - case (null, null) => null - case _ => - val credentialsProvider: CredentialsProvider = new BasicCredentialsProvider() - credentialsProvider.setCredentials( - AuthScope.ANY, - new UsernamePasswordCredentials(userName, password)) - credentialsProvider - } - - val httpClientConfigCallback = new RestClientBuilder.HttpClientConfigCallback { - override def customizeHttpClient( - httpClientBuilder: HttpAsyncClientBuilder): HttpAsyncClientBuilder = { - if (credentialsProvider != null) { - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) - logInfo("Elasticsearch auth by userName,password...") - } - httpClientBuilder - } - } - - val requestConfigCallback = new RestClientBuilder.RequestConfigCallback { - override def customizeRequestConfig( - requestConfigBuilder: RequestConfig.Builder): RequestConfig.Builder = { - if (credentialsProvider != null) { - requestConfigBuilder.setAuthenticationEnabled(true) - } - requestConfigBuilder.setConnectionRequestTimeout(config.connectRequestTimeout) - requestConfigBuilder.setConnectTimeout(config.connectTimeout) - requestConfigBuilder.setMaxRedirects(config.maxRedirects) - requestConfigBuilder.setRedirectsEnabled(config.redirectsEnabled) - requestConfigBuilder.setConnectTimeout(config.socketTimeout) - requestConfigBuilder.setRelativeRedirectsAllowed(config.relativeRedirectsAllowed) - requestConfigBuilder.setContentCompressionEnabled(config.contentCompressionEnabled) - requestConfigBuilder.setNormalizeUri(config.normalizeUri) - } - } - restClientBuilder.setHttpClientConfigCallback(httpClientConfigCallback) - restClientBuilder.setRequestConfigCallback(requestConfigCallback) - } - - def setHeader(): RestClientBuilder = { - val headers = new BasicHeader("Content-Type", config.contentType) - restClientBuilder.setDefaultHeaders(Array(headers)) - config.pathPrefix match { - case null => null - case path => restClientBuilder.setPathPrefix(path) - } - } - - configCallback() - setHeader() - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/conf/ES7Config.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/conf/ES7Config.scala deleted file mode 100644 index 2ea777316a..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/conf/ES7Config.scala +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch7.conf - -import org.apache.http.HttpHost - -import java.util.Properties - -class ES7Config(parameters: Properties) extends Serializable { - - val sinkOption: ES7SinkConfigOption = ES7SinkConfigOption(properties = parameters) - - val disableFlushOnCheckpoint: Boolean = sinkOption.disableFlushOnCheckpoint.get() - - val host: List[HttpHost] = sinkOption.host.get().toList - - val userName: String = sinkOption.userName.get() - - val password: String = sinkOption.password.get() - - val connectRequestTimeout: Int = sinkOption.connectRequestTimeout.get() - - val connectTimeout: Int = sinkOption.connectTimeout.get() - - val contentType: String = sinkOption.contentType.get() - - val pathPrefix: String = sinkOption.pathPrefix.get() - - val staleConnectionCheckEnabled: Boolean = sinkOption.staleConnectionCheckEnabled.get() - - val redirectsEnabled: Boolean = sinkOption.redirectsEnabled.get() - - val maxRedirects: Int = sinkOption.maxRedirects.get() - - val relativeRedirectsAllowed: Boolean = sinkOption.relativeRedirectsAllowed.get() - - val authenticationEnabled: Boolean = sinkOption.authenticationEnabled.get() - - val socketTimeout: Int = sinkOption.socketTimeout.get() - - val contentCompressionEnabled: Boolean = sinkOption.contentCompressionEnabled.get() - - val normalizeUri: Boolean = sinkOption.normalizeUri.get() - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/conf/ES7SinkConfigOption.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/conf/ES7SinkConfigOption.scala deleted file mode 100644 index 60f8b44a32..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/conf/ES7SinkConfigOption.scala +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch7.conf - -import org.apache.streampark.common.conf.ConfigOption -import org.apache.streampark.common.util.ConfigUtils - -import org.apache.http.HttpHost - -import java.net.InetSocketAddress -import java.util.Properties - -import org.apache.streampark.common.util.Implicits._ - -object ES7SinkConfigOption { - - val ES_SINK_PREFIX = "es.sink" - - /** - * @param properties - * @return - */ - def apply( - prefixStr: String = ES_SINK_PREFIX, - properties: Properties = new Properties): ES7SinkConfigOption = - new ES7SinkConfigOption(prefixStr, properties) - -} - -class ES7SinkConfigOption(prefixStr: String, properties: Properties) extends Serializable { - - implicit val (prefix, prop) = (prefixStr, properties) - - val SIGN_COMMA = "," - - val SIGN_COLON = ":" - - val disableFlushOnCheckpoint: ConfigOption[Boolean] = ConfigOption( - key = "es.disableFlushOnCheckpoint", - required = false, - classType = classOf[Boolean], - defaultValue = false) - - val host: ConfigOption[Array[HttpHost]] = ConfigOption( - key = "host", - required = true, - classType = classOf[Array[InetSocketAddress]], - handle = key => - properties - .getProperty(key) - .split(SIGN_COMMA) - .map( - x => { - x.split(SIGN_COLON) match { - case Array(host, port) => new HttpHost(host, port.toInt) - } - }) - ) - - val userName: ConfigOption[String] = ConfigOption( - key = "es.auth.user", - required = false, - classType = classOf[String], - defaultValue = null) - - val password: ConfigOption[String] = ConfigOption( - key = "es.auth.password", - required = false, - classType = classOf[String], - defaultValue = null) - - val connectRequestTimeout: ConfigOption[Int] = ConfigOption( - key = "es.connect.request.timeout", - required = false, - classType = classOf[Int], - defaultValue = -1) - - val connectTimeout: ConfigOption[Int] = ConfigOption( - key = "es.connect.timeout", - required = false, - classType = classOf[Int], - defaultValue = -1) - - val maxRetry: ConfigOption[Int] = ConfigOption( - key = "es.rest.max.retry.timeout", - required = false, - classType = classOf[Int], - defaultValue = 10000) - - val contentType: ConfigOption[String] = ConfigOption( - key = "es.rest.content.type", - required = false, - classType = classOf[String], - defaultValue = "application/json") - - val pathPrefix: ConfigOption[String] = ConfigOption( - key = "es.rest.path.prefix", - required = false, - classType = classOf[String], - defaultValue = null) - - val staleConnectionCheckEnabled: ConfigOption[Boolean] = ConfigOption( - key = "es.connect.check.enable", - required = false, - classType = classOf[Boolean], - defaultValue = false) - - val redirectsEnabled: ConfigOption[Boolean] = ConfigOption( - key = "es.redirects.enable", - required = false, - classType = classOf[Boolean], - defaultValue = false) - - val maxRedirects: ConfigOption[Int] = ConfigOption( - key = "es.max.redirects", - required = false, - classType = classOf[Int], - defaultValue = 50) - - val relativeRedirectsAllowed: ConfigOption[Boolean] = ConfigOption( - key = "es.relative.redirects.allowed", - required = false, - classType = classOf[Boolean], - defaultValue = true) - - val authenticationEnabled: ConfigOption[Boolean] = ConfigOption( - key = "es.authentication.enable", - required = false, - classType = classOf[Boolean], - defaultValue = true) - - val socketTimeout: ConfigOption[Int] = ConfigOption( - key = "es.socket.timeout", - required = false, - classType = classOf[Int], - defaultValue = -1) - - val contentCompressionEnabled: ConfigOption[Boolean] = ConfigOption( - key = "es.content.compression.enable", - required = false, - classType = classOf[Boolean], - defaultValue = true) - - val normalizeUri: ConfigOption[Boolean] = ConfigOption( - key = "es.normalize.uri", - required = false, - classType = classOf[Boolean], - defaultValue = true) - - def getInternalConfig(): JavaMap[String, String] = { - ConfigUtils.getConf(prop, prefix)(alias = "") - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/internal/ESSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/internal/ESSinkFunction.scala deleted file mode 100644 index 1533ea6a48..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/internal/ESSinkFunction.scala +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch7.internal - -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.enums.ApiType.ApiType -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.function.TransformFunction - -import org.apache.flink.api.common.functions.RuntimeContext -import org.apache.flink.streaming.connectors.elasticsearch.{ElasticsearchSinkFunction, RequestIndexer} -import org.elasticsearch.action.ActionRequest -import org.elasticsearch.action.delete.DeleteRequest -import org.elasticsearch.action.index.IndexRequest -import org.elasticsearch.action.update.UpdateRequest - -class ESSinkFunction[T](apiType: ApiType = ApiType.scala) - extends ElasticsearchSinkFunction[T] - with Logger { - private[this] var scalaFunc: (T => ActionRequest) with Serializable = _ - private[this] var javaFunc: TransformFunction[T, ActionRequest] = _ - - // for Scala - def this(scalaFunc: T => ActionRequest) = { - this(ApiType.scala) - this.scalaFunc = new (T => ActionRequest) with Serializable { - override def apply(v: T): ActionRequest = scalaFunc.apply(v) - } - } - - // for Java - def this(javaFunc: TransformFunction[T, ActionRequest]) = { - this(ApiType.java) - this.javaFunc = javaFunc - } - - def createIndexRequest(element: T): ActionRequest = apiType match { - case ApiType.java => javaFunc.transform(element) - case ApiType.scala => scalaFunc(element) - } - - override def process(element: T, ctx: RuntimeContext, requestIndexer: RequestIndexer): Unit = { - val request = createIndexRequest(element) - request match { - case indexRequest: IndexRequest => requestIndexer.add(indexRequest) - case deleteRequest: DeleteRequest => requestIndexer.add(deleteRequest) - case updateRequest: UpdateRequest => requestIndexer.add(updateRequest) - case _ => - logError( - "ElasticsearchSinkFunction add ActionRequest is deprecated, please use IndexRequest|DeleteRequest|UpdateRequest ") - requestIndexer.add(request) - } - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/sink/ES7Sink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/sink/ES7Sink.scala deleted file mode 100644 index 2399baad43..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/sink/ES7Sink.scala +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch7.sink - -import org.apache.streampark.common.conf.ConfigConst._ -import org.apache.streampark.common.util.{ConfigUtils, Logger, Utils} -import org.apache.streampark.flink.connector.elasticsearch7.bean.RestClientFactoryImpl -import org.apache.streampark.flink.connector.elasticsearch7.conf.ES7Config -import org.apache.streampark.flink.connector.elasticsearch7.internal.ESSinkFunction -import org.apache.streampark.flink.connector.function.TransformFunction -import org.apache.streampark.flink.connector.sink.Sink -import org.apache.streampark.flink.core.scala.StreamingContext - -import org.apache.flink.streaming.api.datastream.{DataStream => JavaDataStream, DataStreamSink} -import org.apache.flink.streaming.api.scala.DataStream -import org.apache.flink.streaming.connectors.elasticsearch.ActionRequestFailureHandler -import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase.{CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY, CONFIG_KEY_BULK_FLUSH_BACKOFF_ENABLE, CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES, CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE, CONFIG_KEY_BULK_FLUSH_INTERVAL_MS, CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, CONFIG_KEY_BULK_FLUSH_MAX_SIZE_MB, FlushBackoffType} -import org.apache.flink.streaming.connectors.elasticsearch.util.RetryRejectedExecutionFailureHandler -import org.apache.flink.streaming.connectors.elasticsearch7.{ElasticsearchSink, RestClientFactory} -import org.apache.http.HttpHost -import org.elasticsearch.action.ActionRequest - -import java.util.Properties - -import scala.annotation.meta.param -import org.apache.streampark.common.util.Implicits._ - -object ES7Sink { - - val functionNullHintMsg = "ES pocess element func must not null" - val sinkNullHintMsg = "Sink Stream must not null" - - def apply( - @(transient @param) - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null)(implicit ctx: StreamingContext): ES7Sink = - new ES7Sink(ctx, property, parallelism, name, uid) -} - -class ES7Sink( - @(transient @param) ctx: StreamingContext, - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null) - extends Sink - with Logger { - - def this(ctx: StreamingContext) { - this(ctx, new Properties(), 0, null, null) - } - - val prop: Properties = ctx.parameter.getProperties - - Utils.copyProperties(property, prop) - - private val config: ES7Config = new ES7Config(prop) - - private def process[T]( - stream: DataStream[T], - restClientFactory: Option[RestClientFactory], - failureHandler: ActionRequestFailureHandler, - f: T => ActionRequest): DataStreamSink[T] = { - require(stream != null, sinkNullHintMsg) - require(f != null, functionNullHintMsg) - val sinkFunc: ESSinkFunction[T] = new ESSinkFunction(f) - val esSink: ElasticsearchSink[T] = buildESSink(restClientFactory, failureHandler, sinkFunc) - if (config.disableFlushOnCheckpoint) { - esSink.disableFlushOnCheckpoint() - } - val sink = stream.addSink(esSink) - afterSink(sink, parallelism, name, uid) - } - - private def process[T]( - stream: JavaDataStream[T], - restClientFactory: Option[RestClientFactory], - failureHandler: ActionRequestFailureHandler, - f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = { - require(stream != null, sinkNullHintMsg) - require(f != null, functionNullHintMsg) - val sinkFunc: ESSinkFunction[T] = new ESSinkFunction(f) - val esSink: ElasticsearchSink[T] = buildESSink(restClientFactory, failureHandler, sinkFunc) - if (config.disableFlushOnCheckpoint) { - esSink.disableFlushOnCheckpoint() - } - val sink = stream.addSink(esSink) - afterSink(sink, parallelism, name, uid) - } - - private def buildESSink[T]( - restClientFactory: Option[RestClientFactory], - failureHandler: ActionRequestFailureHandler, - sinkFunc: ESSinkFunction[T]): ElasticsearchSink[T] = { - val sinkBuilder = new ElasticsearchSink.Builder[T](config.host, sinkFunc) - sinkBuilder.setFailureHandler(failureHandler) - restClientFactory match { - case Some(factory) => - sinkBuilder.setRestClientFactory(factory) - case None => - sinkBuilder.setRestClientFactory(new RestClientFactoryImpl(config)) - } - - def doConfig(param: (String, String)): Unit = param match { - // parameter of sink.es.bulk.flush.max.actions - case (CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, v) => sinkBuilder.setBulkFlushMaxActions(v.toInt) - // parameter of sink.es.bulk.flush.max.size.mb - case (CONFIG_KEY_BULK_FLUSH_MAX_SIZE_MB, v) => sinkBuilder.setBulkFlushMaxSizeMb(v.toInt) - // parameter of sink.es.bulk.flush.interval.ms - case (CONFIG_KEY_BULK_FLUSH_INTERVAL_MS, v) => sinkBuilder.setBulkFlushInterval(v.toInt) - // parameter of sink.es.bulk.flush.backoff.enable - case (CONFIG_KEY_BULK_FLUSH_BACKOFF_ENABLE, v) => sinkBuilder.setBulkFlushBackoff(v.toBoolean) - // parameter of sink.es.bulk.flush.backoff.type value of [ CONSTANT or EXPONENTIAL ] - case (CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE, v) => - sinkBuilder.setBulkFlushBackoffType(FlushBackoffType.valueOf(v)) - // parameter of sink.es.bulk.flush.backoff.retries - case (CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES, v) => - sinkBuilder.setBulkFlushBackoffRetries(v.toInt) - // parameter of sink.es.bulk.flush.backoff.delay - case (CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY, v) => - sinkBuilder.setBulkFlushBackoffDelay(v.toLong) - // other... - case _ => - } - // set value from properties - config.sinkOption.getInternalConfig().foreach(doConfig) - sinkBuilder.build() - } - - def sink[T]( - stream: DataStream[T], - restClientFactory: Option[RestClientFactory] = None, - failureHandler: ActionRequestFailureHandler = new RetryRejectedExecutionFailureHandler)( - implicit f: T => ActionRequest): DataStreamSink[T] = { - process(stream, restClientFactory, failureHandler, f) - } - - def sink[T]( - stream: JavaDataStream[T], - restClientFactory: RestClientFactory, - failureHandler: ActionRequestFailureHandler, - f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = { - process(stream, Some(restClientFactory), failureHandler, f) - } - - def sink[T]( - stream: JavaDataStream[T], - restClientFactory: RestClientFactory, - f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = { - process(stream, Some(restClientFactory), new RetryRejectedExecutionFailureHandler, f) - } - - def sink[T]( - stream: JavaDataStream[T], - f: TransformFunction[T, ActionRequest]): DataStreamSink[T] = { - process(stream, None, new RetryRejectedExecutionFailureHandler, f) - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/util/ElasticsearchUtils.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/util/ElasticsearchUtils.scala deleted file mode 100644 index 8550706714..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-elasticsearch/streampark-flink-connector-elasticsearch7/src/main/scala/org/apache/streampark/flink/connector/elasticsearch7/util/ElasticsearchUtils.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.elasticsearch7.util - -import org.elasticsearch.action.index.IndexRequest -import org.elasticsearch.common.bytes.BytesArray -import org.elasticsearch.common.xcontent.XContentType - -object ElasticsearchUtils { - - def indexRequest(index: String, id: String, source: String)(implicit - xContentType: XContentType = XContentType.JSON): IndexRequest = { - require(source != null, "IndexRequest error:source can not be null...") - require(xContentType != null, "IndexRequest error:xContentType can not be null...") - val indexReq = new IndexRequest(index).id(id) - val mapping = List("source" -> new BytesArray(source), "contentType" -> xContentType) - mapping.foreach { - x => - val field = indexReq.getClass.getDeclaredField(x._1) - field.setAccessible(true) - field.set(indexReq, x._2) - } - indexReq - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/pom.xml deleted file mode 100644 index 0d0bc4e7b7..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/pom.xml +++ /dev/null @@ -1,105 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink-connector - 2.2.0-SNAPSHOT - - - streampark-flink-connector-hbase_${scala.binary.version} - StreamPark : Flink Connector Hbase - - - - - org.apache.streampark - streampark-flink-connector-base_${scala.binary.version} - ${project.version} - - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - - org.apache.hadoop - hadoop-client-api - - - - org.apache.hadoop - hadoop-client-runtime - - - - org.apache.hbase - hbase-client - - - log4j - log4j - - - - - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/bean/HBaseQuery.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/bean/HBaseQuery.java deleted file mode 100644 index cf72893f6b..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/bean/HBaseQuery.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.hbase.bean; - -import org.apache.streampark.common.util.HBaseClient; - -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; - -import java.io.IOException; -import java.io.Serializable; -import java.util.Properties; - -/** Wrapping a HBase query condition object, wrapping the scan and get two query methods. */ -public class HBaseQuery extends Scan implements Serializable { - - private final String table; - private volatile Table htable; - private transient Scan scan; - private transient Get get; - - public HBaseQuery(String table, Scan scan) throws IOException { - super(scan); - this.table = table; - this.scan = scan; - } - - public HBaseQuery(String table, Get get) { - super(get); - this.table = table; - this.get = get; - } - - public Table getTable(Properties prop) { - if (htable == null) { - synchronized (HBaseQuery.class) { - if (htable == null) { - htable = HBaseClient.apply(prop).table(this.getTable()); - } - } - } - return htable; - } - - public String getTable() { - return table; - } - - public Scan getScan() { - return scan; - } - - public Get getGet() { - return get; - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/function/HBaseQueryFunction.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/function/HBaseQueryFunction.java deleted file mode 100644 index eadffa7299..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/function/HBaseQueryFunction.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.hbase.function; - -import org.apache.streampark.flink.connector.hbase.bean.HBaseQuery; - -import java.io.Serializable; - -@FunctionalInterface -public interface HBaseQueryFunction extends Serializable { - /** - * get hbase query obj - * - * @param last: last one - * @return HBaseQuery: HBaseQuery - */ - HBaseQuery query(T last); -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/function/HBaseResultFunction.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/function/HBaseResultFunction.java deleted file mode 100644 index 41a2927316..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/function/HBaseResultFunction.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.hbase.function; - -import org.apache.hadoop.hbase.client.Result; - -import java.io.Serializable; - -@FunctionalInterface -public interface HBaseResultFunction extends Serializable { - /** - * get hbase result - * - * @param result Result - * @return T t - */ - T result(Result result); -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/source/HBaseJavaSource.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/source/HBaseJavaSource.java deleted file mode 100644 index a9dd19f7ef..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/java/org/apache/streampark/flink/connector/hbase/source/HBaseJavaSource.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.hbase.source; - -import org.apache.streampark.common.util.Utils; -import org.apache.streampark.flink.connector.function.RunningFunction; -import org.apache.streampark.flink.connector.hbase.function.HBaseQueryFunction; -import org.apache.streampark.flink.connector.hbase.function.HBaseResultFunction; -import org.apache.streampark.flink.connector.hbase.internal.HBaseSourceFunction; -import org.apache.streampark.flink.core.scala.StreamingContext; - -import org.apache.flink.streaming.api.datastream.DataStreamSource; - -import java.util.Properties; - -public class HBaseJavaSource { - private final StreamingContext context; - private final Properties property; - - public HBaseJavaSource(StreamingContext context, Properties property) { - this.context = context; - this.property = property; - } - - public DataStreamSource getDataStream( - HBaseQueryFunction queryFunction, - HBaseResultFunction resultFunction, - RunningFunction runningFunc) { - - Utils.requireNotNull(queryFunction, "QueryFunction must not be null"); - Utils.requireNotNull(resultFunction, "ResultFunction must not be null"); - HBaseSourceFunction sourceFunction = - new HBaseSourceFunction<>(property, queryFunction, resultFunction, runningFunc, null); - return context.getJavaEnv().addSource(sourceFunction); - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/internal/HBaseSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/internal/HBaseSinkFunction.scala deleted file mode 100644 index e136607da2..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/internal/HBaseSinkFunction.scala +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.hbase.internal - -import org.apache.streampark.common.conf.ConfigConst.{DEFAULT_HBASE_COMMIT_BATCH, DEFAULT_HBASE_WRITE_SIZE, KEY_HBASE_COMMIT_BATCH, KEY_HBASE_WRITE_SIZE} -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.enums.ApiType.ApiType -import org.apache.streampark.common.util.{HBaseClient, Logger} -import org.apache.streampark.flink.connector.function.TransformFunction - -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction} -import org.apache.hadoop.hbase.TableName -import org.apache.hadoop.hbase.client._ - -import java.lang.{Iterable => JIter} -import java.util.Properties -import java.util.concurrent.{Executors, ScheduledExecutorService, TimeUnit} -import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong} - -import org.apache.streampark.common.util.Implicits._ -import scala.collection.mutable.ArrayBuffer - -class HBaseSinkFunction[T](apiType: ApiType = ApiType.scala, tabName: String, prop: Properties) - extends RichSinkFunction[T] - with Logger { - - private var connection: Connection = _ - private var table: Table = _ - private var mutator: BufferedMutator = _ - private val offset: AtomicLong = new AtomicLong(0L) - private val scheduled: AtomicBoolean = new AtomicBoolean(false) - private var timestamp = 0L - - private val commitBatch = - prop.getOrElse(KEY_HBASE_COMMIT_BATCH, s"$DEFAULT_HBASE_COMMIT_BATCH").toInt - private val writeBufferSize = - prop.getOrElse(KEY_HBASE_WRITE_SIZE, s"$DEFAULT_HBASE_WRITE_SIZE").toLong - - private val mutations = new ArrayBuffer[Mutation]() - private val putArray = new ArrayBuffer[Put]() - - private[this] var scalaTransformFunc: T => JIter[Mutation] = _ - private[this] var javaTransformFunc: TransformFunction[T, JIter[Mutation]] = _ - - // for Scala - def this(tabName: String, properties: Properties, scalaTransformFunc: T => JIter[Mutation]) = { - - this(ApiType.scala, tabName, properties) - this.scalaTransformFunc = scalaTransformFunc - } - - // for JAVA - def this( - tabName: String, - properties: Properties, - javaTransformFunc: TransformFunction[T, JIter[Mutation]]) = { - - this(ApiType.java, tabName, properties) - this.javaTransformFunc = javaTransformFunc - } - - @transient private var service: ScheduledExecutorService = _ - - override def open(parameters: Configuration): Unit = { - service = Executors.newSingleThreadScheduledExecutor() - connection = HBaseClient(prop).connection - val tableName = TableName.valueOf(tabName) - val mutatorParam = new BufferedMutatorParams(tableName) - .writeBufferSize(writeBufferSize) - .listener(new BufferedMutator.ExceptionListener { - override def onException( - exception: RetriesExhaustedWithDetailsException, - mutator: BufferedMutator): Unit = { - for (i <- 0.until(exception.getNumExceptions)) { - logger.error( - s"[StreamPark] HBaseSink Failed to sent put ${exception.getRow(i)},error:${exception.getLocalizedMessage}") - } - } - }) - mutator = connection.getBufferedMutator(mutatorParam) - table = connection.getTable(tableName) - } - - override def invoke(value: T, context: SinkFunction.Context): Unit = { - val list = apiType match { - case ApiType.java => javaTransformFunc.transform(value) - case ApiType.scala => scalaTransformFunc(value) - } - - list.foreach { - case put: Put => putArray += put - case other => mutations += other - } - - offset.incrementAndGet() % commitBatch match { - case 0 => execBatch() - case _ => - if (!scheduled.get()) { - scheduled.set(true) - service.schedule( - new Runnable { - override def run(): Unit = { - scheduled.set(false) - execBatch() - } - }, - 10, - TimeUnit.SECONDS) - } - } - - } - - override def close(): Unit = { - execBatch() - if (mutator != null) { - mutator.flush() - mutator.close() - } - if (table != null) { - table.close() - } - } - - private[this] def execBatch(): Unit = { - if (offset.get() > 0) { - val start = System.currentTimeMillis() - // put ... - mutator.mutate(putArray) - mutator.flush() - putArray.clear() - // mutation... - if (mutations.nonEmpty) { - table.batch(mutations, new Array[AnyRef](mutations.length)) - logInfo( - s"HBaseSink batchSize:${mutations.length} use ${System.currentTimeMillis() - start} MS") - mutations.clear() - } - offset.set(0L) - timestamp = System.currentTimeMillis() - } - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/internal/HBaseSourceFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/internal/HBaseSourceFunction.scala deleted file mode 100644 index d91c168b94..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/internal/HBaseSourceFunction.scala +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.hbase.internal - -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.enums.ApiType.ApiType -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.function.RunningFunction -import org.apache.streampark.flink.connector.hbase.bean.HBaseQuery -import org.apache.streampark.flink.connector.hbase.function.{HBaseQueryFunction, HBaseResultFunction} -import org.apache.streampark.flink.util.FlinkUtils - -import org.apache.flink.api.common.state.ListState -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.runtime.state.{CheckpointListener, FunctionInitializationContext, FunctionSnapshotContext} -import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction -import org.apache.flink.streaming.api.functions.source.RichSourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.hadoop.hbase.client.{Result, Table} - -import java.lang -import java.util.Properties - -import org.apache.streampark.common.util.Implicits._ -import scala.util.{Success, Try} - -class HBaseSourceFunction[R: TypeInformation](apiType: ApiType = ApiType.scala, prop: Properties) - extends RichSourceFunction[R] - with CheckpointedFunction - with CheckpointListener - with Logger { - - @volatile private[this] var running = true - private[this] var scalaRunningFunc: Unit => Boolean = _ - private[this] var javaRunningFunc: RunningFunction = _ - - @transient private[this] var table: Table = _ - - @volatile var query: HBaseQuery = _ - - private[this] var scalaQueryFunc: R => HBaseQuery = _ - private[this] var scalaResultFunc: Result => R = _ - - private[this] var javaQueryFunc: HBaseQueryFunction[R] = _ - private[this] var javaResultFunc: HBaseResultFunction[R] = _ - - @transient private var state: ListState[R] = _ - private val OFFSETS_STATE_NAME: String = "hbase-source-query-states" - private[this] var last: R = _ - - // for Scala - def this( - prop: Properties, - queryFunc: R => HBaseQuery, - resultFunc: Result => R, - runningFunc: Unit => Boolean) = { - - this(ApiType.scala, prop) - this.scalaQueryFunc = queryFunc - this.scalaResultFunc = resultFunc - this.scalaRunningFunc = if (runningFunc == null) _ => true else runningFunc - - } - - // for JAVA - def this( - prop: Properties, - queryFunc: HBaseQueryFunction[R], - resultFunc: HBaseResultFunction[R], - runningFunc: RunningFunction) { - - this(ApiType.java, prop) - this.javaQueryFunc = queryFunc - this.javaResultFunc = resultFunc - this.javaRunningFunc = - if (runningFunc != null) runningFunc - else - new RunningFunction { - override def running(): lang.Boolean = true - } - - } - - @throws[Exception] - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def run(ctx: SourceContext[R]): Unit = { - while (this.running) { - apiType match { - case ApiType.scala => - if (scalaRunningFunc()) { - ctx.getCheckpointLock.synchronized { - // Returns the query object of the last (or recovered from checkpoint) query to the user, and the user constructs the conditions for the next query based on this. - query = scalaQueryFunc(last) - require( - query != null && query.getTable != null, - "[StreamPark] HBaseSource query and query's param table must not be null ") - table = query.getTable(prop) - table - .getScanner(query) - .foreach( - x => { - last = scalaResultFunc(x) - ctx.collectWithTimestamp(last, System.currentTimeMillis()) - }) - } - } - case ApiType.java => - if (javaRunningFunc.running()) { - ctx.getCheckpointLock.synchronized { - // Returns the query object of the last (or recovered from checkpoint) query to the user, and the user constructs the conditions for the next query based on this. - query = javaQueryFunc.query(last) - require( - query != null && query.getTable != null, - "[StreamPark] HBaseSource query and query's param table must not be null ") - table = query.getTable(prop) - table - .getScanner(query) - .foreach( - x => { - last = javaResultFunc.result(x) - ctx.collectWithTimestamp(last, System.currentTimeMillis()) - }) - } - } - } - } - } - - override def cancel(): Unit = this.running = false - - override def close(): Unit = { - super.close() - if (table != null) { - table.close() - } - } - - override def snapshotState(context: FunctionSnapshotContext): Unit = { - if (running) { - state.clear() - if (last != null) { - state.add(last) - } - } else { - logError("HBaseSource snapshotState called on closed source") - } - } - - override def initializeState(context: FunctionInitializationContext): Unit = { - // Recover from checkpoint... - logInfo("HBaseSource snapshotState initialize") - state = FlinkUtils.getUnionListState[R](context, OFFSETS_STATE_NAME) - Try(state.get.head) match { - case Success(q) => last = q - case _ => - } - } - - override def notifyCheckpointComplete(checkpointId: Long): Unit = { - logInfo(s"HBaseSource checkpointComplete: $checkpointId") - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/request/HBaseRequest.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/request/HBaseRequest.scala deleted file mode 100644 index 03d8759968..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/request/HBaseRequest.scala +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.hbase.request - -import org.apache.streampark.common.util.{Logger, Utils} -import org.apache.streampark.flink.connector.hbase.bean.HBaseQuery - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.scala._ -import org.apache.flink.streaming.api.scala.async.{ResultFuture, RichAsyncFunction} -import org.apache.hadoop.hbase.client.{Result, ResultScanner, Table} - -import java.util.Properties -import java.util.concurrent.{CompletableFuture, Executors, ExecutorService, TimeUnit} -import java.util.function.{Consumer, Supplier} - -import scala.annotation.meta.param -import org.apache.streampark.common.util.Implicits._ - -object HBaseRequest { - - def apply[T: TypeInformation]( - @(transient @param) stream: DataStream[T], - property: Properties = new Properties()): HBaseRequest[T] = - new HBaseRequest[T](stream, property) - -} - -class HBaseRequest[T: TypeInformation]( - @(transient @param) private val stream: DataStream[T], - property: Properties = new Properties()) { - - def requestOrdered[R: TypeInformation]( - queryFunc: T => HBaseQuery, - resultFunc: (T, Result) => R, - timeout: Long = 1000, - capacity: Int = 10)(implicit prop: Properties): DataStream[R] = { - Utils.copyProperties(property, prop) - val async = new HBaseAsyncFunction[T, R](prop, queryFunc, resultFunc, capacity) - AsyncDataStream.orderedWait(stream, async, timeout, TimeUnit.MILLISECONDS, capacity) - } - - def requestUnordered[R: TypeInformation]( - queryFunc: T => HBaseQuery, - resultFunc: (T, Result) => R, - timeout: Long = 1000, - capacity: Int = 10)(implicit prop: Properties): DataStream[R] = { - Utils.copyProperties(property, prop) - val async = new HBaseAsyncFunction[T, R](prop, queryFunc, resultFunc, capacity) - AsyncDataStream.unorderedWait(stream, async, timeout, TimeUnit.MILLISECONDS, capacity) - } - -} - -class HBaseAsyncFunction[T: TypeInformation, R: TypeInformation]( - prop: Properties, - queryFunc: T => HBaseQuery, - resultFunc: (T, Result) => R, - capacity: Int) - extends RichAsyncFunction[T, R] - with Logger { - @transient private[this] var table: Table = _ - @transient private[this] var executorService: ExecutorService = _ - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - executorService = Executors.newFixedThreadPool(capacity) - } - - override def asyncInvoke(input: T, resultFuture: async.ResultFuture[R]): Unit = { - CompletableFuture - .supplyAsync( - new Supplier[ResultScanner]() { - override def get(): ResultScanner = { - val query = queryFunc(input) - require( - query != null && query.getTable != null, - "[StreamPark] HBaseRequest query and query's attr table must not be null ") - table = query.getTable(prop) - table.getScanner(query) - } - }, - executorService - ) - .thenAccept(new Consumer[ResultScanner] { - override def accept(result: ResultScanner): Unit = { - val list = result.toList - if (list.isEmpty) { - resultFuture.complete(List(resultFunc(input, Result.EMPTY_RESULT))) - } else { - resultFuture.complete(list.map(r => resultFunc(input, r))) - } - } - }) - } - - override def timeout(input: T, resultFuture: ResultFuture[R]): Unit = { - logWarn("HBaseASync request timeout. retrying... ") - asyncInvoke(input, resultFuture) - } - - override def close(): Unit = { - super.close() - table.close() - if (!executorService.isShutdown) { - executorService.shutdown() - } - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/sink/HBaseOutputFormat.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/sink/HBaseOutputFormat.scala deleted file mode 100644 index ef5771a05f..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/sink/HBaseOutputFormat.scala +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.hbase.sink - -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.function.TransformFunction -import org.apache.streampark.flink.connector.hbase.internal.HBaseSinkFunction - -import org.apache.flink.api.common.io.RichOutputFormat -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.hadoop.hbase.client.Mutation - -import java.lang.{Iterable => JIter} -import java.util.Properties - -class HBaseOutputFormat[T: TypeInformation](tabName: String, prop: Properties) - extends RichOutputFormat[T] - with Logger { - - var sinkFunction: HBaseSinkFunction[T] = _ - - // for Scala - def this(tabName: String, properties: Properties, scalaTransformFunc: T => JIter[Mutation]) = { - this(tabName, properties) - this.sinkFunction = new HBaseSinkFunction[T](tabName, properties, scalaTransformFunc) - } - - // for JAVA - def this( - tabName: String, - properties: Properties, - javaTransformFunc: TransformFunction[T, JIter[Mutation]]) = { - - this(tabName, properties) - this.sinkFunction = new HBaseSinkFunction[T](tabName, properties, javaTransformFunc) - } - - var configuration: Configuration = _ - - override def configure(configuration: Configuration): Unit = this.configuration = configuration - - override def open(taskNumber: Int, numTasks: Int): Unit = sinkFunction.open(this.configuration) - - override def writeRecord(record: T): Unit = sinkFunction.invoke(record, null) - - override def close(): Unit = sinkFunction.close() -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/sink/HBaseSink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/sink/HBaseSink.scala deleted file mode 100644 index 43061a151e..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/sink/HBaseSink.scala +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.hbase.sink - -import org.apache.streampark.common.conf.ConfigConst._ -import org.apache.streampark.common.util.{ConfigUtils, Logger, Utils} -import org.apache.streampark.flink.connector.function.TransformFunction -import org.apache.streampark.flink.connector.hbase.internal.HBaseSinkFunction -import org.apache.streampark.flink.connector.sink.Sink -import org.apache.streampark.flink.core.scala.StreamingContext - -import org.apache.flink.streaming.api.datastream.{DataStream => JavaDataStream, DataStreamSink} -import org.apache.flink.streaming.api.scala.DataStream -import org.apache.hadoop.hbase.client._ - -import java.lang.{Iterable => JIter} -import java.util.Properties - -import scala.annotation.meta.param - -object HBaseSink { - - def apply( - @(transient @param) - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null)(implicit ctx: StreamingContext): HBaseSink = - new HBaseSink(ctx, property, parallelism, name, uid) - -} - -class HBaseSink( - @(transient @param) ctx: StreamingContext, - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null)(implicit alias: String = "") - extends Sink - with Logger { - - def this(ctx: StreamingContext) { - this(ctx, new Properties, 0, null, null) - } - - def sink[T](stream: DataStream[T], tableName: String)(implicit - fun: T => JIter[Mutation]): DataStreamSink[T] = { - val prop: Properties = checkProp(stream, tableName, fun) - val sinkFun = new HBaseSinkFunction[T](tableName, prop, fun) - val sink = stream.addSink(sinkFun) - afterSink(sink, parallelism, name, uid) - } - - def sink[T]( - stream: JavaDataStream[T], - tableName: String, - fun: TransformFunction[T, JIter[Mutation]]): DataStreamSink[T] = { - val prop: Properties = checkProp(stream, tableName, fun) - val sinkFun = new HBaseSinkFunction[T](tableName, prop, fun) - val sink = stream.addSink(sinkFun) - afterSink(sink, parallelism, name, uid) - } - - private def checkProp[T](stream: Object, tableName: String, fun: Object): Properties = { - implicit val prop: Properties = - ConfigUtils.getConf(ctx.parameter.toMap, HBASE_PREFIX, HBASE_PREFIX)(alias) - Utils.copyProperties(property, prop) - require(stream != null, () => s"Sink Stream must not null") - require(tableName != null, () => s"Sink tableName must not null") - require(fun != null, () => s"Func must not null") - prop - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/source/HBaseSource.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/source/HBaseSource.scala deleted file mode 100644 index 726ff74ee6..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-hbase/src/main/scala/org/apache/streampark/flink/connector/hbase/source/HBaseSource.scala +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.hbase.source - -import org.apache.streampark.common.util.Utils -import org.apache.streampark.flink.connector.hbase.bean.HBaseQuery -import org.apache.streampark.flink.connector.hbase.internal.HBaseSourceFunction -import org.apache.streampark.flink.core.scala.StreamingContext - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.streaming.api.scala.DataStream -import org.apache.hadoop.hbase.client._ - -import java.util.Properties - -import scala.annotation.meta.param - -object HBaseSource { - - def apply(@(transient @param) property: Properties = new Properties())(implicit - ctx: StreamingContext): HBaseSource = new HBaseSource(ctx, property) - -} - -/** - * Support end-to-end exactly once and replay - * - * @param ctx - * @param property - */ -class HBaseSource( - @(transient @param) val ctx: StreamingContext, - property: Properties = new Properties()) { - - def getDataStream[R: TypeInformation]( - query: R => HBaseQuery, - func: Result => R, - running: Unit => Boolean)(implicit prop: Properties = new Properties()): DataStream[R] = { - Utils.copyProperties(property, prop) - val hBaseFunc = new HBaseSourceFunction[R](prop, query, func, running) - ctx.addSource(hBaseFunc) - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/pom.xml deleted file mode 100644 index 3e37c22d69..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/pom.xml +++ /dev/null @@ -1,93 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink-connector - 2.2.0-SNAPSHOT - - - streampark-flink-connector-http_${scala.binary.version} - StreamPark : Flink Connector Http - - - - - org.apache.streampark - streampark-flink-connector-base_${scala.binary.version} - ${version} - - - - - org.asynchttpclient - async-http-client - - - - org.apache.httpcomponents - httpclient - - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/conf/HttpConfigOption.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/conf/HttpConfigOption.scala deleted file mode 100644 index 8b7166ff75..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/conf/HttpConfigOption.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.http.conf - -import java.util.Properties - -object HttpConfigOption { - - val HTTP_SINK_PREFIX: String = "http.sink" - - /** - * @param properties - * @return - */ - def apply( - prefixStr: String = HTTP_SINK_PREFIX, - properties: Properties = new Properties): HttpConfigOption = - new HttpConfigOption(prefixStr, properties) - -} - -class HttpConfigOption(prefixStr: String, properties: Properties) { - - implicit val (prefix, prop) = (prefixStr, properties) - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/function/HttpSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/function/HttpSinkFunction.scala deleted file mode 100644 index 5bfbe42f61..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/function/HttpSinkFunction.scala +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.http.function - -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.conf.ThresholdConf -import org.apache.streampark.flink.connector.failover.{FailoverChecker, SinkBuffer} -import org.apache.streampark.flink.connector.http.conf.HttpConfigOption -import org.apache.streampark.flink.connector.http.internal.HttpSinkWriter - -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction - -import java.util.Properties -import java.util.concurrent.locks.ReentrantLock - -import scala.collection.mutable - -class HttpSinkFunction( - properties: mutable.Map[String, String], - header: Map[String, String], - method: String) - extends RichSinkFunction[String] - with Logger { - - private[this] object Lock { - @volatile var initialized = false - val lock = new ReentrantLock() - } - - @transient var sinkBuffer: SinkBuffer = _ - @transient var thresholdConf: ThresholdConf = _ - @transient var httpSinkWriter: HttpSinkWriter = _ - @transient var failoverChecker: FailoverChecker = _ - @volatile var isClosed: Boolean = false - - override def open(config: Configuration): Unit = { - if (!Lock.initialized) { - Lock.lock.lock() - if (!Lock.initialized) { - Lock.initialized = true - - val prop: Properties = new Properties() - properties.foreach { case (k, v) => prop.put(k, v) } - thresholdConf = ThresholdConf(HttpConfigOption.HTTP_SINK_PREFIX, prop) - val bufferSize = 1 - val table = thresholdConf.failoverTable - require( - table != null && table.nonEmpty, - () => s"Http async insert failoverTable must not null") - - httpSinkWriter = HttpSinkWriter(thresholdConf, header) - failoverChecker = FailoverChecker(thresholdConf.delayTime) - sinkBuffer = SinkBuffer(httpSinkWriter, thresholdConf.delayTime, bufferSize) - failoverChecker.addSinkBuffer(sinkBuffer) - logInfo("HttpSink initialize... ") - } - Lock.lock.unlock() - } - } - - override def invoke(url: String): Unit = { - sinkBuffer.put(s"$method///$url") - } - - override def close(): Unit = { - if (!isClosed) { - Lock.lock.synchronized { - if (!isClosed) { - if (sinkBuffer != null) sinkBuffer.close() - if (httpSinkWriter != null) httpSinkWriter.close() - if (failoverChecker != null) failoverChecker.close() - isClosed = true - super.close() - } - } - } - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/internal/HttpSinkWriter.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/internal/HttpSinkWriter.scala deleted file mode 100644 index d3ee49947c..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/internal/HttpSinkWriter.scala +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.http.internal - -import org.apache.streampark.common.util.{Logger, ThreadUtils} -import org.apache.streampark.flink.connector.conf.ThresholdConf -import org.apache.streampark.flink.connector.failover.{SinkRequest, SinkWriter} - -import org.asynchttpclient.{AsyncHttpClient, Dsl} - -import java.util.concurrent._ - -import org.apache.streampark.common.util.Implicits._ -import scala.collection.mutable.ListBuffer - -case class HttpSinkWriter(thresholdConf: ThresholdConf, header: Map[String, String]) - extends SinkWriter - with Logger { - private val callbackServiceFactory = - ThreadUtils.threadFactory("HttpSink-writer-callback-executor") - private val threadFactory: ThreadFactory = ThreadUtils.threadFactory("HttpSink-writer") - - var callbackService: ExecutorService = new ThreadPoolExecutor( - math.max(Runtime.getRuntime.availableProcessors / 4, 2), - Integer.MAX_VALUE, - 60L, - TimeUnit.SECONDS, - new LinkedBlockingQueue[Runnable], - callbackServiceFactory) - - var tasks: ListBuffer[HttpWriterTask] = ListBuffer[HttpWriterTask]() - var recordQueue: BlockingQueue[SinkRequest] = - new LinkedBlockingQueue[SinkRequest](thresholdConf.queueCapacity) - var asyncHttpClient: AsyncHttpClient = Dsl.asyncHttpClient - - var service: ExecutorService = - Executors.newFixedThreadPool(thresholdConf.numWriters, threadFactory) - - for (i <- 0 until thresholdConf.numWriters) { - val task = - HttpWriterTask(i, thresholdConf, asyncHttpClient, header, recordQueue, callbackService) - tasks.add(task) - service.submit(task) - } - - def write(request: SinkRequest): Unit = - try { - recordQueue.put(request) - } catch { - case e: InterruptedException => - logError(s"Interrupted error while putting data to queue,error:$e") - Thread.currentThread.interrupt() - throw new RuntimeException(e) - } - - override def close(): Unit = { - logInfo("Closing HttpSink-writer...") - tasks.foreach(_.close()) - ThreadUtils.shutdownExecutorService(service) - ThreadUtils.shutdownExecutorService(callbackService) - asyncHttpClient.close() - logInfo(s"${classOf[HttpSinkWriter].getSimpleName} is closed") - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/internal/HttpWriterTask.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/internal/HttpWriterTask.scala deleted file mode 100644 index 09dd4ad9c9..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/internal/HttpWriterTask.scala +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.http.internal - -import org.apache.streampark.common.util.{JsonUtils, Logger} -import org.apache.streampark.flink.connector.conf.ThresholdConf -import org.apache.streampark.flink.connector.failover.{FailoverWriter, SinkRequest} - -import io.netty.handler.codec.http.HttpHeaders -import org.apache.http.client.methods._ -import org.asynchttpclient.{AsyncHttpClient, ListenableFuture, Request, Response} - -import java.util -import java.util.concurrent.{BlockingQueue, ExecutorService, TimeUnit} - -import org.apache.streampark.common.util.Implicits._ -import scala.util.Try - -case class HttpWriterTask( - id: Int, - thresholdConf: ThresholdConf, - asyncHttpClient: AsyncHttpClient, - header: Map[String, String], - queue: BlockingQueue[SinkRequest], - callbackService: ExecutorService) - extends Runnable - with AutoCloseable - with Logger { - - @volatile var isWorking = false - - val httpMethods: List[String] = List[String]( - HttpGet.METHOD_NAME, - HttpPost.METHOD_NAME, - HttpPut.METHOD_NAME, - HttpPatch.METHOD_NAME, - HttpDelete.METHOD_NAME, - HttpOptions.METHOD_NAME, - HttpTrace.METHOD_NAME - ) - - val failoverWriter: FailoverWriter = - new FailoverWriter(thresholdConf.storageType, thresholdConf.getFailoverConfig) - - def buildRequest(url: String): Request = { - - val method = httpMethods.filter(x => url.startsWith(x)).head - - val uriAndParams = url.drop(method.length + 3).split("\\?") - - val uri = uriAndParams.head - - val builder = method match { - case HttpGet.METHOD_NAME => asyncHttpClient.prepareGet(uri) - case HttpDelete.METHOD_NAME => asyncHttpClient.prepareDelete(uri) - case HttpOptions.METHOD_NAME => asyncHttpClient.prepareOptions(uri) - case HttpTrace.METHOD_NAME => asyncHttpClient.prepareTrace(uri) - case HttpPost.METHOD_NAME => asyncHttpClient.preparePost(uri) - case HttpPatch.METHOD_NAME => asyncHttpClient.preparePatch(uri) - case HttpPut.METHOD_NAME => asyncHttpClient.preparePut(uri) - } - if (header != null && header.nonEmpty) { - header.foreach { case (k, v) => builder.setHeader(k, v) } - } - - Try(uriAndParams(1).trim).getOrElse(null) match { - case null => - case params => - val paramMap = new util.HashMap[String, String]() - params - .split("&") - .foreach( - x => { - val param = x.split("=") - paramMap.put(param.head, param.last) - }) - if (paramMap.nonEmpty) { - builder.setHeader(HttpHeaders.Names.CONTENT_TYPE, HttpHeaders.Values.APPLICATION_JSON) - val json = JsonUtils.write(paramMap) - builder.setBody(json.getBytes) - } - } - builder.setRequestTimeout(thresholdConf.timeout).build() - } - - override def run(): Unit = - try { - isWorking = true - logInfo(s"Start writer task, id = $id") - while (isWorking || queue.nonEmpty) { - val req = queue.poll(100, TimeUnit.MILLISECONDS) - if (req != null) { - val url = req.records.head - val sinkRequest = SinkRequest(List(url), req.attemptCounter) - val request = buildRequest(url) - val whenResponse = asyncHttpClient.executeRequest(request) - val callback = respCallback(whenResponse, sinkRequest) - whenResponse.addListener(callback, callbackService) - if (req.attemptCounter > 0) { - logInfo(s"get retry url from queue,attemptCounter:${req.attemptCounter}") - } - } - } - } catch { - case e: Exception => - logError("Error while inserting data", e) - throw new RuntimeException(e) - } finally { - logInfo(s"Task id = $id is finished") - } - - def respCallback(whenResponse: ListenableFuture[Response], sinkRequest: SinkRequest): Runnable = - new Runnable { - override def run(): Unit = { - Try(whenResponse.get()).getOrElse(null) match { - case null => - logError( - s"""Error HttpSink executing callback, params = $thresholdConf,can not get Response. """) - handleFailedResponse(null, sinkRequest) - case resp if resp.getStatusCode != 200 => - logError( - s"""Error HttpSink executing callback, params = $thresholdConf, StatusCode = ${resp.getStatusCode} """) - handleFailedResponse(resp, sinkRequest) - case _ => - } - } - } - - /** - * if send data to Http Failed, retry maxRetries, if still failed,flush data to failoverStorage - * - * @param response - * @param sinkRequest - */ - def handleFailedResponse(response: Response, sinkRequest: SinkRequest): Unit = - try { - if (sinkRequest.attemptCounter >= thresholdConf.maxRetries) { - failoverWriter.write( - sinkRequest.copy(records = sinkRequest.records.map(_.replaceFirst("^[A-Z]+///", "")))) - logWarn( - s"""Failed to send data to Http, Http response = $response. Ready to flush data to ${thresholdConf.storageType}""") - } else { - sinkRequest.incrementCounter() - logWarn( - s"Next attempt to send data to Http, table = ${sinkRequest.table}, buffer size = ${sinkRequest.size}, current attempt num = ${sinkRequest.attemptCounter}, max attempt num = ${thresholdConf.maxRetries}, response = $response") - queue.put(sinkRequest) - } - } catch { - case e: Exception => new RuntimeException(s"[StreamPark] handleFailedResponse,error:$e") - } - - override def close(): Unit = { - isWorking = false - failoverWriter.close() - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/sink/HttpSink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/sink/HttpSink.scala deleted file mode 100644 index 475f6e4abf..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-http/src/main/scala/org/apache/streampark/flink/connector/http/sink/HttpSink.scala +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.http.sink - -import org.apache.streampark.common.util.Utils -import org.apache.streampark.flink.connector.http.function.HttpSinkFunction -import org.apache.streampark.flink.connector.sink.Sink -import org.apache.streampark.flink.core.scala.StreamingContext - -import org.apache.flink.streaming.api.datastream.{DataStream => JavaDataStream, DataStreamSink} -import org.apache.flink.streaming.api.scala.DataStream -import org.apache.http.client.methods._ - -import java.util.Properties - -import scala.annotation.meta.param -import org.apache.streampark.common.util.Implicits._ - -object HttpSink { - - def apply( - @(transient @param) - header: Map[String, String] = Map.empty[String, String], - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null)(implicit ctx: StreamingContext): HttpSink = - new HttpSink(ctx, property, header, parallelism, name, uid) - -} - -class HttpSink( - @(transient @param) ctx: StreamingContext, - property: Properties = new Properties(), - header: Map[String, String] = Map.empty[String, String], - parallelism: Int = 0, - name: String = null, - uid: String = null) - extends Sink { - - // for java - def this(ctx: StreamingContext) { - this(ctx, new Properties(), Map.empty[String, String], 0, null, null) - } - - val prop: Properties = ctx.parameter.getProperties - Utils.copyProperties(property, prop) - - def get(stream: DataStream[String]): DataStreamSink[String] = sink(stream, HttpGet.METHOD_NAME) - - def get(stream: JavaDataStream[String]): DataStreamSink[String] = - sink(new DataStream[String](stream), HttpGet.METHOD_NAME) - - def post(stream: DataStream[String]): DataStreamSink[String] = sink(stream, HttpPost.METHOD_NAME) - - def post(stream: JavaDataStream[String]): DataStreamSink[String] = - sink(new DataStream[String](stream), HttpPost.METHOD_NAME) - - def patch(stream: DataStream[String]): DataStreamSink[String] = - sink(stream, HttpPatch.METHOD_NAME) - - def patch(stream: JavaDataStream[String]): DataStreamSink[String] = - sink(new DataStream[String](stream), HttpPatch.METHOD_NAME) - - def put(stream: DataStream[String]): DataStreamSink[String] = sink(stream, HttpPut.METHOD_NAME) - - def put(stream: JavaDataStream[String]): DataStreamSink[String] = - sink(new DataStream[String](stream), HttpPut.METHOD_NAME) - - def delete(stream: DataStream[String]): DataStreamSink[String] = - sink(stream, HttpDelete.METHOD_NAME) - - def delete(stream: JavaDataStream[String]): DataStreamSink[String] = - sink(new DataStream[String](stream), HttpDelete.METHOD_NAME) - - def options(stream: DataStream[String]): DataStreamSink[String] = - sink(stream, HttpOptions.METHOD_NAME) - - def options(stream: JavaDataStream[String]): DataStreamSink[String] = - sink(new DataStream[String](stream), HttpOptions.METHOD_NAME) - - def trace(stream: DataStream[String]): DataStreamSink[String] = - sink(stream, HttpTrace.METHOD_NAME) - - def trace(stream: JavaDataStream[String]): DataStreamSink[String] = - sink(new DataStream[String](stream), HttpTrace.METHOD_NAME) - - private[this] def sink(stream: DataStream[String], method: String): DataStreamSink[String] = { - val sinkFun = new HttpSinkFunction(prop, header, method) - val sink = stream.addSink(sinkFun) - afterSink(sink, parallelism, name, uid) - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/pom.xml deleted file mode 100644 index 3a12f26787..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/pom.xml +++ /dev/null @@ -1,89 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink-connector - 2.2.0-SNAPSHOT - - - streampark-flink-connector-influx_${scala.binary.version} - StreamPark : Flink Connector Influxdb - - - - - org.apache.streampark - streampark-flink-connector-base_${scala.binary.version} - ${project.version} - - - - org.influxdb - influxdb-java - 2.17 - - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/java/org/apache/streampark/flink/connector/influx/function/InfluxFieldFunction.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/java/org/apache/streampark/flink/connector/influx/function/InfluxFieldFunction.java deleted file mode 100644 index fbc2e41356..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/java/org/apache/streampark/flink/connector/influx/function/InfluxFieldFunction.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.influx.function; - -import java.io.Serializable; -import java.util.Map; - -public interface InfluxFieldFunction extends Serializable { - Map transform(T value); -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/java/org/apache/streampark/flink/connector/influx/function/InfluxTagFunction.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/java/org/apache/streampark/flink/connector/influx/function/InfluxTagFunction.java deleted file mode 100644 index 3b96b555d8..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/java/org/apache/streampark/flink/connector/influx/function/InfluxTagFunction.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.influx.function; - -import java.io.Serializable; -import java.util.Map; - -public interface InfluxTagFunction extends Serializable { - Map transform(T value); -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/bean/InfluxEntity.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/bean/InfluxEntity.scala deleted file mode 100644 index b0508e1168..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/bean/InfluxEntity.scala +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.influx.bean - -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.enums.ApiType.ApiType -import org.apache.streampark.flink.connector.influx.function.{InfluxFieldFunction, InfluxTagFunction} - -/** - * @param database - * database - * @param measurement - * measurement - * @param retentionPolicy - * retentionPolicy - * @param tagFun - * tag function - * @param fieldFun - * field function - * @tparam T - */ -case class InfluxEntity[T]( - apiType: ApiType = ApiType.scala, - database: String, - measurement: String, - retentionPolicy: String) { - var scalaTagFun: T => Map[String, String] = _ - var scalaFieldFun: T => Map[String, Object] = _ - - var javaTagFun: InfluxTagFunction[T] = _ - var javaFieldFun: InfluxFieldFunction[T] = _ - - // for java - def this( - database: String, - measurement: String, - retentionPolicy: String, - javaTagFun: InfluxTagFunction[T], - javaFieldFun: InfluxFieldFunction[T]) { - this(ApiType.java, database, measurement, retentionPolicy) - this.javaTagFun = javaTagFun - this.javaFieldFun = javaFieldFun - } - - // for scala - def this( - database: String, - measurement: String, - retentionPolicy: String, - scalaTagFun: T => Map[String, String], - scalaFieldFun: T => Map[String, Object]) { - this(ApiType.scala, database, measurement, retentionPolicy) - this.scalaTagFun = scalaTagFun - this.scalaFieldFun = scalaFieldFun - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/function/InfluxFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/function/InfluxFunction.scala deleted file mode 100644 index 24a73e08de..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/function/InfluxFunction.scala +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.influx.function - -import org.apache.streampark.common.conf.ConfigKeys.{KEY_JDBC_PASSWORD, KEY_JDBC_URL, KEY_JDBC_USER} -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.influx.bean.InfluxEntity - -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction -import org.apache.flink.streaming.api.functions.sink.SinkFunction.Context -import org.influxdb.{InfluxDB, InfluxDBFactory} -import org.influxdb.dto.Point - -import java.util.Properties -import java.util.concurrent.TimeUnit - -import org.apache.streampark.common.util.Implicits._; - -class InfluxFunction[T](config: Properties)(implicit endpoint: InfluxEntity[T]) - extends RichSinkFunction[T] - with Logger { - - var influxDB: InfluxDB = _ - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - val url = config.getOrElse(KEY_JDBC_URL, null) - require(url != null) - val username = config.getOrElse(KEY_JDBC_USER, null) - val password = config.getOrElse(KEY_JDBC_PASSWORD, null) - influxDB = (username, password, url) match { - case (null, _, u) => InfluxDBFactory.connect(u) - case _ => InfluxDBFactory.connect(url, username, password) - } - influxDB.enableBatch(2000, 100, TimeUnit.MILLISECONDS) - } - - override def invoke(value: T, context: Context): Unit = { - val tag: JavaMap[String, String] = endpoint.apiType match { - case ApiType.java => endpoint.javaTagFun.transform(value) - case ApiType.scala => endpoint.scalaTagFun(value) - } - val fields: JavaMap[String, Object] = endpoint.apiType match { - case ApiType.java => endpoint.javaFieldFun.transform(value) - case ApiType.scala => endpoint.scalaFieldFun(value) - } - val point = Point - .measurement(endpoint.measurement) - .time(System.currentTimeMillis(), TimeUnit.MILLISECONDS) - .tag(tag) - .fields(fields) - .build() - influxDB.write(endpoint.database, endpoint.retentionPolicy, point) - } - - override def close(): Unit = if (influxDB != null) { - influxDB.flush() - influxDB.close() - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/sink/InfluxOutputFormat.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/sink/InfluxOutputFormat.scala deleted file mode 100644 index 7927a53290..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/sink/InfluxOutputFormat.scala +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.influx.sink - -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.influx.bean.InfluxEntity -import org.apache.streampark.flink.connector.influx.function.InfluxFunction - -import org.apache.flink.api.common.io.RichOutputFormat -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration - -import java.util.Properties - -class InfluxOutputFormat[T: TypeInformation](implicit prop: Properties, endpoint: InfluxEntity[T]) - extends RichOutputFormat[T] - with Logger { - - private val sinkFunction = new InfluxFunction[T](prop) - - private var configuration: Configuration = _ - - override def configure(configuration: Configuration): Unit = this.configuration = configuration - - override def open(taskNumber: Int, numTasks: Int): Unit = sinkFunction.open(this.configuration) - - override def writeRecord(record: T): Unit = sinkFunction.invoke(record, null) - - override def close(): Unit = sinkFunction.close() -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/sink/InfluxSink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/sink/InfluxSink.scala deleted file mode 100644 index 519cf46817..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-influx/src/main/scala/org/apache/streampark/flink/connector/influx/sink/InfluxSink.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.influx.sink - -import org.apache.streampark.common.util.{ConfigUtils, Utils} -import org.apache.streampark.flink.connector.influx.bean.InfluxEntity -import org.apache.streampark.flink.connector.influx.function.InfluxFunction -import org.apache.streampark.flink.connector.sink.Sink -import org.apache.streampark.flink.core.scala.StreamingContext - -import org.apache.flink.streaming.api.datastream.DataStreamSink -import org.apache.flink.streaming.api.scala.DataStream - -import java.util.Properties - -import scala.annotation.meta.param - -object InfluxSink { - - def apply( - @(transient @param) - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null)(implicit ctx: StreamingContext): InfluxSink = - new InfluxSink(ctx, property, parallelism, name, uid) - -} - -class InfluxSink( - @(transient @param) ctx: StreamingContext, - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null) - extends Sink { - - def sink[T](stream: DataStream[T], alias: String = "")(implicit - entity: InfluxEntity[T]): DataStreamSink[T] = { - val prop = ConfigUtils.getInfluxConfig(ctx.parameter.toMap)(alias) - Utils.copyProperties(property, prop) - val sinkFun = new InfluxFunction[T](prop) - val sink = stream.addSink(sinkFun) - afterSink(sink, parallelism, name, uid) - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/pom.xml deleted file mode 100644 index f70e9967fa..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/pom.xml +++ /dev/null @@ -1,83 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink-connector - 2.2.0-SNAPSHOT - - - streampark-flink-connector-jdbc_${scala.binary.version} - StreamPark : Flink Connector JDBC - - - - - org.apache.streampark - streampark-flink-connector-base_${scala.binary.version} - ${project.version} - - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/java/org/apache/streampark/flink/connector/jdbc/sink/JdbcJavaSink.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/java/org/apache/streampark/flink/connector/jdbc/sink/JdbcJavaSink.java deleted file mode 100644 index f49763bae9..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/java/org/apache/streampark/flink/connector/jdbc/sink/JdbcJavaSink.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.jdbc.sink; - -import org.apache.streampark.common.util.ConfigUtils; -import org.apache.streampark.common.util.Utils; -import org.apache.streampark.flink.connector.function.TransformFunction; -import org.apache.streampark.flink.connector.jdbc.internal.JdbcSinkFunction; -import org.apache.streampark.flink.core.scala.StreamingContext; - -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamSink; - -import java.util.Properties; - -public class JdbcJavaSink { - - private final StreamingContext context; - private Properties jdbc; - private TransformFunction sqlFunc; - private String alias = ""; - - public JdbcJavaSink(StreamingContext context) { - this.context = context; - } - - public JdbcJavaSink alias(String alias) { - this.alias = alias; - return this; - } - - public JdbcJavaSink jdbc(Properties jdbc) { - this.jdbc = jdbc; - return this; - } - - public JdbcJavaSink sql(TransformFunction func) { - this.sqlFunc = func; - return this; - } - - public DataStreamSink sink(DataStream dataStream) { - Utils.requireNotNull(sqlFunc, "TransformFunction can not be null"); - this.jdbc = - this.jdbc == null ? ConfigUtils.getJdbcConf(context.parameter().toMap(), alias) : this.jdbc; - JdbcSinkFunction sinkFun = new JdbcSinkFunction<>(this.jdbc, this.sqlFunc); - return dataStream.addSink(sinkFun); - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/java/org/apache/streampark/flink/connector/jdbc/source/JdbcJavaSource.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/java/org/apache/streampark/flink/connector/jdbc/source/JdbcJavaSource.java deleted file mode 100644 index bcc6cb24ad..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/java/org/apache/streampark/flink/connector/jdbc/source/JdbcJavaSource.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.jdbc.source; - -import org.apache.streampark.common.util.ConfigUtils; -import org.apache.streampark.common.util.Utils; -import org.apache.streampark.flink.connector.function.RunningFunction; -import org.apache.streampark.flink.connector.function.SQLQueryFunction; -import org.apache.streampark.flink.connector.function.SQLResultFunction; -import org.apache.streampark.flink.connector.jdbc.internal.JdbcSourceFunction; -import org.apache.streampark.flink.core.scala.StreamingContext; - -import org.apache.flink.streaming.api.datastream.DataStreamSource; - -import java.util.Properties; - -public class JdbcJavaSource { - - private final StreamingContext context; - private Properties jdbc; - private String alias = null; - - public JdbcJavaSource(StreamingContext context) { - this.context = context; - } - - public JdbcJavaSource jdbc(Properties jdbc) { - this.jdbc = jdbc; - return this; - } - - public JdbcJavaSource alias(String alias) { - this.alias = alias; - return this; - } - - public DataStreamSource getDataStream( - SQLQueryFunction queryFunction, - SQLResultFunction resultFunction, - RunningFunction runningFunc) { - - Utils.requireNotNull(queryFunction, "'queryFunction' must not be null"); - Utils.requireNotNull(resultFunction, "'resultFunction' must not be null"); - this.jdbc = - this.jdbc == null ? ConfigUtils.getJdbcConf(context.parameter().toMap(), alias) : this.jdbc; - JdbcSourceFunction sourceFunction = - new JdbcSourceFunction<>(jdbc, queryFunction, resultFunction, runningFunc, null); - return context.getJavaEnv().addSource(sourceFunction); - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/bean/Transaction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/bean/Transaction.scala deleted file mode 100644 index 67153c46b3..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/bean/Transaction.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.jdbc.bean - -import org.apache.streampark.common.util.Utils - -import scala.collection.mutable - -case class Transaction( - transactionId: String = Utils.uuid(), - sql: mutable.MutableList[String] = mutable.MutableList.empty[String], - var insertMode: Boolean = true, - var invoked: Boolean = false) - extends Serializable { - def +(text: String): Unit = sql += text - - override def toString: String = - s"(transactionId:$transactionId,size:${sql.size},insertMode:$insertMode,invoked:$invoked)" -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/Jdbc2PCSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/Jdbc2PCSinkFunction.scala deleted file mode 100644 index 801a0be7e2..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/Jdbc2PCSinkFunction.scala +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.jdbc.internal - -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.enums.ApiType.ApiType -import org.apache.streampark.common.util.{JdbcUtils, Logger} -import org.apache.streampark.flink.connector.function.TransformFunction -import org.apache.streampark.flink.connector.jdbc.bean.Transaction - -import org.apache.flink.api.common.ExecutionConfig -import org.apache.flink.api.common.typeutils.base.VoidSerializer -import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer -import org.apache.flink.streaming.api.functions.sink.{SinkFunction, TwoPhaseCommitSinkFunction} - -import java.sql.{Connection, SQLException, Statement} -import java.util.{Optional, Properties} - -/** - * (flink checkpoint + db transactionId) Simulates commit read. Use of flink's checkpoint mechanism, - * and only submit data that has been confirmed - * - * @param apiType - * @param jdbc - * @tparam T - */ -class Jdbc2PCSinkFunction[T](apiType: ApiType = ApiType.scala, jdbc: Properties) - extends TwoPhaseCommitSinkFunction[T, Transaction, Void]( - new KryoSerializer[Transaction](classOf[Transaction], new ExecutionConfig), - VoidSerializer.INSTANCE) - with Logger { - - private[this] val buffer: collection.mutable.Map[String, Transaction] = - collection.mutable.Map.empty[String, Transaction] - - private var scalaToSQLFn: T => String = _ - private var javaToSQLFunc: TransformFunction[T, String] = _ - - def this(jdbc: Properties, toSQLFn: T => String) { - this(ApiType.scala, jdbc) - this.scalaToSQLFn = toSQLFn - } - - def this(jdbc: Properties, toSQLFn: TransformFunction[T, String]) { - this(ApiType.java, jdbc) - require(toSQLFn != null, "[StreamPark] ToSQLFunction can not be null") - this.javaToSQLFunc = toSQLFn - } - - override def initializeUserContext(): Optional[Void] = super.initializeUserContext() - - override def beginTransaction(): Transaction = { - logInfo("Jdbc2PCSink beginTransaction.") - Transaction() - } - - override def invoke(transaction: Transaction, value: T, context: SinkFunction.Context): Unit = { - val sql = apiType match { - case ApiType.scala => scalaToSQLFn(value) - case ApiType.java => javaToSQLFunc.transform(value) - } - if (!sql.toUpperCase.trim.startsWith("INSERT")) { - transaction.insertMode = false - } - transaction.invoked = true - transaction + sql - } - - /** - * Save the sql statement to be operated in the state. If this step fails, it will be rolled back - * - * @param transaction - */ - override def preCommit(transaction: Transaction): Unit = { - // Prevent called preCommit directly without called the invoke method - if (transaction.invoked) { - logInfo(s"Jdbc2PCSink preCommit.TransactionId:${transaction.transactionId}") - buffer += transaction.transactionId -> transaction - } - } - - /** - * When the data checkpoint is completed or the recovery is finished, this method will be called, - * here use the transaction feature of db. The current operation is at the second stage: If the - * current batch of data is saved successfully, the whole process is successful; If it fails, will - * be thrown an exception, resulting in the checkpoint will also be rolled back, at the same time, - * the next time to started, it will start from the last consumption location, ensuring that - * end-to-end exactly once. - */ - override def commit(transaction: Transaction): Unit = { - if (transaction.invoked && transaction.sql.nonEmpty) { - logInfo(s"Jdbc2PCSink commit,TransactionId:${transaction.transactionId}") - var connection: Connection = null - var statement: Statement = null - try { - connection = JdbcUtils.getConnection(jdbc) - connection.setAutoCommit(false) - statement = connection.createStatement() - if (transaction.insertMode) { - transaction.sql.foreach(statement.addBatch) - statement.executeBatch - statement.clearBatch() - } else { - transaction.sql.foreach(statement.executeUpdate) - } - connection.commit() - // successful, clean state - buffer -= transaction.transactionId - } catch { - case t: Throwable => - logError(s"Jdbc2PCSink commit Exception:${t.getMessage}") - throw t - } finally { - JdbcUtils.close(statement, connection) - } - } - } - - override def abort(transaction: Transaction): Unit = { - logInfo(s"Jdbc2PCSink abort,TransactionId:${transaction.transactionId}") - buffer -= transaction.transactionId - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/JdbcASyncFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/JdbcASyncFunction.scala deleted file mode 100644 index b28e76d86e..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/JdbcASyncFunction.scala +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.jdbc.internal - -import org.apache.streampark.common.util.{JdbcUtils, Logger} - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.scala.async.{ResultFuture, RichAsyncFunction} - -import java.util.Properties -import java.util.concurrent.{CompletableFuture, Executors, ExecutorService} -import java.util.function.{Consumer, Supplier} - -/** - * Based on thread pool - * - * @param sqlFun - * @param resultFun - * @param jdbc - * @tparam T - * @tparam R - */ -class JdbcASyncFunction[T: TypeInformation, R: TypeInformation]( - sqlFun: T => String, - resultFun: (T, Map[String, _]) => R, - jdbc: Properties, - capacity: Int = 10) - extends RichAsyncFunction[T, R] - with Logger { - - @transient private[this] var executorService: ExecutorService = _ - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - executorService = Executors.newFixedThreadPool(capacity) - } - - override def close(): Unit = { - super.close() - if (!executorService.isShutdown) { - executorService.shutdown() - } - } - - @throws[Exception] - def asyncInvoke(input: T, resultFuture: ResultFuture[R]): Unit = { - CompletableFuture - .supplyAsync( - new Supplier[Iterable[Map[String, _]]] { - override def get(): Iterable[Map[String, _]] = JdbcUtils.select(sqlFun(input))(jdbc) - }, - executorService) - .thenAccept(new Consumer[Iterable[Map[String, _]]] { - override def accept(result: Iterable[Map[String, _]]): Unit = { - val list = result.toList - if (list.isEmpty) { - resultFuture.complete(List(resultFun(input, Map.empty[String, Any]))) - } else { - resultFuture.complete(list.map(x => resultFun(input, x))) - } - } - }) - - } - - override def timeout(input: T, resultFuture: ResultFuture[R]): Unit = { - logWarn("JdbcASync request timeout. retrying... ") - asyncInvoke(input, resultFuture) - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/JdbcSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/JdbcSinkFunction.scala deleted file mode 100644 index 6fb365e316..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/JdbcSinkFunction.scala +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.jdbc.internal - -import org.apache.streampark.common.conf.ConfigKeys.{DEFAULT_JDBC_INSERT_BATCH, KEY_JDBC_INSERT_BATCH} -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.enums.ApiType.ApiType -import org.apache.streampark.common.util.{JdbcUtils, Logger} -import org.apache.streampark.flink.connector.function.TransformFunction - -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction} - -import java.sql.{Connection, PreparedStatement, Statement} -import java.util.Properties -import java.util.concurrent.atomic.AtomicLong - -class JdbcSinkFunction[T](apiType: ApiType = ApiType.scala, jdbc: Properties) - extends RichSinkFunction[T] - with Logger { - private var connection: Connection = _ - private var statement: Statement = _ - private var scalaToSQLFn: T => String = _ - private var javaToSQLFunc: TransformFunction[T, String] = _ - private val offset: AtomicLong = new AtomicLong(0L) - private var timestamp: Long = 0L - - private val batchSize = jdbc.remove(KEY_JDBC_INSERT_BATCH) match { - case null => DEFAULT_JDBC_INSERT_BATCH - case batch => batch.toString.toInt - } - - def this(jdbc: Properties, toSQLFn: T => String) { - this(ApiType.scala, jdbc) - this.scalaToSQLFn = toSQLFn - } - - def this(jdbc: Properties, toSQLFn: TransformFunction[T, String]) { - this(ApiType.java, jdbc) - require(toSQLFn != null, "[StreamPark] ToSQLFunction can not be null") - this.javaToSQLFunc = toSQLFn - } - - @throws[Exception] - override def open(parameters: Configuration): Unit = { - require(jdbc != null, "[StreamPark] JdbcSink jdbc can not be null") - logInfo("JdbcSink Open....") - connection = JdbcUtils.getConnection(jdbc) - connection.setAutoCommit(false) - if (batchSize > 1) { - statement = connection.createStatement() - } - } - - override def invoke(value: T, context: SinkFunction.Context): Unit = { - require(connection != null) - val sql = apiType match { - case ApiType.scala => scalaToSQLFn(value) - case ApiType.java => javaToSQLFunc.transform(value) - } - batchSize match { - case 1 => - try { - statement = connection.prepareStatement(sql) - statement.asInstanceOf[PreparedStatement].executeUpdate - connection.commit() - } catch { - case e: Exception => - logError(s"JdbcSink invoke error:$sql") - throw e - case _: Throwable => - } - case batch => - try { - statement.addBatch(sql) - (offset.incrementAndGet() % batch, System.currentTimeMillis()) match { - case (0, _) => execBatch() - case (_, current) if current - timestamp > 1000 => execBatch() - case _ => - } - } catch { - case e: Exception => - logError(s"JdbcSink batch invoke error:$sql") - throw e - case _: Throwable => - } - } - } - - override def close(): Unit = { - execBatch() - JdbcUtils.close(statement, connection) - } - - private[this] def execBatch(): Unit = { - if (offset.get() > 0) { - offset.set(0L) - val start = System.currentTimeMillis() - val count = statement.executeBatch().sum - statement.clearBatch() - connection.commit() - logInfo(s"JdbcSink batch $count use ${System.currentTimeMillis() - start} MS") - timestamp = System.currentTimeMillis() - } - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/JdbcSourceFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/JdbcSourceFunction.scala deleted file mode 100644 index 1ea8390317..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/internal/JdbcSourceFunction.scala +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.jdbc.internal - -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.enums.ApiType.ApiType -import org.apache.streampark.common.util.{JdbcUtils, Logger} -import org.apache.streampark.flink.connector.function.{RunningFunction, SQLQueryFunction, SQLResultFunction} -import org.apache.streampark.flink.util.FlinkUtils - -import org.apache.flink.api.common.state.ListState -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.runtime.state.{CheckpointListener, FunctionInitializationContext, FunctionSnapshotContext} -import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction -import org.apache.flink.streaming.api.functions.source.RichSourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext - -import java.lang -import java.util.Properties - -import scala.collection.Map -import scala.util.{Success, Try} - -class JdbcSourceFunction[R: TypeInformation](apiType: ApiType = ApiType.scala, jdbc: Properties) - extends RichSourceFunction[R] - with CheckpointedFunction - with CheckpointListener - with Logger { - - @volatile private[this] var running = true - private[this] var scalaRunningFunc: Unit => Boolean = (_) => true - private[this] var javaRunningFunc: RunningFunction = _ - - private[this] var scalaSqlFunc: R => String = _ - private[this] var scalaResultFunc: Function[Iterable[Map[String, _]], Iterable[R]] = _ - private[this] var javaSqlFunc: SQLQueryFunction[R] = _ - private[this] var javaResultFunc: SQLResultFunction[R] = _ - @transient private var state: ListState[R] = _ - private val OFFSETS_STATE_NAME: String = "jdbc-source-query-states" - private[this] var last: R = _ - - // for Scala - def this( - jdbc: Properties, - sqlFunc: R => String, - resultFunc: Iterable[Map[String, _]] => Iterable[R], - runningFunc: Unit => Boolean) = { - - this(ApiType.scala, jdbc) - this.scalaSqlFunc = sqlFunc - this.scalaResultFunc = resultFunc - this.scalaRunningFunc = if (runningFunc == null) _ => true else runningFunc - } - - // for JAVA - def this( - jdbc: Properties, - javaSqlFunc: SQLQueryFunction[R], - javaResultFunc: SQLResultFunction[R], - runningFunc: RunningFunction) { - - this(ApiType.java, jdbc) - this.javaSqlFunc = javaSqlFunc - this.javaResultFunc = javaResultFunc - this.javaRunningFunc = - if (runningFunc != null) runningFunc - else - new RunningFunction { - override def running(): lang.Boolean = true - } - } - - @throws[Exception] - override def run(ctx: SourceContext[R]): Unit = { - while (this.running) { - apiType match { - case ApiType.scala => - if (scalaRunningFunc()) { - ctx.getCheckpointLock.synchronized { - val sql = scalaSqlFunc(last) - val result: List[Map[String, _]] = JdbcUtils.select(sql)(jdbc) - scalaResultFunc(result).foreach( - x => { - last = x - ctx.collectWithTimestamp(last, System.currentTimeMillis()) - }) - } - } - case ApiType.java => - if (javaRunningFunc.running()) { - ctx.getCheckpointLock.synchronized { - val sql = javaSqlFunc.query(last) - val result: List[Map[String, _]] = JdbcUtils.select(sql)(jdbc) - javaResultFunc - .result(result.map(_)) - .foreach( - x => { - last = x - ctx.collectWithTimestamp(last, System.currentTimeMillis()) - }) - } - } - } - } - } - - override def cancel(): Unit = this.running = false - - override def snapshotState(context: FunctionSnapshotContext): Unit = { - if (running) { - state.clear() - if (last != null) { - state.add(last) - } - } else { - logError("JdbcSource snapshotState called on closed source") - } - } - - override def initializeState(context: FunctionInitializationContext): Unit = { - logInfo("JdbcSource snapshotState initialize") - state = FlinkUtils.getUnionListState[R](context, OFFSETS_STATE_NAME) - Try(state.get.head) match { - case Success(q) => last = q - case _ => - } - } - - override def notifyCheckpointComplete(checkpointId: Long): Unit = { - logInfo(s"JdbcSource checkpointComplete: $checkpointId") - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/request/JdbcRequest.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/request/JdbcRequest.scala deleted file mode 100644 index 5fd3a13408..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/request/JdbcRequest.scala +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.jdbc.request - -import org.apache.streampark.common.util.Utils -import org.apache.streampark.flink.connector.jdbc.internal.JdbcASyncFunction - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.streaming.api.scala.{AsyncDataStream, DataStream} - -import java.util.Properties -import java.util.concurrent.TimeUnit - -import scala.annotation.meta.param - -object JdbcRequest { - - def apply[T: TypeInformation]( - @(transient @param) stream: DataStream[T], - property: Properties = new Properties()): JdbcRequest[T] = - new JdbcRequest[T](stream, property) - -} - -class JdbcRequest[T: TypeInformation]( - @(transient @param) private val stream: DataStream[T], - property: Properties = new Properties()) { - - /** - * @param sqlFun - * @param jdbc - * @tparam R - * @return - */ - def requestOrdered[R: TypeInformation]( - @(transient @param) sqlFun: T => String, - @(transient @param) resultFun: (T, Map[String, _]) => R, - timeout: Long = 1000, - capacity: Int = 10)(implicit jdbc: Properties): DataStream[R] = { - Utils.copyProperties(property, jdbc) - val async = new JdbcASyncFunction[T, R](sqlFun, resultFun, jdbc) - AsyncDataStream.orderedWait(stream, async, timeout, TimeUnit.MILLISECONDS, capacity) - } - - def requestUnordered[R: TypeInformation]( - @(transient @param) sqlFun: T => String, - @(transient @param) resultFun: (T, Map[String, _]) => R, - timeout: Long = 1000, - capacity: Int = 10)(implicit jdbc: Properties): DataStream[R] = { - Utils.copyProperties(property, jdbc) - val async = new JdbcASyncFunction[T, R](sqlFun, resultFun, jdbc) - AsyncDataStream.unorderedWait(stream, async, timeout, TimeUnit.MILLISECONDS, capacity) - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/sink/JdbcSink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/sink/JdbcSink.scala deleted file mode 100644 index c5ca134e2c..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/sink/JdbcSink.scala +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.jdbc.sink - -import org.apache.streampark.common.conf.ConfigKeys._ -import org.apache.streampark.common.enums.Semantic -import org.apache.streampark.common.util.{ConfigUtils, Logger} -import org.apache.streampark.flink.connector.jdbc.internal.{Jdbc2PCSinkFunction, JdbcSinkFunction} -import org.apache.streampark.flink.connector.sink.Sink -import org.apache.streampark.flink.core.scala.StreamingContext - -import org.apache.flink.api.common.io.RichOutputFormat -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.datastream.DataStreamSink -import org.apache.flink.streaming.api.scala.DataStream - -import java.util.Properties - -import scala.annotation.meta.param - -object JdbcSink { - - /** - * @param ctx - * : StreamingContext - * @param alias - * : Instance alias (used to distinguish between multiple different database instances...) - * @return - */ - def apply( - @(transient @param) - parallelism: Int = 0, - alias: String = "", - name: String = null, - uid: String = null)(implicit ctx: StreamingContext): JdbcSink = - new JdbcSink(ctx, parallelism, alias, name, uid) - -} - -class JdbcSink( - @(transient @param) ctx: StreamingContext, - parallelism: Int = 0, - alias: String = "", - name: String = null, - uid: String = null) - extends Sink - with Logger { - - /** - * @param stream - * : DataStream - * @param toSQLFn - * : The function converted to SQL is provided by the user. - * @tparam T - * : The data type of the stream in the DataStream - * @return - */ - def sink[T](stream: DataStream[T])(implicit toSQLFn: T => String): DataStreamSink[T] = { - val prop = ConfigUtils.getJdbcConf(ctx.parameter.toMap, alias) - val semantic = Semantic.of(prop.getProperty(KEY_SEMANTIC, Semantic.NONE.name())) - val sink = semantic match { - case Semantic.EXACTLY_ONCE => - val sinkFun = new Jdbc2PCSinkFunction[T](prop, toSQLFn) - if (parallelism > 1) { - logWarn(s"'parallelism':$parallelism, Jdbc Semantic EXACTLY_ONCE,parallelism bust be 1.") - } - stream.addSink(sinkFun) - case _ => - val sinkFun = new JdbcSinkFunction[T](prop, toSQLFn) - stream.addSink(sinkFun) - } - afterSink(sink, parallelism, name, uid) - } -} - -class JdbcOutputFormat[T: TypeInformation](implicit prop: Properties, toSQlFun: T => String) - extends RichOutputFormat[T] - with Logger { - - val sinkFunction = new JdbcSinkFunction[T](prop, toSQlFun) - - var configuration: Configuration = _ - - override def configure(configuration: Configuration): Unit = this.configuration = configuration - - override def open(taskNumber: Int, numTasks: Int): Unit = sinkFunction.open(this.configuration) - - override def writeRecord(record: T): Unit = sinkFunction.invoke(record, null) - - override def close(): Unit = sinkFunction.close() -} - -//-------------Jdbc2PCSinkFunction exactly-once support --------------------------------------------------------------------------------------- - -class Jdbc2PCOutputFormat[T: TypeInformation](implicit prop: Properties, toSQlFun: T => String) - extends RichOutputFormat[T] - with Logger { - - private val sinkFunction = new Jdbc2PCSinkFunction[T](prop, toSQlFun) - - private var configuration: Configuration = _ - - override def configure(configuration: Configuration): Unit = this.configuration = configuration - - override def open(taskNumber: Int, numTasks: Int): Unit = sinkFunction.open(this.configuration) - - override def writeRecord(record: T): Unit = sinkFunction.invoke(record, null) - - override def close(): Unit = sinkFunction.close() -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/source/JdbcSource.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/source/JdbcSource.scala deleted file mode 100644 index 0e2cab8826..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-jdbc/src/main/scala/org/apache/streampark/flink/connector/jdbc/source/JdbcSource.scala +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.jdbc.source - -import org.apache.streampark.common.util.Utils -import org.apache.streampark.flink.connector.jdbc.internal.JdbcSourceFunction -import org.apache.streampark.flink.core.scala.StreamingContext - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.streaming.api.scala.DataStream - -import java.util.Properties - -import scala.annotation.meta.param -import scala.collection.Map - -object JdbcSource { - - def apply(@(transient @param) property: Properties = new Properties())(implicit - ctx: StreamingContext): JdbcSource = new JdbcSource(ctx, property) - -} - -class JdbcSource( - @(transient @param) val ctx: StreamingContext, - property: Properties = new Properties()) { - - /** - * @param sqlFun - * @param fun - * @param jdbc - * @tparam R - * @return - */ - def getDataStream[R: TypeInformation]( - sqlFun: R => String, - fun: Iterable[Map[String, _]] => Iterable[R], - running: Unit => Boolean)(implicit jdbc: Properties = new Properties()): DataStream[R] = { - Utils.copyProperties(property, jdbc) - val mysqlFun = new JdbcSourceFunction[R](jdbc, sqlFun, fun, running) - ctx.addSource(mysqlFun) - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/pom.xml deleted file mode 100644 index ba941ec6ff..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/pom.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink-connector - 2.2.0-SNAPSHOT - - - streampark-flink-connector-kafka_${scala.binary.version} - StreamPark : Flink Connector Kafka - - - - - org.apache.streampark - streampark-flink-connector-base_${scala.binary.version} - ${project.version} - - - - org.apache.flink - flink-connector-kafka - ${flink.connector.version} - - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/java/org/apache/streampark/flink/connector/kafka/sink/KafkaJavaSink.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/java/org/apache/streampark/flink/connector/kafka/sink/KafkaJavaSink.java deleted file mode 100644 index abcd9dc1bd..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/java/org/apache/streampark/flink/connector/kafka/sink/KafkaJavaSink.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.kafka.sink; - -import org.apache.streampark.flink.connector.kafka.bean.KafkaEqualityPartitioner; -import org.apache.streampark.flink.core.scala.StreamingContext; - -import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.api.common.serialization.SimpleStringSchema; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamSink; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; - -import java.util.Properties; - -public class KafkaJavaSink { - - private final StreamingContext context; - /** common param */ - private Properties property = new Properties(); - - private Integer parallelism = 0; - private String name; - private String uid; - // ---end--- - - private String alias = ""; - private String topic = ""; - private SerializationSchema serializer = (SerializationSchema) new SimpleStringSchema(); - private FlinkKafkaPartitioner partitioner; - - public KafkaJavaSink(StreamingContext context) { - this.context = context; - // The default partitioner is KafkaEqualityPartitioner - partitioner = new KafkaEqualityPartitioner(context.getParallelism()); - } - - public KafkaJavaSink parallelism(Integer parallelism) { - if (parallelism != null) { - this.parallelism = parallelism; - } - return this; - } - - public KafkaJavaSink name(String name) { - if (name != null) { - this.name = name; - } - return this; - } - - public KafkaJavaSink uid(String uid) { - if (uid != null) { - this.uid = uid; - } - return this; - } - - public KafkaJavaSink property(Properties property) { - if (property != null) { - this.property = property; - } - return this; - } - - public KafkaJavaSink alias(String alias) { - if (alias != null) { - this.alias = alias; - } - return this; - } - - /** - * Set the topic to sink - * - * @param topic: topic name - * @return KafkaSink: KafkaSink instance - */ - public KafkaJavaSink topic(String topic) { - if (topic != null) { - this.topic = topic; - } - return this; - } - - /** - * set SerializationSchema - * - * @param serializer: serializer - * @return KafkaSink: KafkaSink instance - */ - public KafkaJavaSink serializer(SerializationSchema serializer) { - if (serializer != null) { - this.serializer = serializer; - } - return this; - } - - /** - * set FlinkKafkaPartitioner - * - * @param partitioner: FlinkKafkaPartitioner - * @return KafkaSink: KafkaSink - */ - public KafkaJavaSink partitioner(FlinkKafkaPartitioner partitioner) { - if (partitioner != null) { - this.partitioner = partitioner; - } - return this; - } - - public DataStreamSink sink(DataStream source) { - return this.sink(source, this.topic); - } - - public DataStreamSink sink(DataStream source, String topic) { - this.topic(topic); - KafkaSink scalaSink = - new KafkaSink(this.context, this.property, this.parallelism, this.name, this.uid); - return scalaSink.sink( - new org.apache.flink.streaming.api.scala.DataStream<>(source), - this.alias, - this.topic, - this.serializer, - this.partitioner); - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/java/org/apache/streampark/flink/connector/kafka/source/KafkaJavaSource.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/java/org/apache/streampark/flink/connector/kafka/source/KafkaJavaSource.java deleted file mode 100644 index 32cc0d8e45..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/java/org/apache/streampark/flink/connector/kafka/source/KafkaJavaSource.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.kafka.source; - -import org.apache.streampark.flink.connector.kafka.bean.KafkaRecord; -import org.apache.streampark.flink.core.scala.StreamingContext; - -import org.apache.flink.api.common.eventtime.WatermarkStrategy; -import org.apache.flink.streaming.api.datastream.DataStreamSource; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; -import org.apache.flink.streaming.connectors.kafka.KafkaDeserializationSchema; - -import java.util.Properties; - -public class KafkaJavaSource { - - private final StreamingContext context; - private String[] topics; - private String alias = ""; - private KafkaDeserializationSchema deserializer; - private WatermarkStrategy> strategy; - private Properties property = new Properties(); - - public KafkaJavaSource(StreamingContext context) { - this.context = context; - this.deserializer = (KafkaDeserializationSchema) new KafkaStringDeserializationSchema(); - } - - public KafkaJavaSource property(Properties property) { - if (property != null) { - this.property = property; - } - return this; - } - - public KafkaJavaSource topic(String... topic) { - if (topic != null) { - this.topics = topic; - } - return this; - } - - public KafkaJavaSource alias(String alias) { - if (alias != null) { - this.alias = alias; - } - return this; - } - - public KafkaJavaSource deserializer(KafkaDeserializationSchema deserializer) { - if (deserializer != null) { - this.deserializer = deserializer; - } - return this; - } - - public KafkaJavaSource strategy(WatermarkStrategy> strategy) { - if (strategy != null) { - this.strategy = strategy; - } - return this; - } - - public DataStreamSource> getDataStream() { - FlinkKafkaConsumer> consumer = - KafkaSource.getSource( - this.context, - this.property, - this.topics, - this.alias, - this.deserializer, - this.strategy, - null); - return context.getJavaEnv().addSource(consumer); - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/bean/KafkaEqualityPartitioner.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/bean/KafkaEqualityPartitioner.scala deleted file mode 100644 index 1854b17658..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/bean/KafkaEqualityPartitioner.scala +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.kafka.bean - -import org.apache.streampark.common.util.Logger - -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner - -import javax.annotation.Nullable - -import java.util.concurrent.atomic.AtomicInteger - -/** - * KafkaEqualityPartitionerEquality Partitioner, the partitioner can evenly write data to - * each partition - * - * @param parallelism - * @tparam T - */ -class KafkaEqualityPartitioner[T](parallelism: Int) extends FlinkKafkaPartitioner[T] with Logger { - - private[this] var parallelInstanceId = 0 - - private[this] val partitionIndex: AtomicInteger = new AtomicInteger(0) - - override def open(parallelInstanceId: Int, parallelInstances: Int): Unit = { - logInfo(s"KafkaEqualityPartitioner: parallelism $parallelism") - require( - parallelInstanceId >= 0 && parallelInstances > 0, - "[StreamPark] KafkaEqualityPartitioner:Id of this subtask cannot be negative,Number of subtasks must be larger than 0." - ) - this.parallelInstanceId = parallelInstanceId - } - - override def partition( - record: T, - key: Array[Byte], - value: Array[Byte], - targetTopic: String, - partitions: Array[Int]): Int = { - require( - partitions != null && partitions.length > 0, - "[StreamPark] KafkaEqualityPartitioner:Partitions of the target topic is empty.") - (parallelism, partitions.length) match { - // kafka have 1 partition - case (_, 1) => 0 - case (x, y) if x % y == 0 => partitions(parallelInstanceId % partitions.length) - case (_, y) => - partitionIndex.get() match { - case x if x == y - 1 => partitionIndex.getAndSet(0) - case _ => partitionIndex.incrementAndGet() - } - } - } - - override def equals(o: Any): Boolean = this == o || o.isInstanceOf[KafkaEqualityPartitioner[T]] - - override def hashCode: Int = classOf[KafkaEqualityPartitioner[T]].hashCode - - def checkArgument(condition: Boolean, @Nullable errorMessage: String): Unit = - if (!condition) throw new IllegalArgumentException(errorMessage) - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/bean/KafkaRecord.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/bean/KafkaRecord.scala deleted file mode 100644 index 45df932aff..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/bean/KafkaRecord.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.kafka.bean - -import org.apache.flink.api.common.typeinfo.TypeInformation - -class KafkaRecord[T: TypeInformation]( - val topic: String, - val partition: Long, - val timestamp: Long, - val offset: Long, - val key: String, - val value: T) diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/sink/KafkaSink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/sink/KafkaSink.scala deleted file mode 100644 index 8a60922a8b..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/sink/KafkaSink.scala +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.kafka.sink - -import org.apache.streampark.common.conf.ConfigKeys -import org.apache.streampark.common.util.{ConfigUtils, Utils} -import org.apache.streampark.flink.connector.kafka.bean.KafkaEqualityPartitioner -import org.apache.streampark.flink.connector.sink.Sink -import org.apache.streampark.flink.core.scala.StreamingContext - -import org.apache.flink.api.common.serialization.{SerializationSchema, SimpleStringSchema} -import org.apache.flink.streaming.api.datastream.DataStreamSink -import org.apache.flink.streaming.api.scala.DataStream -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer.{DEFAULT_KAFKA_PRODUCERS_POOL_SIZE, Semantic} -import org.apache.flink.streaming.connectors.kafka.internals.KeyedSerializationSchemaWrapper -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner - -import java.util.{Optional, Properties} - -import scala.annotation.meta.param -import scala.util.Try - -object KafkaSink { - def apply( - @(transient @param) - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null)(implicit ctx: StreamingContext): KafkaSink = - new KafkaSink(ctx, property, parallelism, name, uid) -} - -class KafkaSink( - @(transient @param) val ctx: StreamingContext, - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null) - extends Sink { - - /** - * for scala - * - * @param stream - * @param alias - * @param topic - * @param serializer - * serializer, if not specified, used SimpleStringSchema - * @param partitioner - * kafka partitioner, used KafkaEqualityPartitioner as default partitioner, the - * partitioner can evenly write data to each partition. Note: The default used in Flink is FlinkFixedPartitioner, which is need to pay attention to the - * parallelism of sink and the number of kafka partitions, otherwise it will appear to write to - * a partition.) - * @tparam T - * @return - */ - def sink[T]( - stream: DataStream[T], - alias: String = "", - topic: String = "", - serializer: SerializationSchema[T] = - new SimpleStringSchema().asInstanceOf[SerializationSchema[T]], - partitioner: FlinkKafkaPartitioner[T] = new KafkaEqualityPartitioner[T](ctx.getParallelism)) - : DataStreamSink[T] = { - - val producer = { - val prop = ConfigUtils.getKafkaSinkConf(ctx.parameter.toMap, topic, alias) - Utils.copyProperties(property, prop) - val topicId = prop.remove(ConfigKeys.KEY_KAFKA_TOPIC).toString - - /** kafkaProducersPoolSize will be used under EXACTLY_ONCE semantics */ - val semantic = Try(Some(prop.remove(ConfigKeys.KEY_KAFKA_SEMANTIC).toString.toUpperCase)) - .getOrElse(None) match { - case None => Semantic.AT_LEAST_ONCE - case Some("AT_LEAST_ONCE") => Semantic.AT_LEAST_ONCE - case Some("EXACTLY_ONCE") => Semantic.EXACTLY_ONCE - case Some("NONE") => Semantic.NONE - case _ => - throw new IllegalArgumentException( - "[StreamPark] kafka.sink semantic error,must be (AT_LEAST_ONCE|EXACTLY_ONCE|NONE) ") - } - val schema = new KeyedSerializationSchemaWrapper[T](serializer) - - val customPartitioner = partitioner match { - case null => Optional.ofNullable(null).asInstanceOf[Optional[FlinkKafkaPartitioner[T]]] - case part => Optional.of(part) - } - new FlinkKafkaProducer[T]( - topicId, - schema, - prop, - customPartitioner, - semantic, - DEFAULT_KAFKA_PRODUCERS_POOL_SIZE) - } - - /** - * versions 0.10+ allow attaching the records' event timestamp when writing them to Kafka; this - * method is not available for earlier Kafka versions - */ - producer.setWriteTimestampToKafka(true) - - val sink = stream.addSink(producer) - afterSink(sink, parallelism, name, uid) - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/source/KafkaSource.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/source/KafkaSource.scala deleted file mode 100644 index a5e0feb469..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-kafka/src/main/scala/org/apache/streampark/flink/connector/kafka/source/KafkaSource.scala +++ /dev/null @@ -1,277 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.kafka.source - -import org.apache.streampark.common.conf.ConfigKeys._ -import org.apache.streampark.common.util.{ConfigUtils, Utils} -import org.apache.streampark.flink.connector.kafka.bean.KafkaRecord -import org.apache.streampark.flink.core.scala.StreamingContext - -import org.apache.flink.api.common.eventtime.WatermarkStrategy -import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation} -import org.apache.flink.api.java.typeutils.TypeExtractor.getForClass -import org.apache.flink.streaming.api.scala._ -import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, KafkaDeserializationSchema} -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition -import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord} - -import java.io -import java.util.Properties -import java.util.regex.Pattern - -import scala.annotation.meta.param -import org.apache.streampark.common.util.Implicits._ -import scala.util.{Failure, Success, Try} - -object KafkaSource { - - def apply(@(transient @param) property: Properties = new Properties())(implicit - ctx: StreamingContext): KafkaSource = new KafkaSource(ctx, property) - - def getSource[T: TypeInformation]( - ctx: StreamingContext, - property: Properties = new Properties(), - topic: io.Serializable, - alias: String, - deserializer: KafkaDeserializationSchema[T], - strategy: WatermarkStrategy[KafkaRecord[T]]): FlinkKafkaConsumer[KafkaRecord[T]] = { - - val prop = ConfigUtils.getConf(ctx.parameter.toMap, KAFKA_SOURCE_PREFIX + alias) - Utils.copyProperties(property, prop) - require( - prop != null && prop.nonEmpty && prop.exists( - x => x._1 == KEY_KAFKA_TOPIC || x._1 == KEY_KAFKA_PATTERN)) - - // start.form parameter... - val timestamp = Try(Some(prop(s"$KEY_KAFKA_START_FROM.$KEY_KAFKA_START_FROM_TIMESTAMP").toLong)) - .getOrElse(None) - val startFrom = StartFrom.startForm(prop) - require( - !(timestamp.nonEmpty && startFrom != null), - s"[StreamPark] start.form timestamp and offset cannot be defined at the same time") - - // topic parameter - val topicOpt = Try(Some(prop.remove(KEY_KAFKA_TOPIC).toString)).getOrElse(None) - val regexOpt = Try(Some(prop.remove(KEY_KAFKA_PATTERN).toString)).getOrElse(None) - - val kfkDeserializer = new KafkaDeserializer[T](deserializer) - - val consumer = (topicOpt, regexOpt) match { - case (Some(_), Some(_)) => - throw new IllegalArgumentException( - "[StreamPark] topic and regex cannot be defined at the same time") - case (Some(top), _) => - val topics = top.split(",|\\s+") - val topicList = topic match { - case null => topics.toList - case x: String => List(x) - case x: Array[String] => x.toList - case x: List[String] => x - case _ => - throw new IllegalArgumentException( - "[StreamPark] topic type must be String(one topic) or List[String](more topic)") - } - new FlinkKafkaConsumer(topicList, kfkDeserializer, prop) - case (_, Some(reg)) => - val pattern: Pattern = topic match { - case null => reg.r.pattern - case x: String => x.r.pattern - case _ => - throw new IllegalArgumentException( - "[StreamPark] subscriptionPattern type must be String(regex)") - } - val kfkDeserializer = new KafkaDeserializer[T](deserializer) - new FlinkKafkaConsumer(pattern, kfkDeserializer, prop) - case _ => null - } - - val autoCommit = prop.getOrElse(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true").toBoolean - (ctx.getCheckpointConfig.isCheckpointingEnabled, autoCommit) match { - case (true, _) => consumer.setCommitOffsetsOnCheckpoints(true) - case (_, false) => - throw new IllegalArgumentException( - "[StreamPark] error:flink checkpoint was disable,and kafka autoCommit was false.you can enable checkpoint or enable kafka autoCommit...") - case _ => - } - - if (strategy != null) { - val assignerWithPeriodicWatermarks = - consumer.getClass.getMethod("assignTimestampsAndWatermarks", classOf[WatermarkStrategy[T]]) - assignerWithPeriodicWatermarks.setAccessible(true) - assignerWithPeriodicWatermarks.invoke(consumer, strategy) - } - - (timestamp, startFrom) match { - // Globally set Timestamp, effective for all topics. - case (Some(t), _) => consumer.setStartFromTimestamp(t) - // Specify the offset for each topic and partition - case _ => - val startFroms = (topicOpt, regexOpt) match { - case (Some(top), _) => - topic match { - case null => startFrom.toList - case x: String => startFrom.filter(_.topic == x).toList - case x: Array[_] => - val topics = if (topic == null) top.split(",|\\s+").toList else x.toList - startFrom.filter(s => topics.contains(s.topic)).toList - case x: List[_] => - val topics = if (topic == null) top.split(",|\\s+").toList else x - startFrom.filter(s => topics.contains(s.topic)).toList - case _ => List.empty[StartFrom] - } - case (_, Some(reg)) => - topic match { - case null => startFrom.filter(s => reg.r.findFirstIn(s.topic).nonEmpty).toList - case x: String => startFrom.filter(s => x.r.findFirstIn(s.topic).nonEmpty).toList - case _ => List.empty[StartFrom] - } - case _ => null - } - - // startOffsets... - val startOffsets = new java.util.HashMap[KafkaTopicPartition, java.lang.Long]() - startFroms - .filter(x => x != null && x.partitionOffset != null) - .foreach( - start => { - start.partitionOffset.foreach( - x => startOffsets.put(new KafkaTopicPartition(start.topic, x._1), x._2)) - }) - - if (startOffsets.nonEmpty) { - consumer.setStartFromSpecificOffsets(startOffsets) - } - } - consumer - } - -} - -class KafkaSource( - @(transient @param) private[this] val ctx: StreamingContext, - property: Properties = new Properties()) { - - /** - * commit offset method:
  Flink kafka consumer commit offset method needs to - * distinguish whether checkpoint is enabled.
   1) Checkpoint off: commit offset - * depends on auto commit of kafka client. Need to set enable.auto.commit, auto.commit.interval.ms - * parameters to consumer properties, It will periodically auto commit offset to kafka at regular - * intervals.
   2) Checkpoint is enabled: At this time, the offset consumed by - * the job is managed and fault-tolerant by Flink in the state. Submitting offsets to kafka at - * this time is generally used as an external progress monitor. I want to know the location and - * lag of job consumption in real time. At this point, setCommitOffsetsOnCheckpoints needs to be - * true to set the offset to be submitted to kafka when the checkpoint is successful. At this - * time, the interval of commit offset depends on the interval of checkpoint - * - * Get DStream stream - * - * @param topic - * a group of topics or a single topic - * @param alias - * Aliases to distinguish different kafka connection instances - * @param deserializer - * DeserializationSchema - * @param strategy - * Watermarks strategy - * @tparam T - */ - def getDataStream[T: TypeInformation]( - topic: java.io.Serializable = null, - alias: String = "", - deserializer: KafkaDeserializationSchema[T] = - new KafkaStringDeserializationSchema().asInstanceOf[KafkaDeserializationSchema[T]], - strategy: WatermarkStrategy[KafkaRecord[T]] = null): DataStream[KafkaRecord[T]] = { - - val consumer = - KafkaSource.getSource[T](this.ctx, property, topic, alias, deserializer, strategy) - ctx.addSource(consumer) - } - -} - -class KafkaDeserializer[T: TypeInformation](deserializer: KafkaDeserializationSchema[T]) - extends KafkaDeserializationSchema[KafkaRecord[T]] { - - override def deserialize(record: ConsumerRecord[Array[Byte], Array[Byte]]): KafkaRecord[T] = { - val key = if (record.key() == null) null else new String(record.key(), "UTF-8") - val value = deserializer.deserialize(record) - val offset = record.offset() - val partition = record.partition() - val topic = record.topic() - val timestamp = record.timestamp() - new KafkaRecord[T](topic, partition, timestamp, offset, key, value) - } - - override def getProducedType: TypeInformation[KafkaRecord[T]] = getForClass( - classOf[KafkaRecord[T]]) - - override def isEndOfStream(nextElement: KafkaRecord[T]): Boolean = false - -} - -class KafkaStringDeserializationSchema extends KafkaDeserializationSchema[String] { - - override def isEndOfStream(nextElement: String): Boolean = false - - override def deserialize(record: ConsumerRecord[Array[Byte], Array[Byte]]): String = - new String(record.value(), "UTF-8") - - override def getProducedType: TypeInformation[String] = BasicTypeInfo.STRING_TYPE_INFO - -} - -object StartFrom { - - def startForm(prop: Properties): Array[StartFrom] = { - val startProp = prop.filter(_._1.startsWith(KEY_KAFKA_START_FROM)) - startProp.foreach(x => prop.remove(x._1)) - val topic = Try( - startProp(s"$KEY_KAFKA_START_FROM.$KEY_KAFKA_START_FROM_OFFSET.$KEY_KAFKA_TOPIC").split(",")) - .getOrElse(Array.empty[String]) - if (topic.isEmpty) Array.empty[StartFrom] - else { - topic - .map( - x => { - val offset = - Try(Some(startProp(s"$KEY_KAFKA_START_FROM.$KEY_KAFKA_START_FROM_OFFSET.$x"))) - .getOrElse(None) - offset match { - case Some(o) => - Try( - o.split(",") - .map( - x => { - val array = x.split(":") - array.head.toInt -> array.last.toLong - })) match { - case Success(v) => new StartFrom(x, v) - case Failure(_) => - throw new IllegalArgumentException( - s"[StreamPark] topic:$x start.form offset error, e.g: 1:10000,2:10000,3:10002") - } - case _ => null - } - }) - .filter(_ != null) - } - } - -} - -class StartFrom(val topic: String, val partitionOffset: Array[(Int, Long)]) diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/pom.xml deleted file mode 100644 index d96e3465e6..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/pom.xml +++ /dev/null @@ -1,89 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink-connector - 2.2.0-SNAPSHOT - - - streampark-flink-connector-mongo_${scala.binary.version} - StreamPark : Flink Connector Mongo - - - - - org.apache.streampark - streampark-flink-connector-base_${scala.binary.version} - ${project.version} - - - - org.mongodb - mongo-java-driver - 3.12.2 - - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/java/org/apache/streampark/flink/connector/mongo/function/MongoQueryFunction.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/java/org/apache/streampark/flink/connector/mongo/function/MongoQueryFunction.java deleted file mode 100644 index a9fe78302b..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/java/org/apache/streampark/flink/connector/mongo/function/MongoQueryFunction.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.mongo.function; - -import com.mongodb.client.FindIterable; -import com.mongodb.client.MongoCollection; -import org.bson.Document; - -@FunctionalInterface -public interface MongoQueryFunction { - - /** - * return query condition - * - * @param last: last one - * @param collection MongoCollection - * @return FindIterable - */ - FindIterable query(T last, MongoCollection collection); -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/java/org/apache/streampark/flink/connector/mongo/function/MongoResultFunction.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/java/org/apache/streampark/flink/connector/mongo/function/MongoResultFunction.java deleted file mode 100644 index 7a01a8c5d8..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/java/org/apache/streampark/flink/connector/mongo/function/MongoResultFunction.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.mongo.function; - -import com.mongodb.client.MongoCursor; -import org.bson.Document; - -@FunctionalInterface -public interface MongoResultFunction { - - /** - * process query result - * - * @param cursor: MongoCursor - * @return Iterable: Iterable - */ - Iterable result(MongoCursor cursor); -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/java/org/apache/streampark/flink/connector/mongo/source/MongoJavaSource.java b/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/java/org/apache/streampark/flink/connector/mongo/source/MongoJavaSource.java deleted file mode 100644 index 5c451f0a17..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/java/org/apache/streampark/flink/connector/mongo/source/MongoJavaSource.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.mongo.source; - -import org.apache.streampark.common.util.Utils; -import org.apache.streampark.flink.connector.function.RunningFunction; -import org.apache.streampark.flink.connector.mongo.function.MongoQueryFunction; -import org.apache.streampark.flink.connector.mongo.function.MongoResultFunction; -import org.apache.streampark.flink.connector.mongo.internal.MongoSourceFunction; -import org.apache.streampark.flink.core.scala.StreamingContext; - -import org.apache.flink.streaming.api.datastream.DataStreamSource; - -import java.util.Properties; - -public class MongoJavaSource { - private final StreamingContext context; - private final Properties property; - - public MongoJavaSource(StreamingContext context, Properties property) { - this.context = context; - this.property = property; - } - - public DataStreamSource getDataStream( - String collectionName, - MongoQueryFunction queryFunction, - MongoResultFunction resultFunction, - RunningFunction runningFunc) { - - Utils.requireNotNull(collectionName, "'collectionName' must not be null"); - Utils.requireNotNull(queryFunction, "'queryFunction' must not be null"); - Utils.requireNotNull(resultFunction, "'resultFunction' must not be null"); - MongoSourceFunction sourceFunction = - new MongoSourceFunction<>( - collectionName, property, queryFunction, resultFunction, runningFunc, null); - return context.getJavaEnv().addSource(sourceFunction); - } -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/scala/org/apache/streampark/flink/connector/mongo/internal/MongoSourceFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/scala/org/apache/streampark/flink/connector/mongo/internal/MongoSourceFunction.scala deleted file mode 100644 index 91ee78b03d..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/scala/org/apache/streampark/flink/connector/mongo/internal/MongoSourceFunction.scala +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.mongo.internal - -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.enums.ApiType.ApiType -import org.apache.streampark.common.util.{Logger, MongoConfig} -import org.apache.streampark.flink.connector.function.RunningFunction -import org.apache.streampark.flink.connector.mongo.function.{MongoQueryFunction, MongoResultFunction} -import org.apache.streampark.flink.util.FlinkUtils - -import com.mongodb.MongoClient -import com.mongodb.client.{FindIterable, MongoCollection, MongoCursor} -import org.apache.flink.api.common.state.ListState -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.runtime.state.{CheckpointListener, FunctionInitializationContext, FunctionSnapshotContext} -import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction -import org.apache.flink.streaming.api.functions.source.RichSourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.bson.Document - -import java.lang -import java.util.Properties - -import org.apache.streampark.common.util.Implicits._ -import scala.util.{Success, Try} - -class MongoSourceFunction[R: TypeInformation]( - apiType: ApiType, - prop: Properties = new Properties(), - collection: String) - extends RichSourceFunction[R] - with CheckpointedFunction - with CheckpointListener - with Logger { - - @volatile private[this] var running = true - private[this] var scalaRunningFunc: Unit => Boolean = _ - private[this] var javaRunningFunc: RunningFunction = _ - - var client: MongoClient = _ - var mongoCollection: MongoCollection[Document] = _ - - private[this] var scalaQueryFunc: (R, MongoCollection[Document]) => FindIterable[Document] = _ - private[this] var scalaResultFunc: MongoCursor[Document] => List[R] = _ - - private[this] var javaQueryFunc: MongoQueryFunction[R] = _ - private[this] var javaResultFunc: MongoResultFunction[R] = _ - - @transient private var state: ListState[R] = _ - private val OFFSETS_STATE_NAME: String = "mongo-source-query-states" - private[this] var last: R = _ - - // for Scala - def this( - collectionName: String, - prop: Properties, - scalaQueryFunc: (R, MongoCollection[Document]) => FindIterable[Document], - scalaResultFunc: MongoCursor[Document] => List[R], - runningFunc: Unit => Boolean) = { - - this(ApiType.scala, prop, collectionName) - this.scalaQueryFunc = scalaQueryFunc - this.scalaResultFunc = scalaResultFunc - this.scalaRunningFunc = if (runningFunc == null) _ => true else runningFunc - } - - // for JAVA - def this( - collectionName: String, - prop: Properties, - queryFunc: MongoQueryFunction[R], - resultFunc: MongoResultFunction[R], - runningFunc: RunningFunction) { - - this(ApiType.java, prop, collectionName) - this.javaQueryFunc = queryFunc - this.javaResultFunc = resultFunc - this.javaRunningFunc = - if (runningFunc != null) runningFunc - else - new RunningFunction { - override def running(): lang.Boolean = true - } - } - - override def cancel(): Unit = this.running = false - - override def open(parameters: Configuration): Unit = { - client = MongoConfig.getClient(prop) - val db = MongoConfig.getProperty(prop, MongoConfig.database) - val database = client.getDatabase(db) - mongoCollection = database.getCollection(collection) - } - - @throws[Exception] - override def run(ctx: SourceContext[R]): Unit = { - while (this.running) { - apiType match { - case ApiType.scala => - if (scalaRunningFunc()) { - ctx.getCheckpointLock.synchronized { - val find = scalaQueryFunc(last, mongoCollection) - if (find != null) { - scalaResultFunc(find.iterator).foreach( - x => { - last = x - ctx.collectWithTimestamp(last, System.currentTimeMillis()) - }) - } - } - } - case ApiType.java => - if (javaRunningFunc.running()) { - ctx.getCheckpointLock.synchronized { - val find = javaQueryFunc.query(last, mongoCollection) - if (find != null) { - javaResultFunc - .result(find.iterator) - .foreach( - x => { - last = x - ctx.collectWithTimestamp(last, System.currentTimeMillis()) - }) - } - } - } - } - } - } - - override def close(): Unit = { - client.close() - } - - override def snapshotState(context: FunctionSnapshotContext): Unit = { - if (running) { - state.clear() - if (last != null) { - state.add(last) - } - } else { - logError("MongoSource snapshotState called on closed source") - } - } - - override def initializeState(context: FunctionInitializationContext): Unit = { - // restore from checkpoint - logInfo("MongoSource snapshotState initialize") - state = FlinkUtils.getUnionListState[R](context, OFFSETS_STATE_NAME) - Try(state.get.head) match { - case Success(q) => last = q - case _ => - } - } - - override def notifyCheckpointComplete(checkpointId: Long): Unit = { - logInfo(s"MongoSource checkpointComplete: $checkpointId") - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/scala/org/apache/streampark/flink/connector/mongo/source/MongoSource.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/scala/org/apache/streampark/flink/connector/mongo/source/MongoSource.scala deleted file mode 100644 index 9856b49b2f..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-mongo/src/main/scala/org/apache/streampark/flink/connector/mongo/source/MongoSource.scala +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.mongo.source - -import org.apache.streampark.common.util.Utils -import org.apache.streampark.flink.connector.mongo.internal.MongoSourceFunction -import org.apache.streampark.flink.core.scala.StreamingContext - -import com.mongodb.client.{FindIterable, MongoCollection, MongoCursor} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.streaming.api.scala.DataStream -import org.bson.Document - -import java.util.Properties - -import scala.annotation.meta.param - -object MongoSource { - - def apply(@(transient @param) property: Properties = new Properties())(implicit - ctx: StreamingContext): MongoSource = new MongoSource(ctx, property) - -} - -class MongoSource( - @(transient @param) val ctx: StreamingContext, - property: Properties = new Properties()) { - - def getDataStream[R: TypeInformation]( - collection: String, - queryFun: (R, MongoCollection[Document]) => FindIterable[Document], - resultFun: MongoCursor[Document] => List[R], - running: Unit => Boolean)(implicit prop: Properties = new Properties()): DataStream[R] = { - - Utils.copyProperties(property, prop) - val mongoFun = new MongoSourceFunction[R](collection, prop, queryFun, resultFun, running) - ctx.addSource(mongoFun) - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/pom.xml b/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/pom.xml deleted file mode 100644 index 5086d9ea95..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/pom.xml +++ /dev/null @@ -1,95 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink-connector - 2.2.0-SNAPSHOT - - - streampark-flink-connector-redis_${scala.binary.version} - StreamPark : Flink Connector Redis - - - - - org.apache.flink - flink-streaming-java${scala.binary.flink.version} - ${flink.version} - - - - org.apache.streampark - streampark-flink-connector-base_${scala.binary.version} - ${project.version} - - - - org.apache.bahir - flink-connector-redis_${scala.binary.version} - 1.0 - - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - - diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/bean/RedisContainer.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/bean/RedisContainer.scala deleted file mode 100644 index 9a075f0d0a..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/bean/RedisContainer.scala +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.redis.bean - -import org.apache.streampark.common.util.Logger - -import org.apache.commons.pool2.impl.GenericObjectPoolConfig -import org.apache.flink.streaming.connectors.redis.common.config.{FlinkJedisConfigBase, FlinkJedisPoolConfig, FlinkJedisSentinelConfig} -import org.apache.flink.streaming.connectors.redis.common.container.{RedisContainer => BahirRedisContainer} -import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommand._ -import redis.clients.jedis.{Jedis, JedisPool, JedisSentinelPool} - -class RedisContainer(container: BahirRedisContainer) { - - def open(): Unit = { - container.open() - } - - lazy val jedis: Jedis = { - val method = container.getClass.getDeclaredMethod("getInstance") - method.setAccessible(true) - method.invoke(container).asInstanceOf[Jedis] - } - - def invoke[T]( - mapper: RedisMapper[T], - input: T, - transaction: Option[redis.clients.jedis.Transaction]): Unit = { - val key = mapper.getKeyFromData(input) - val value = mapper.getValueFromData(input) - mapper.getCommandDescription.getCommand match { - case RPUSH => - transaction match { - case Some(t) => t.rpush(key, value) - case _ => this.container.rpush(key, value) - } - case LPUSH => - transaction match { - case Some(t) => t.lpush(key, value) - case _ => this.container.lpush(key, value) - } - case SADD => - transaction match { - case Some(t) => t.sadd(key, value) - case _ => this.container.sadd(key, value) - } - case SET => - transaction match { - case Some(t) => t.set(key, value) - case _ => this.container.set(key, value) - } - case PFADD => - transaction match { - case Some(t) => t.pfadd(key, value) - case _ => this.container.pfadd(key, value) - } - case PUBLISH => - transaction match { - case Some(t) => t.publish(key, value) - case _ => this.container.publish(key, value) - } - case ZADD => - transaction match { - case Some(t) => t.zadd(mapper.getCommandDescription.getAdditionalKey, value.toDouble, key) - case _ => this.container.zadd(mapper.getCommandDescription.getAdditionalKey, value, key) - } - case ZREM => - transaction match { - case Some(t) => t.zrem(mapper.getCommandDescription.getAdditionalKey, key) - case _ => this.container.zrem(mapper.getCommandDescription.getAdditionalKey, key) - } - case HSET => - transaction match { - case Some(t) => t.hset(mapper.getCommandDescription.getAdditionalKey, key, value) - case _ => this.container.hset(mapper.getCommandDescription.getAdditionalKey, key, value) - } - case other => - throw new IllegalArgumentException( - "[StreamPark] RedisSink:Cannot process such data type: " + other) - } - } - - def expire(key: String, ttl: Int): Unit = { - ttl match { - case Int.MaxValue => - case _ => jedis.expire(key, ttl) - } - } - - def close(): Unit = { - container.close() - } - -} - -object RedisContainer extends Logger { - - def getContainer(jedisConfig: FlinkJedisConfigBase): RedisContainer = { - val genericObjectPoolConfig = new GenericObjectPoolConfig - genericObjectPoolConfig.setMaxIdle(jedisConfig.getMaxIdle) - genericObjectPoolConfig.setMaxTotal(jedisConfig.getMaxTotal) - genericObjectPoolConfig.setMinIdle(jedisConfig.getMinIdle) - try { - val bahirRedisContainer = jedisConfig match { - case jedisPoolConfig: FlinkJedisPoolConfig => - val jedisPool = new JedisPool( - genericObjectPoolConfig, - jedisPoolConfig.getHost, - jedisPoolConfig.getPort, - jedisPoolConfig.getConnectionTimeout, - jedisPoolConfig.getPassword, - jedisPoolConfig.getDatabase) - new BahirRedisContainer(jedisPool) - case _ => - val jedisSentinelConfig = jedisConfig.asInstanceOf[FlinkJedisSentinelConfig] - val jedisSentinelPool = new JedisSentinelPool( - jedisSentinelConfig.getMasterName, - jedisSentinelConfig.getSentinels, - genericObjectPoolConfig, - jedisSentinelConfig.getSoTimeout, - jedisSentinelConfig.getPassword, - jedisSentinelConfig.getDatabase) - new BahirRedisContainer(jedisSentinelPool) - } - val redisContainer = new RedisContainer(bahirRedisContainer) - redisContainer.open() - redisContainer - } catch { - case e: Exception => - logError("RedisSink:Redis has not been properly initialized: ", e) - throw e - } - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/bean/RedisMapper.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/bean/RedisMapper.scala deleted file mode 100644 index dabffa95fd..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/bean/RedisMapper.scala +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.redis.bean - -import org.apache.streampark.common.enums.ApiType -import org.apache.streampark.common.enums.ApiType.ApiType -import org.apache.streampark.flink.connector.function.TransformFunction - -import org.apache.flink.streaming.connectors.redis.common.mapper.{RedisCommand, RedisCommandDescription, RedisMapper => BahirRedisMapper} - -object RedisMapper { - - val additionalFailoverTableNullHint = "Redis additionalKey insert failoverTable must not null"; - val insertFailoverTableNullHint = "Redis cmd insert failoverTable must not null"; - - - def map[T]( - cmd: RedisCommand, - additionalKey: String, - scalaKeyFun: T => String, - scalaValueFun: T => String): RedisMapper[T] = { - require(cmd != null, () => insertFailoverTableNullHint) - require(additionalKey != null, () => failoverTableNullHint) - require(scalaKeyFun != null, () => s"Redis scalaKeyFun insert failoverTable must not null") - require(scalaValueFun != null, () => s"Redis scalaValueFun insert failoverTable must not null") - new RedisMapper[T](cmd, additionalKey, scalaKeyFun, scalaValueFun) - } - - def map[T]( - cmd: RedisCommand, - additionalKey: String, - javaKeyFun: TransformFunction[T, String], - javaValueFun: TransformFunction[T, String]): RedisMapper[T] = { - require(cmd != null, () => insertFailoverTableNullHint) - require(additionalKey != null, () => additionalFailoverTableNullHint) - require(javaKeyFun != null, () => s"Redis javaKeyFun insert failoverTable must not null") - require(javaValueFun != null, () => s"Redis javaValueFun insert failoverTable must not null") - new RedisMapper[T](cmd, additionalKey, javaKeyFun, javaValueFun) - } -} - -class RedisMapper[T](apiType: ApiType = ApiType.scala, cmd: RedisCommand, additionalKey: String) - extends BahirRedisMapper[T] { - - private[this] var scalaKeyFun: T => String = _ - private[this] var scalaValueFun: T => String = _ - private[this] var javaKeyFun: TransformFunction[T, String] = _ - private[this] var javaValueFun: TransformFunction[T, String] = _ - - // for scala - def this() = { - this(ApiType.scala, null, "") - } - - // for scala - def this( - cmd: RedisCommand, - additionalKey: String, - scalaKeyFun: T => String, - scalaValueFun: T => String) = { - this(ApiType.scala, cmd, additionalKey) - this.scalaKeyFun = scalaKeyFun - this.scalaValueFun = scalaValueFun - } - - // for java - def this( - cmd: RedisCommand, - additionalKey: String, - javaKeyFun: TransformFunction[T, String], - javaValueFun: TransformFunction[T, String]) = { - this(ApiType.java, cmd, additionalKey) - this.javaKeyFun = javaKeyFun - this.javaValueFun = javaValueFun - } - - override def getCommandDescription: RedisCommandDescription = - new RedisCommandDescription(cmd, additionalKey) - - override def getKeyFromData(r: T): String = apiType match { - case ApiType.java => javaKeyFun.transform(r) - case ApiType.scala => scalaKeyFun(r) - } - - override def getValueFromData(r: T): String = apiType match { - case ApiType.java => javaValueFun.transform(r) - case ApiType.scala => scalaValueFun(r) - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/bean/RedisTransaction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/bean/RedisTransaction.scala deleted file mode 100644 index 6e52ca0af8..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/bean/RedisTransaction.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.redis.bean - -import org.apache.streampark.common.util.Utils - -import scala.collection.mutable - -case class RedisTransaction[T]( - transactionId: String = Utils.uuid(), - mapper: mutable.MutableList[(RedisMapper[T], T, Int)] = - mutable.MutableList.empty[(RedisMapper[T], T, Int)], - var invoked: Boolean = false) - extends Serializable { - def +(redisMapper: (RedisMapper[T], T, Int)): Unit = mapper += redisMapper - - override def toString: String = - s"(transactionId:$transactionId,size:${mapper.size},invoked:$invoked)" -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/conf/RedisConfig.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/conf/RedisConfig.scala deleted file mode 100644 index 7eda5b51d7..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/conf/RedisConfig.scala +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.redis.conf - -import java.util.Properties - -class RedisConfig(parameters: Properties) extends Serializable { - - val sinkOption: RedisSinkConfigOption = RedisSinkConfigOption(properties = parameters) - - val connectType: String = sinkOption.connectType.get() - - val host: String = sinkOption.host.get() - - val port: Int = sinkOption.port.get() - - val sentinels: Set[String] = - if (connectType.equals(sinkOption.DEFAULT_CONNECT_TYPE)) Set() - else { - host - .split(sinkOption.SIGN_COMMA) - .map( - x => { - if (x.contains(sinkOption.SIGN_COLON)) x; - else { - throw new IllegalArgumentException( - s"Redis sentinel host invalid {$x} must match host:port ") - } - }) - .toSet - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/conf/RedisSinkConfigOption.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/conf/RedisSinkConfigOption.scala deleted file mode 100644 index f41168515b..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/conf/RedisSinkConfigOption.scala +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.redis.conf - -import org.apache.streampark.common.conf.ConfigOption -import org.apache.streampark.common.util.ConfigUtils - -import java.util.Properties - -import org.apache.streampark.common.util.Implicits._ - -object RedisSinkConfigOption { - val REDIS_SINK_PREFIX = "redis.sink" - - /** - * @param properties - * @return - */ - def apply( - prefixStr: String = REDIS_SINK_PREFIX, - properties: Properties = new Properties): RedisSinkConfigOption = - new RedisSinkConfigOption(prefixStr, properties) - -} - -class RedisSinkConfigOption(prefixStr: String, properties: Properties) extends Serializable { - - implicit val (prefix, prop) = (prefixStr, properties) - - val DEFAULT_CONNECT_TYPE: String = "jedisPool" - - val SIGN_COMMA = "," - - val SIGN_COLON = ":" - - val host: ConfigOption[String] = ConfigOption( - key = "host", - required = true, - classType = classOf[String], - handle = key => { - properties.remove(key).toString - }) - - val connectType: ConfigOption[String] = ConfigOption( - key = "connectType", - required = false, - defaultValue = DEFAULT_CONNECT_TYPE, - classType = classOf[String], - handle = k => { - val value: String = properties - .remove(k) - .toString - if (value == null || value.isEmpty) DEFAULT_CONNECT_TYPE else value - } - ) - - val port: ConfigOption[Int] = ConfigOption( - key = "port", - required = false, - defaultValue = 6379, - classType = classOf[Int], - handle = k => { - val value: String = properties - .remove(k) - .toString - if (value == null || value.isEmpty) 6379 else value.toInt - } - ) - - def getInternalConfig(): Properties = { - ConfigUtils.getConf(prop, prefix)(alias = "") - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/internal/Redis2PCSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/internal/Redis2PCSinkFunction.scala deleted file mode 100644 index e7c5bd41ab..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/internal/Redis2PCSinkFunction.scala +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.redis.internal - -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.redis.bean.{RedisContainer, RedisMapper, RedisTransaction} - -import org.apache.flink.api.common.ExecutionConfig -import org.apache.flink.api.common.typeutils.base.VoidSerializer -import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer -import org.apache.flink.streaming.api.functions.sink.{SinkFunction, TwoPhaseCommitSinkFunction} -import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisConfigBase -import redis.clients.jedis.exceptions.JedisException - -class Redis2PCSinkFunction[T](jedisConfig: FlinkJedisConfigBase, mapper: RedisMapper[T], ttl: Int) - extends TwoPhaseCommitSinkFunction[T, RedisTransaction[T], Void]( - new KryoSerializer[RedisTransaction[T]](classOf[RedisTransaction[T]], new ExecutionConfig), - VoidSerializer.INSTANCE) - with Logger { - - override def beginTransaction(): RedisTransaction[T] = { - logInfo("Redis2PCSink beginTransaction.") - RedisTransaction[T]() - } - - override def invoke( - transaction: RedisTransaction[T], - value: T, - context: SinkFunction.Context): Unit = { - transaction.invoked = true - transaction + (mapper, value, ttl) - } - - override def preCommit(transaction: RedisTransaction[T]): Unit = { - // Prevent called preCommit directly without called the invoke method - if (transaction.invoked) { - logInfo(s"Redis2PCSink preCommit.TransactionId:${transaction.transactionId}") - } - } - - override def commit(redisTransaction: RedisTransaction[T]): Unit = { - if (redisTransaction.invoked && redisTransaction.mapper.nonEmpty) { - try { - val redisContainer = RedisContainer.getContainer(jedisConfig) - val transaction = redisContainer.jedis.multi() - redisTransaction.mapper.foreach( - x => { - redisContainer.invoke[T](x._1, x._2, Some(transaction)) - val key = mapper.getKeyFromData(x._2) - transaction.expire(key, x._3) - }) - transaction.exec() - transaction.close() - redisContainer.close() - redisTransaction.mapper.clear() - } catch { - case t: Exception => - logError(s"Redis2PCSink commit Throwable:${t.getMessage}") - throw t - } - } - } - - override def abort(transaction: RedisTransaction[T]): Unit = { - logInfo(s"Redis2PCSink abort,TransactionId:${transaction.transactionId}") - transaction.mapper.clear() - } - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/internal/RedisSinkFunction.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/internal/RedisSinkFunction.scala deleted file mode 100644 index bac21b7198..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/internal/RedisSinkFunction.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.redis.internal - -import org.apache.streampark.common.util.Logger -import org.apache.streampark.flink.connector.redis.bean.{RedisContainer, RedisMapper} - -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.sink.SinkFunction -import org.apache.flink.streaming.connectors.redis.{RedisSink => BahirRedisSink} -import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisConfigBase - -import java.io.IOException - -class RedisSinkFunction[T](jedisConfig: FlinkJedisConfigBase, mapper: RedisMapper[T], ttl: Int) - extends BahirRedisSink[T](jedisConfig, mapper) - with Logger { - - private[this] var redisContainer: RedisContainer = _ - - @throws[Exception] - override def open(parameters: Configuration): Unit = { - redisContainer = RedisContainer.getContainer(jedisConfig) - } - - override def invoke(input: T, context: SinkFunction.Context): Unit = { - redisContainer.invoke[T](mapper, input, None) - val key = mapper.getKeyFromData(input) - redisContainer.expire(key, ttl) - } - - @throws[IOException] - override def close(): Unit = if (redisContainer != null) redisContainer.close() - -} diff --git a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/sink/RedisSink.scala b/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/sink/RedisSink.scala deleted file mode 100644 index a7f10fe56e..0000000000 --- a/streampark-flink/streampark-flink-connector/streampark-flink-connector-redis/src/main/scala/org/apache/streampark/flink/connector/redis/sink/RedisSink.scala +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.connector.redis.sink - -import org.apache.streampark.common.util.Utils -import org.apache.streampark.flink.connector.redis.bean.RedisMapper -import org.apache.streampark.flink.connector.redis.conf.RedisConfig -import org.apache.streampark.flink.connector.redis.internal.{Redis2PCSinkFunction, RedisSinkFunction} -import org.apache.streampark.flink.connector.sink.Sink -import org.apache.streampark.flink.core.scala.StreamingContext -import org.apache.streampark.flink.util.FlinkUtils - -import org.apache.flink.streaming.api.CheckpointingMode -import org.apache.flink.streaming.api.datastream.{DataStream => JavaDataStream, DataStreamSink} -import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions -import org.apache.flink.streaming.api.scala.DataStream -import org.apache.flink.streaming.connectors.redis.common.config.{FlinkJedisConfigBase, FlinkJedisPoolConfig, FlinkJedisSentinelConfig} - -import java.lang.reflect.Field -import java.util -import java.util.Properties - -import scala.annotation.meta.param -import org.apache.streampark.common.util.Implicits._ -import scala.util.Try - -object RedisSink { - - def apply( - @(transient @param) - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null)(implicit ctx: StreamingContext): RedisSink = - new RedisSink(ctx, property, parallelism, name, uid) -} - -class RedisSink( - @(transient @param) ctx: StreamingContext, - property: Properties = new Properties(), - parallelism: Int = 0, - name: String = null, - uid: String = null) - extends Sink { - - def this(ctx: StreamingContext) { - this(ctx, new Properties(), 0, null, null) - } - - private val allProperties: util.Map[String, String] = ctx.parameter.toMap - val prop = ctx.parameter.getProperties - Utils.copyProperties(property, prop) - private val redisConfig: RedisConfig = new RedisConfig(prop) - val enableCheckpoint: Boolean = FlinkUtils.isCheckpointEnabled(allProperties) - - val cpMode: CheckpointingMode = Try( - CheckpointingMode.valueOf( - allProperties.get(ExecutionCheckpointingOptions.CHECKPOINTING_MODE.key()))) - .getOrElse(ExecutionCheckpointingOptions.CHECKPOINTING_MODE.defaultValue()) - - lazy val config: FlinkJedisConfigBase = { - val connectType: String = redisConfig.connectType - - val internalProp: Properties = redisConfig.sinkOption.getInternalConfig() - - def setFieldValue(field: Field, targetObject: Any, value: String): Unit = { - field.setAccessible(true) - field.getType.getSimpleName match { - case "String" => field.set(targetObject, value) - case "int" | "Integer" => field.set(targetObject, value.toInt) - case "long" | "Long" => field.set(targetObject, value.toLong) - case "boolean" | "Boolean" => field.set(targetObject, value.toBoolean) - case _ => - } - } - - redisConfig.connectType match { - case "sentinel" => - val builder = new FlinkJedisSentinelConfig.Builder().setSentinels(redisConfig.sentinels) - internalProp.foreach( - x => { - val field = Try(builder.getClass.getDeclaredField(x._1)).getOrElse { - throw new IllegalArgumentException( - s""" - |Redis config error,property:${x._1} invalid,init FlinkJedisSentinelConfig error, property options: - |, - | sentinels>, - |, - |, - |, - |, - |, - |, - | - |""".stripMargin) - } - setFieldValue(field, builder, x._2) - }) - builder.build() - - case "jedisPool" => - val builder: FlinkJedisPoolConfig.Builder = - new FlinkJedisPoolConfig.Builder().setHost(redisConfig.host).setPort(redisConfig.port) - internalProp.foreach( - x => { - val field = Try(builder.getClass.getDeclaredField(x._1)).getOrElse { - throw new IllegalArgumentException( - s""" - |Redis config error,property:${x._1} invalid,init FlinkJedisPoolConfig error,property options: - |, - |, - |, - |, - |, - |, - |, - | - |""".stripMargin) - } - setFieldValue(field, builder, x._2) - }) - - builder.build() - case _ => - throw throw new IllegalArgumentException( - s"Redis connectType must be jedisPool|sentinel $connectType") - } - } - - /** - * scala stream - * - * @param stream - * @param mapper - * @param ttl - * @tparam T - * @return - */ - def sink[T]( - stream: DataStream[T], - mapper: RedisMapper[T], - ttl: Int = Int.MaxValue): DataStreamSink[T] = { - require(stream != null, () => s"Sink Stream must not null") - require(mapper != null, () => s"Redis mapper must not null") - require(ttl > 0, () => s"Redis ttl must greater than 0") - val sinkFun = (enableCheckpoint, cpMode) match { - case (false, CheckpointingMode.EXACTLY_ONCE) => - throw new IllegalArgumentException("Redis sink EXACTLY_ONCE must enable checkpoint") - case (true, CheckpointingMode.EXACTLY_ONCE) => - new Redis2PCSinkFunction[T](config, mapper, ttl) - case _ => new RedisSinkFunction[T](config, mapper, ttl) - } - val sink = stream.addSink(sinkFun) - afterSink(sink, parallelism, name, uid) - } - - /** - * java stream - * - * @param stream - * @param mapper - * @param ttl - * @tparam T - * @return - */ - def sink[T](stream: JavaDataStream[T], mapper: RedisMapper[T], ttl: Int): DataStreamSink[T] = { - sink(new DataStream[T](stream), mapper, ttl) - } - - /** - * java stream - * - * @param stream - * @param mapper - * @tparam T - * @return - */ - def sink[T](stream: JavaDataStream[T], mapper: RedisMapper[T]): DataStreamSink[T] = { - sink(new DataStream[T](stream), mapper, Int.MaxValue) - } - -} diff --git a/streampark-flink/streampark-flink-core/pom.xml b/streampark-flink/streampark-flink-core/pom.xml deleted file mode 100644 index a8645205aa..0000000000 --- a/streampark-flink/streampark-flink-core/pom.xml +++ /dev/null @@ -1,111 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink - 2.2.0-SNAPSHOT - - - streampark-flink-core_${scala.binary.version} - StreamPark : Flink Core - - - - - org.apache.streampark - streampark-common_${scala.binary.version} - provided - - - - org.apache.streampark - streampark-flink-shims-base_${scala.binary.version} - ${project.version} - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version}_${scala.binary.version} - ${project.version} - provided - - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-statebackend-rocksdb - ${flink.version} - true - - - - org.apache.flink - flink-connector-jdbc - ${flink.connector.version} - true - - - - - org.apache.flink - flink-table-api-scala-bridge_${scala.binary.version} - ${flink.version} - true - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - true - - - - org.apache.flink - flink-table-common - ${flink.version} - true - - - - - diff --git a/streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/DataStreamExt.scala b/streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/DataStreamExt.scala deleted file mode 100644 index 1cec9e3c3c..0000000000 --- a/streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/DataStreamExt.scala +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core.scala - -import org.apache.flink.api.common.eventtime.{SerializableTimestampAssigner, WatermarkStrategy} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.streaming.api.functions.{AssignerWithPeriodicWatermarks, AssignerWithPunctuatedWatermarks, ProcessFunction => ProcFunc} -import org.apache.flink.streaming.api.scala.{DataStream => DStream, _} -import org.apache.flink.streaming.api.watermark.Watermark -import org.apache.flink.streaming.api.windowing.time.Time -import org.apache.flink.streaming.runtime.operators.util.{AssignerWithPeriodicWatermarksAdapter, AssignerWithPunctuatedWatermarksAdapter} -import org.apache.flink.util.Collector - -import java.time.Duration - -object DataStreamExt { - - /** - * @param dataStream - * DataStream extension function - * @tparam T - */ - class DataStream[T: TypeInformation](dataStream: DStream[T]) { - - def sideOut(fun: (T, ProcFunc[T, T]#Context) => Unit): DStream[T] = - dataStream.process(new ProcFunc[T, T] { - override def processElement( - value: T, - ctx: ProcFunc[T, T]#Context, - out: Collector[T]): Unit = { - fun(value, ctx) - out.collect(value) - } - }) - - def sideGet[R: TypeInformation](sideTag: String): DStream[R] = - dataStream.getSideOutput(new OutputTag[R](sideTag)) - - def boundedOutOfOrdernessWatermark(func: T => Long, duration: Duration): DStream[T] = { - dataStream.assignTimestampsAndWatermarks( - WatermarkStrategy - .forBoundedOutOfOrderness[T](duration) - .withTimestampAssigner(new SerializableTimestampAssigner[T]() { - override def extractTimestamp(element: T, recordTimestamp: Long): Long = func(element) - })) - } - - def timeLagWatermark(fun: T => Long, maxTimeLag: Time): DStream[T] = { - val assigner = new AssignerWithPeriodicWatermarks[T] { - override def extractTimestamp(element: T, previousElementTimestamp: Long): Long = fun( - element) - - override def getCurrentWatermark: Watermark = new Watermark( - System.currentTimeMillis() - maxTimeLag.toMilliseconds) - } - dataStream.assignTimestampsAndWatermarks( - WatermarkStrategy.forGenerator[T]( - new AssignerWithPeriodicWatermarksAdapter.Strategy[T](assigner))) - } - - def punctuatedWatermark(extractTimeFun: T => Long, checkFunc: T => Boolean): DStream[T] = { - val assigner = new AssignerWithPunctuatedWatermarks[T] { - override def extractTimestamp(element: T, previousElementTimestamp: Long): Long = - extractTimeFun(element) - - override def checkAndGetNextWatermark( - lastElement: T, - extractedTimestamp: Long): Watermark = { - if (checkFunc(lastElement)) new Watermark(extractedTimestamp) - else null - } - } - dataStream.assignTimestampsAndWatermarks( - WatermarkStrategy.forGenerator[T]( - new AssignerWithPunctuatedWatermarksAdapter.Strategy[T](assigner))) - } - - /** - * extension process function, make be called easy - * - * @param processFunction - * @param onTimerFunction - * @tparam R - * @return - */ - def proc[R: TypeInformation]( - processFunction: (T, ProcFunc[T, R]#Context, Collector[R]) => Unit, - onTimerFunction: (Long, ProcFunc[T, R]#OnTimerContext, Collector[R]) => Unit = null): DStream[R] = { - - dataStream.process(new ProcFunc[T, R] { - override def processElement( - value: T, - ctx: ProcFunc[T, R]#Context, - out: Collector[R]): Unit = processFunction(value, ctx, out) - - override def onTimer( - timestamp: Long, - ctx: ProcFunc[T, R]#OnTimerContext, - out: Collector[R]): Unit = { - if (onTimerFunction != null) { - onTimerFunction(timestamp, ctx, out) - } else { - super.onTimer(timestamp, ctx, out) - } - } - }) - } - - } - - /** - * extension ProcessFunction - * - * @param ctx - * @tparam IN - * @tparam OUT - */ - class ProcessFunction[IN, OUT](val ctx: ProcFunc[IN, OUT]#Context) { - def sideOut[R: TypeInformation](outputTag: String, value: R): Unit = { - val tag = new OutputTag[R](outputTag) - ctx.output[R](tag, value) - } - } - -} diff --git a/streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/FlinkStreamTable.scala b/streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/FlinkStreamTable.scala deleted file mode 100644 index 078202b65f..0000000000 --- a/streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/FlinkStreamTable.scala +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core.scala - -import org.apache.streampark.common.conf.ConfigKeys._ -import org.apache.streampark.common.util.{Logger, SystemPropertyUtils} -import org.apache.streampark.flink.core.{FlinkTableInitializer, StreamTableContext} -import org.apache.streampark.flink.core.TableExt - -import org.apache.flink.api.common.JobExecutionResult -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.{Table, TableConfig} - -import scala.language.implicitConversions - -trait FlinkStreamTable extends Logger { - - implicit final def tableExt(table: Table): TableExt.Table = - new TableExt.Table(table) - - implicit final def tableConversions(table: Table): TableExt.TableConversions = - new TableExt.TableConversions(table) - - implicit final lazy val parameter: ParameterTool = context.parameter - - implicit var context: StreamTableContext = _ - - var jobExecutionResult: JobExecutionResult = _ - - def main(args: Array[String]): Unit = { - init(args) - ready() - handle() - jobExecutionResult = context.start() - destroy() - } - - private[this] def init(args: Array[String]): Unit = { - SystemPropertyUtils.setAppHome(KEY_APP_HOME, classOf[FlinkStreamTable]) - context = new StreamTableContext( - FlinkTableInitializer.initialize(args, configStream, configTable)) - } - - def configStream(env: StreamExecutionEnvironment, parameter: ParameterTool): Unit = {} - - def configTable(tableConfig: TableConfig, parameter: ParameterTool): Unit = {} - - def ready(): Unit = {} - - def handle(): Unit - - def destroy(): Unit = {} - -} diff --git a/streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/FlinkStreaming.scala b/streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/FlinkStreaming.scala deleted file mode 100644 index aea0898f28..0000000000 --- a/streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/FlinkStreaming.scala +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core.scala - -import org.apache.streampark.common.conf.ConfigKeys._ -import org.apache.streampark.common.util.{Logger, SystemPropertyUtils, Utils} -import org.apache.streampark.flink.core.{FlinkStreamingInitializer, StreamEnvConfig} -import org.apache.streampark.flink.core.EnhancerImplicit._ - -import org.apache.flink.api.common.JobExecutionResult -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.apache.flink.streaming.api.scala._ - -import scala.language.implicitConversions - -class StreamingContext( - val parameter: ParameterTool, - private val environment: StreamExecutionEnvironment) - extends StreamExecutionEnvironment(environment.getJavaEnv) { - - /** for scala */ - def this(args: (ParameterTool, StreamExecutionEnvironment)) = - this(args._1, args._2) - - /** for Java */ - def this(args: StreamEnvConfig) = - this(FlinkStreamingInitializer.initialize(args)) - - /** Recommend use this Api to start task */ - def start(): JobExecutionResult = execute() - - @deprecated override def execute(): JobExecutionResult = { - val appName = parameter.getAppName(required = true) - execute(appName) - } - - @deprecated override def execute(jobName: String): JobExecutionResult = { - Utils.printLogo(s"FlinkStreaming $jobName Starting...") - super.execute(jobName) - } -} - -trait FlinkStreaming extends Serializable with Logger { - - implicit final def streamExt[T: TypeInformation]( - dataStream: DataStream[T]): DataStreamExt.DataStream[T] = - new DataStreamExt.DataStream[T](dataStream) - - implicit final def procFuncExt[IN: TypeInformation, OUT: TypeInformation]( - ctx: ProcessFunction[IN, OUT]#Context): DataStreamExt.ProcessFunction[IN, OUT] = - new DataStreamExt.ProcessFunction[IN, OUT](ctx) - - implicit final lazy val parameter: ParameterTool = context.parameter - - implicit var context: StreamingContext = _ - - var jobExecutionResult: JobExecutionResult = _ - - final def main(args: Array[String]): Unit = { - init(args) - ready() - handle() - jobExecutionResult = context.start() - destroy() - } - - private[this] def init(args: Array[String]): Unit = { - SystemPropertyUtils.setAppHome(KEY_APP_HOME, classOf[FlinkStreaming]) - context = new StreamingContext(FlinkStreamingInitializer.initialize(args, config)) - } - - def ready(): Unit = {} - - def config(env: StreamExecutionEnvironment, parameter: ParameterTool): Unit = {} - - def handle(): Unit - - def destroy(): Unit = {} - -} diff --git a/streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/FlinkTable.scala b/streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/FlinkTable.scala deleted file mode 100644 index 6be8ddc43e..0000000000 --- a/streampark-flink/streampark-flink-core/src/main/scala/org/apache/streampark/flink/core/scala/FlinkTable.scala +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core.scala - -import org.apache.streampark.common.conf.ConfigKeys._ -import org.apache.streampark.common.util.{Logger, SystemPropertyUtils} -import org.apache.streampark.flink.core.{FlinkTableInitializer, TableContext} - -import org.apache.flink.api.common.JobExecutionResult -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.table.api.TableConfig - -trait FlinkTable extends Logger { - - var jobExecutionResult: JobExecutionResult = _ - - implicit final lazy val parameter: ParameterTool = context.parameter - - implicit var context: TableContext = _ - - def main(args: Array[String]): Unit = { - init(args) - ready() - handle() - jobExecutionResult = context.start() - destroy() - } - - private[this] def init(args: Array[String]): Unit = { - SystemPropertyUtils.setAppHome(KEY_APP_HOME, classOf[FlinkTable]) - context = new TableContext(FlinkTableInitializer.initialize(args, config)) - } - - def ready(): Unit = {} - - def config(tableConfig: TableConfig, parameter: ParameterTool): Unit = {} - - def handle(): Unit - - def destroy(): Unit = {} - -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/pom.xml b/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/pom.xml index 4aa7c87cf4..de1378abec 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/pom.xml +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/pom.xml @@ -55,13 +55,6 @@ test - - org.apache.streampark - streampark-flink-core_${scala.binary.version} - ${project.version} - test - - org.apache.flink flink-table-planner_${scala.binary.version} diff --git a/streampark-flink/streampark-flink-sqlclient/pom.xml b/streampark-flink/streampark-flink-sqlclient/pom.xml index 6ae21c6d22..c5d006099a 100644 --- a/streampark-flink/streampark-flink-sqlclient/pom.xml +++ b/streampark-flink/streampark-flink-sqlclient/pom.xml @@ -34,12 +34,6 @@ enumeratum_${scala.binary.version} - - org.apache.streampark - streampark-flink-core_${scala.binary.version} - ${project.version} - - org.apache.streampark streampark-flink-shims-base_${scala.binary.version} diff --git a/streampark-flink/streampark-flink-sqlclient/src/main/scala/org/apache/streampark/flink/cli/SqlClient.scala b/streampark-flink/streampark-flink-sqlclient/src/main/scala/org/apache/streampark/flink/cli/SqlClient.scala index a2d967b4b8..bd2252c07b 100644 --- a/streampark-flink/streampark-flink-sqlclient/src/main/scala/org/apache/streampark/flink/cli/SqlClient.scala +++ b/streampark-flink/streampark-flink-sqlclient/src/main/scala/org/apache/streampark/flink/cli/SqlClient.scala @@ -17,10 +17,9 @@ package org.apache.streampark.flink.cli -import org.apache.streampark.common.conf.ConfigKeys.{KEY_APP_CONF, KEY_FLINK_SQL, KEY_FLINK_TABLE_MODE} -import org.apache.streampark.common.util.{DeflaterUtils, PropertiesUtils} -import org.apache.streampark.flink.core.{SqlCommand, SqlCommandParser} -import org.apache.streampark.flink.core.scala.{FlinkStreamTable, FlinkTable} +import org.apache.streampark.common.conf.ConfigKeys.{KEY_APP_HOME, KEY_APP_CONF, KEY_FLINK_SQL, KEY_FLINK_TABLE_MODE} +import org.apache.streampark.common.util.{DeflaterUtils, PropertiesUtils, SystemPropertyUtils} +import org.apache.streampark.flink.core.{FlinkTableInitializer, SqlCommand, SqlCommandParser, StreamTableContext, TableContext} import org.apache.commons.lang3.StringUtils import org.apache.flink.api.common.RuntimeExecutionMode @@ -85,12 +84,22 @@ object SqlClient extends App { "Usage: runtime execution-mode invalid, optional [STREAMING|BATCH|AUTOMATIC]") } - private[this] object BatchSqlApp extends FlinkTable { - override def handle(): Unit = context.sql() + private[this] object BatchSqlApp { + def main(args: Array[String]): Unit = { + SystemPropertyUtils.setAppHome(KEY_APP_HOME, SqlClient.getClass) + val context = new TableContext(FlinkTableInitializer.initialize(args, null)) + context.sql() + context.start() + } } - private[this] object StreamSqlApp extends FlinkStreamTable { - override def handle(): Unit = context.sql() + private[this] object StreamSqlApp { + def main(args: Array[String]): Unit = { + SystemPropertyUtils.setAppHome(KEY_APP_HOME, SqlClient.getClass) + val context = new StreamTableContext(FlinkTableInitializer.initialize(args, null, null)) + context.sql() + context.start() + } } } diff --git a/streampark-spark/pom.xml b/streampark-spark/pom.xml index 309d60ec85..a4ffca154c 100644 --- a/streampark-spark/pom.xml +++ b/streampark-spark/pom.xml @@ -30,7 +30,6 @@ streampark-spark-core - streampark-spark-connector streampark-spark-client streampark-spark-sqlclient diff --git a/streampark-spark/streampark-spark-connector/pom.xml b/streampark-spark/streampark-spark-connector/pom.xml deleted file mode 100644 index 4ece9c4695..0000000000 --- a/streampark-spark/streampark-spark-connector/pom.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-spark - 2.2.0-SNAPSHOT - - - streampark-spark-connector_2.12 - pom - StreamPark : Spark Connector - - - streampark-spark-connector-base - streampark-spark-connector-kafka - - - - 0.3.2 - 2.12.3 - 2.17 - 3.2.1 - - diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-base/pom.xml b/streampark-spark/streampark-spark-connector/streampark-spark-connector-base/pom.xml deleted file mode 100644 index 65da6a9a40..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-base/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - 4.0.0 - - - org.apache.streampark - streampark-spark-connector_2.12 - 2.2.0-SNAPSHOT - - - streampark-spark-connector-base_2.12 - StreamPark : Spark Connector Base - - - - - org.apache.streampark - streampark-spark-core_2.12 - ${project.version} - - - - - diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-base/src/main/scala/org/apache/streampark/spark/connector/sink/ShowSink.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-base/src/main/scala/org/apache/streampark/spark/connector/sink/ShowSink.scala deleted file mode 100644 index 80deee4f8f..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-base/src/main/scala/org/apache/streampark/spark/connector/sink/ShowSink.scala +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.sink - -import org.apache.spark.SparkContext -import org.apache.spark.rdd.RDD -import org.apache.spark.streaming.Time - -/** - * print sink - * - * @param sc - * @param initParams - * @tparam T - */ -class ShowSink[T]( - @transient override val sc: SparkContext, - initParams: Map[String, String] = Map.empty[String, String]) - extends Sink[T] { - - override val prefix: String = "spark.sink.show." - - private lazy val prop = filterProp(param, initParams, prefix) - - private val num = prop.getProperty("num", "10").toInt - - override def sink(rdd: RDD[T], time: Time = Time(System.currentTimeMillis())): Unit = { - val firstNum = rdd.take(num + 1) - // scalastyle:off println - println("-------------------------------------------") - println("Time: " + time) - println("-------------------------------------------") - firstNum.take(num).foreach(println) - if (firstNum.length > num) println("...") - println() - // scalastyle:on println - } -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-base/src/main/scala/org/apache/streampark/spark/connector/sink/Sink.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-base/src/main/scala/org/apache/streampark/spark/connector/sink/Sink.scala deleted file mode 100644 index 841d7b14c7..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-base/src/main/scala/org/apache/streampark/spark/connector/sink/Sink.scala +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.sink - -import org.apache.streampark.common.util.Logger - -import org.apache.spark.SparkContext -import org.apache.spark.rdd.RDD -import org.apache.spark.streaming.Time -import org.apache.spark.streaming.dstream.DStream - -import java.util.Properties - -import scala.annotation.meta.getter -import scala.collection.Map -import scala.util.Try - -/** Base output trait */ -trait Sink[T] extends Serializable with Logger { - - @(transient @getter) - val sc: SparkContext - @(transient @getter) - lazy val sparkConf = sc.getConf - - val prefix: String - - lazy val param: Map[String, String] = sparkConf.getAll.flatMap { - case (k, v) if k.startsWith(prefix) && Try(v.nonEmpty).getOrElse(false) => - Some(k.substring(prefix.length) -> v) - case _ => None - }.toMap - - def filterProp( - param: Map[String, String], - overrides: Map[String, String], - prefix: String = "", - replacement: String = ""): Properties = { - val p = new Properties() - val map = param ++ overrides - val filtered = - if (prefix.isEmpty) map else map.filter(_._1.startsWith(prefix)) - filtered.foreach(x => p.put(x._1.replace(prefix, replacement), x._2)) - p - } - - /** - * sink - * - * @param dStream - * dStream - */ - def sink(dStream: DStream[T]): Unit = { - dStream.foreachRDD((rdd, time) => sink(rdd, time)) - } - - /** - * sink - * - * @param rdd - * spark.RDD - * @param time - * spark.streaming.Time - */ - def sink(rdd: RDD[T], time: Time): Unit -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-base/src/main/scala/org/apache/streampark/spark/connector/source/Source.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-base/src/main/scala/org/apache/streampark/spark/connector/source/Source.scala deleted file mode 100644 index e5c7624803..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-base/src/main/scala/org/apache/streampark/spark/connector/source/Source.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.source - -import org.apache.streampark.common.util.Logger - -import org.apache.spark.SparkConf -import org.apache.spark.streaming.StreamingContext -import org.apache.spark.streaming.dstream.DStream - -import scala.annotation.meta.getter -import scala.reflect.ClassTag -import scala.util.Try - -/** Base source trait */ -trait Source extends Logger with Serializable { - - @(transient @getter) - val ssc: StreamingContext - - @(transient @getter) - lazy val sparkConf: SparkConf = ssc.sparkContext.getConf - - val prefix: String - - lazy val param: Map[String, String] = sparkConf.getAll.flatMap { - case (k, v) if k.startsWith(prefix) && Try(v.nonEmpty).getOrElse(false) => - Some(k.substring(prefix.length) -> v) - case _ => None - }.toMap - - type SourceType - - /** get DStream */ - def getDStream[R: ClassTag](f: SourceType => R): DStream[R] -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/pom.xml b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/pom.xml deleted file mode 100644 index 8790074dd3..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/pom.xml +++ /dev/null @@ -1,81 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-spark-connector_2.12 - 2.2.0-SNAPSHOT - - - streampark-spark-connector-kafka_2.12 - StreamPark : Spark Connector Kafka - - - - - org.apache.streampark - streampark-spark-connector-base_2.12 - ${project.version} - - - redis.clients - jedis - ${redis.version} - - - org.scalikejdbc - scalikejdbc_2.12 - ${scalikejdbc.version} - - - org.apache.kafka - kafka_2.12 - ${kafka.version} - - - org.apache.hbase - hbase-common - ${hbase.version} - provided - - - org.apache.hbase - hbase-server - ${hbase.version} - provided - - - org.slf4j - slf4j-log4j12 - - - org.glassfish - javax.el - - - - - org.apache.hbase - hbase-client - ${hbase.version} - provided - - - diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/DefaultOffset.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/DefaultOffset.scala deleted file mode 100644 index 72846952f8..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/DefaultOffset.scala +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.kafka.offset - -import org.apache.kafka.common.TopicPartition -import org.apache.spark.SparkConf - -/** Default offset manager */ -private[kafka] class DefaultOffset(val sparkConf: SparkConf) extends Offset { - - /** - * get stored offset - * - * @param groupId - * @param topics - * @return - */ - override def get(groupId: String, topics: Set[String]): Map[TopicPartition, Long] = { - Map.empty[TopicPartition, Long] - } - - /** - * update offset - * - * @param groupId - * @param offsetInfos - */ - override def update(groupId: String, offsetInfos: Map[TopicPartition, Long]): Unit = {} - - /** - * delete offset - * - * @param groupId - * @param topics - */ - override def delete(groupId: String, topics: Set[String]): Unit = {} -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/HBaseOffset.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/HBaseOffset.scala deleted file mode 100644 index de7ca0a25c..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/HBaseOffset.scala +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.kafka.offset - -import org.apache.streampark.common.util.HBaseClient -import org.apache.streampark.common.util.Implicits._ - -import org.apache.hadoop.hbase._ -import org.apache.hadoop.hbase.client.{Delete, Put, Scan, Table} -import org.apache.hadoop.hbase.filter._ -import org.apache.hadoop.hbase.util.Bytes -import org.apache.kafka.common.TopicPartition -import org.apache.spark.SparkConf - -import java.util - -import scala.collection.mutable - -private[kafka] class HBaseOffset(val sparkConf: SparkConf) extends Offset { - - private lazy val tableName = storeParams("hbase.table") - private lazy val familyName = - storeParams.getOrElse("hbase.table.family", "tpo") - private lazy val familyNameBytes = Bytes.toBytes(familyName) - private lazy val topicBytes = Bytes.toBytes("topic") - private lazy val partitionBytes = Bytes.toBytes("partition") - private lazy val offsetBytes = Bytes.toBytes("offset") - - @transient - private lazy val table: Table = { - val conn = HBaseClient.apply(storeParams).connection - if (!conn.getAdmin.tableExists(TableName.valueOf(tableName))) { - val tableDesc: HTableDescriptor = new HTableDescriptor(TableName.valueOf(tableName)) - tableDesc.addFamily(new HColumnDescriptor(familyName)) - conn.getAdmin.createTable(tableDesc) - } - conn.getTable(TableName.valueOf(tableName)) - } - - /** - * get stored offset - * - * @param groupId - * @param topics - * @return - */ - override def get(groupId: String, topics: Set[String]): Map[TopicPartition, Long] = { - val storedOffsetMap = new mutable.HashMap[TopicPartition, Long]() - val earliestOffsets = getEarliestOffsets(topics.toSeq) - - topics.foreach(topic => { - val filter = new PrefixFilter(key(groupId, topic).getBytes) - val scan = new Scan().setFilter(filter) - val result = table.getScanner(scan) - result.foreach(r => { - var topic = "" - var partition = 0 - var offset = 0L - while (r.advance()) { - val cell = r.current() - Bytes.toString(CellUtil.cloneQualifier(cell)) match { - case "topic" => - topic = Bytes.toString(CellUtil.cloneValue(cell)) - case "partition" => - partition = Bytes.toInt(CellUtil.cloneValue(cell)) - case "offset" => - offset = Bytes.toLong(CellUtil.cloneValue(cell)) - case _ => - } - } - // if offset invalid, please use earliest offset to instead of - val topicPartition = new TopicPartition(topic, partition) - val finalOffset = earliestOffsets.get(topicPartition) match { - case Some(left) if left > offset => - logWarn( - s"storeType:HBase,consumer group:$groupId,topic:${topicPartition.topic},partition:${topicPartition.partition} offsets was timeOut,updated: $left") - left - case _ => offset - } - storedOffsetMap += topicPartition -> finalOffset - }) - result.close() - }) - - val offsetMaps = reset.toLowerCase() match { - case "latest" => getLatestOffsets(topics.toSeq) ++ storedOffsetMap - case _ => getEarliestOffsets(topics.toSeq) ++ storedOffsetMap - } - - logInfo(s"storeType:HBase,getOffsets [$groupId,${offsetMaps.mkString(",")}] ") - - offsetMaps - } - - /** - * update offset - * - * @param groupId - * @param offsetInfos - */ - override def update(groupId: String, offsetInfos: Map[TopicPartition, Long]): Unit = { - val puts = offsetInfos.map { - case (tp, offset) => - val put: Put = - new Put(Bytes.toBytes(s"${key(groupId, tp.topic)}#${tp.partition}")) - put.addColumn(familyNameBytes, topicBytes, Bytes.toBytes(tp.topic)) - put.addColumn(familyNameBytes, partitionBytes, Bytes.toBytes(tp.partition)) - put.addColumn(familyNameBytes, offsetBytes, Bytes.toBytes(offset)) - put - }.toList - table.put(puts) - logInfo(s"storeType:HBase,updateOffsets [ $groupId,${offsetInfos.mkString(",")} ]") - } - - /** - * delete offset - * - * @param groupId - * @param topics - */ - override def delete(groupId: String, topics: Set[String]): Unit = { - - val filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE) - - topics.foreach(topic => { - val filter = new RowFilter( - CompareFilter.CompareOp.EQUAL, - new BinaryPrefixComparator(Bytes.toBytes(s"${key(groupId, topic)}#"))) - filterList.addFilter(filter) - }) - - val scan = new Scan() - scan.setFilter(filterList) - - val rs = table.getScanner(scan) - val iter = rs.iterator() - - val deletes = new util.ArrayList[Delete]() - while (iter.hasNext) { - val r = iter.next() - deletes.add(new Delete(Bytes.toBytes(new String(r.getRow)))) - } - rs.close() - table.delete(deletes) - logInfo( - s"storeType:HBase,deleteOffsets [ $groupId,${topics.mkString(",")} ] ${deletes.mkString(" ")}") - } -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/KafkaClient.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/KafkaClient.scala deleted file mode 100644 index c7a1c4d26c..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/KafkaClient.scala +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.kafka.offset - -import org.apache.streampark.common.util.Logger - -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.apache.kafka.common.TopicPartition -import org.apache.spark.{SparkConf, SparkContext} -import org.apache.spark.rdd.RDD -import org.apache.spark.streaming.StreamingContext -import org.apache.spark.streaming.dstream.InputDStream -import org.apache.spark.streaming.kafka010._ - -import java.{util => ju} -import java.lang.reflect.Constructor - -import scala.reflect.ClassTag - -class KafkaClient(val sparkConf: SparkConf) extends Logger with Serializable { - - private lazy val offsetManager = { - sparkConf.get("spark.source.kafka.offset.store.class", "none").trim match { - case "none" => - sparkConf - .get("spark.source.kafka.offset.store.type", "none") - .trim - .toLowerCase match { - case "redis" => new RedisOffset(sparkConf) - case "hbase" => new HBaseOffset(sparkConf) - case "kafka" => new DefaultOffset(sparkConf) - case "mysql" => new MySQLOffset(sparkConf) - case "none" => new DefaultOffset(sparkConf) - } - case clazz => - logInfo(s"Custom offset management class $clazz") - val constructors = { - val offsetsManagerClass = Class.forName(clazz) - offsetsManagerClass.getConstructors - .asInstanceOf[Array[Constructor[_ <: SparkConf]]] - } - val constructorTakingSparkConf = constructors.find { - c => c.getParameterTypes.sameElements(Array(classOf[SparkConf])) - } - constructorTakingSparkConf.get - .newInstance(sparkConf) - .asInstanceOf[Offset] - } - } - - def offsetStoreType: String = offsetManager.storeType - - private var canCommitOffsets: CanCommitOffsets = _ - - def createDirectStream[K: ClassTag, V: ClassTag]( - ssc: StreamingContext, - kafkaParams: Map[String, Object], - topics: Set[String]): InputDStream[ConsumerRecord[K, V]] = { - - var consumerOffsets = Map.empty[TopicPartition, Long] - - kafkaParams.get("group.id") match { - case Some(groupId) => - consumerOffsets = offsetManager.get(groupId.toString, topics) - logInfo(s"createDirectStream witch group.id $groupId topics ${topics.mkString(",")}") - case _ => - logInfo(s"createDirectStream witchOut group.id topics ${topics.mkString(",")}") - } - - if (consumerOffsets.nonEmpty) { - logInfo(s"read topics ==[$topics]== from offsets ==[$consumerOffsets]==") - val stream = KafkaUtils.createDirectStream[K, V]( - ssc, - LocationStrategies.PreferConsistent, - ConsumerStrategies - .Assign[K, V](consumerOffsets.keys, kafkaParams, consumerOffsets)) - canCommitOffsets = stream.asInstanceOf[CanCommitOffsets] - stream - } else { - val stream = KafkaUtils.createDirectStream[K, V]( - ssc, - LocationStrategies.PreferConsistent, - ConsumerStrategies.Subscribe[K, V](topics, kafkaParams)) - canCommitOffsets = stream.asInstanceOf[CanCommitOffsets] - stream - } - - } - - def createRDD[K: ClassTag, V: ClassTag]( - sc: SparkContext, - kafkaParams: ju.Map[String, Object], - offsetRanges: Array[OffsetRange], - locationStrategy: LocationStrategy): RDD[ConsumerRecord[K, V]] = { - KafkaUtils.createRDD(sc, kafkaParams, offsetRanges, locationStrategy) - } - - def updateOffset(groupId: String, offsetRanges: Array[OffsetRange]): Unit = { - offsetStoreType match { - case "kafka" => canCommitOffsets.commitAsync(offsetRanges) - case _ => - val tps = - offsetRanges - .map(x => new TopicPartition(x.topic, x.partition) -> x.untilOffset) - .toMap - offsetManager.update(groupId, tps) - } - } - -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/MySQLOffset.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/MySQLOffset.scala deleted file mode 100644 index 4f2a778ab5..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/MySQLOffset.scala +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.kafka.offset - -import org.apache.kafka.common.TopicPartition -import org.apache.spark.SparkConf -import scalikejdbc.{ConnectionPool, DB, _} - -/** - * MySQL Offset Manager - * - * The table model for storing offsets is as follow, (topic+groupId+partition as union unique - * primary key) - * - * | ---------------------------------------------------------------------------------- | | | | - * |:-----------------------------------------------------------------------------------|:------------|:----------|:-------| - * | topic | groupId | partition | offset | - * | ---------------------------------------------------------------------------------- | | | | - * | topic_001 | groupId_001 | 0 | 197 | - * | ---------------------------------------------------------------------------------- | | | | - * | topic_001 | groupId_001 | 1 | 200 | - * | ---------------------------------------------------------------------------------- | | | | - */ -private[kafka] class MySQLOffset(val sparkConf: SparkConf) extends Offset { - - private lazy val jdbcURL = storeParams("mysql.jdbc.url") - private lazy val table = storeParams("mysql.table") - private lazy val user = storeParams("mysql.user") - private lazy val password = storeParams("mysql.password") - - /** - * get stored offset - * - * @param groupId - * @param topics - * @return - */ - override def get(groupId: String, topics: Set[String]): Map[TopicPartition, Long] = { - require(topics.nonEmpty) - ConnectionPool.singleton(jdbcURL, user, password) - DB.getTable(table) match { - case None => - DB.autoCommit { - implicit session => - val sql = - s""" - |create table if not exists $table ( - |`topic` varchar(255), - |`groupId` varchar(255), - |`partition` varchar(255), - |`offset` int(10), - |UNIQUE INDEX `INX`(`topic`, `groupId`,`partition`) - |) - """.stripMargin - SQL(sql).execute.apply() - } - logWarn(s"storeType:MySQL,table: $table is not exist,auto created...") - Map.empty[TopicPartition, Long] - case Some(_) => - DB.readOnly { - implicit session => - val where = topics.size match { - case 1 => s""" `topic` = "${topics.head}" """ - case _ => - s""" `topic` in (${topics.mkString("\"", "\",\"", "\"")}) """ - } - val sql = - s"select `topic`,`partition`,`offset` from $table where `groupId`=? and $where" - SQL(sql) - .bind(groupId) - .map { - result => - new TopicPartition(result.string(1), result.int(2)) -> result - .long(3) - } - .list - .apply() - .toMap - } - } - } - - /** - * update offset - * - * @param groupId - * @param offsetInfos - */ - override def update(groupId: String, offsetInfos: Map[TopicPartition, Long]): Unit = { - DB.localTx { - implicit session => - offsetInfos.foreach { - case (tp, offset) => - val sql = - s"insert into $table(`topic`,`groupId`,`partition`,`offset`) values(?,?,?,?) on duplicate key update `offset`= values(`offset`) " - val updated = SQL(sql) - .bind(tp.topic(), groupId, tp.partition(), offset) - .update() - if (updated == 0) { - throw new Exception(s"Commit kafka topic :${tp.topic()} failed!") - } - logInfo(s"storeType:MySQL,updateOffsets [ $groupId,${offsetInfos.mkString(",")} ]") - } - } - } - - /** - * delete offset - * - * @param groupId - * @param topics - */ - override def delete(groupId: String, topics: Set[String]): Unit = { - DB.autoCommit { - implicit session => - topics.foreach(topic => { - val sql = "delete from $table where topic=? and groupId=?" - SQL(sql).bind(topic, groupId).update() - }) - } - logInfo(s"storeType:MySQL,deleteOffsets [ $groupId,${topics.mkString(",")} ]") - } -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/Offset.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/Offset.scala deleted file mode 100644 index eb8f87b571..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/Offset.scala +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.kafka.offset - -import org.apache.streampark.common.util.Implicits._ -import org.apache.streampark.common.util.Logger - -import org.apache.kafka.common.TopicPartition -import org.apache.spark.SparkConf - -import java.util.Properties - -import scala.language.implicitConversions -import scala.util.Try - -/** Offset Manager */ -trait Offset extends Logger with Serializable { - - val sparkConf: SparkConf - - lazy val storeType: String = storeParams.getOrElse("type", "none") - - implicit lazy val storeParams: Map[String, String] = - sparkConf.getAllWithPrefix(s"spark.source.kafka.offset.store.").toMap - - implicit def toProperty(map: Map[String, String]): Properties = { - require(map != null) - val prop = new Properties() - map.foreach((a) => prop.setProperty(a._1, a._2)) - prop - } - - lazy val reset: String = - sparkConf.get("spark.source.kafka.consume.auto.offset.reset", "largest") - - lazy val (host, port) = sparkConf - .get("spark.source.kafka.consume.bootstrap.servers") - .split(",") - .head - .split(":") match { - case Array(h, p) => (h, p.toInt) - } - - /** - * get stored offset - * - * @param groupId - * @param topics - * @return - */ - def get(groupId: String, topics: Set[String]): Map[TopicPartition, Long] - - /** - * update offset - * - * @param groupId - * @param offsetInfos - */ - def update(groupId: String, offsetInfos: Map[TopicPartition, Long]): Unit - - /** - * delete offset - * - * @param groupId - * @param topics - */ - def delete(groupId: String, topics: Set[String]): Unit - - /** - * generate key - * - * @param groupId - * @param topic - * @return - */ - def key(groupId: String, topic: String): String = s"$groupId#$topic" - - final private val LatestTime = -1L - final private val EarliestTime = -2L - - /** - * get earliest offset - * - * @param topics - * @return - */ - def getEarliestOffsets(topics: Seq[String]): Map[TopicPartition, Long] = - getOffsets(topics, EarliestTime) - - /** - * get latest offset - * - * @param topics - * @return - */ - def getLatestOffsets(topics: Seq[String]): Map[TopicPartition, Long] = - getOffsets(topics, LatestTime) - - /** - * get specific timestamp offset - * - * @param topics - * @param time - * @return - */ - private def getOffsets(topics: Seq[String], time: Long): Map[TopicPartition, Long] = { - - import org.apache.kafka.clients.consumer.KafkaConsumer - val props = new Properties() - props.setProperty("bootstrap.servers", s"$host:$port") - props.setProperty("group.id", s"offsetLookup-${System.currentTimeMillis()}") - props.setProperty("enable.auto.commit", "false") - props.setProperty( - "key.deserializer", - "org.apache.kafka.common.serialization.StringDeserializer") - props.setProperty( - "value.deserializer", - "org.apache.kafka.common.serialization.StringDeserializer") - val consumer = new KafkaConsumer[String, String](props) - - val partitions = topics.flatMap(topic => { - consumer - .partitionsFor(topic) - .asScala - .map(x => new TopicPartition(x.topic(), x.partition())) - }) - - val offsetInfos = time match { - case EarliestTime => - consumer.beginningOffsets(partitions.asJavaCollection) - case LatestTime => consumer.endOffsets(partitions.asJavaCollection) - } - - Try(consumer.close()) - - offsetInfos.asInstanceOf[Map[TopicPartition, Long]] - } - -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/RedisOffset.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/RedisOffset.scala deleted file mode 100644 index c512fc06e1..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/offset/RedisOffset.scala +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.kafka.offset - -import org.apache.streampark.common.util.{RedisEndpoint, RedisUtils} -import org.apache.streampark.common.util.Implicits._ - -import org.apache.kafka.common.TopicPartition -import org.apache.spark.SparkConf -import redis.clients.jedis.Protocol - -import scala.util.Try - -/** Redis Offset Manager */ -private[kafka] class RedisOffset(val sparkConf: SparkConf) extends Offset { - - implicit private[this] def endpoint(implicit params: Map[String, String]): RedisEndpoint = { - val host = params.getOrElse("redis.hosts", Protocol.DEFAULT_HOST) - val port = - params.getOrElse("redis.port", Protocol.DEFAULT_PORT.toString).toInt - val auth = Try(params("redis.auth")).getOrElse(null) - val dbNum = - params.getOrElse("redis.db", Protocol.DEFAULT_DATABASE.toString).toInt - val timeout = - params.getOrElse("redis.timeout", Protocol.DEFAULT_TIMEOUT.toString).toInt - RedisEndpoint(host, port, auth, dbNum, timeout) - } - - override def get(groupId: String, topics: Set[String]): Map[TopicPartition, Long] = { - val earliestOffsets = getEarliestOffsets(topics.toSeq) - val offsetMap = RedisUtils.doRedis { - redis => - topics.flatMap(topic => { - redis.hgetAll(key(groupId, topic)).map { - case (partition, offset) => - // if offset invalid, please use earliest offset to instead of - val tp = new TopicPartition(topic, partition.toInt) - val finalOffset = earliestOffsets.get(tp) match { - case Some(left) if left > offset.toLong => - logWarn( - s"storeType:Redis,consumer group:$groupId,topic:${tp.topic},partition:${tp.partition} offsets Outdated,updated:$left") - left - case _ => offset.toLong - } - tp -> finalOffset - } - }) - } - - val offsetMaps = reset.toLowerCase() match { - case "largest" => getLatestOffsets(topics.toSeq) ++ offsetMap - case _ => getEarliestOffsets(topics.toSeq) ++ offsetMap - } - logInfo(s"getOffsets [$groupId,${offsetMaps.mkString(",")}] ") - offsetMaps - } - - override def update(groupId: String, offsets: Map[TopicPartition, Long]): Unit = { - RedisUtils.doRedis { - redis => - offsets.foreach { - case (tp, offset) => - redis.hset(key(groupId, tp.topic), tp.partition().toString, offset.toString) - } - } - logInfo(s"storeType:Redis,updateOffsets [ $groupId,${offsets.mkString(",")} ]") - } - - override def delete(groupId: String, topics: Set[String]): Unit = { - RedisUtils.doRedis(redis => topics.foreach(x => redis.del(key(groupId, x)))) - logInfo(s"storeType:Redis,deleteOffsets [ $groupId,${topics.mkString(",")} ]") - } - -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/sink/KafkaSink.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/sink/KafkaSink.scala deleted file mode 100644 index 0a485baab5..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/sink/KafkaSink.scala +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.kafka.sink - -import org.apache.streampark.spark.connector.kafka.writer.KafkaWriter.createKafkaOutputWriter -import org.apache.streampark.spark.connector.sink.Sink - -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.spark.SparkContext -import org.apache.spark.rdd.RDD -import org.apache.spark.streaming.Time - -import java.util.UUID - -import scala.reflect.ClassTag - -class KafkaSink[T: ClassTag]( - @transient override val sc: SparkContext, - initParams: Map[String, String] = Map.empty[String, String]) - extends Sink[T] { - - override val prefix: String = "spark.sink.kafka." - - private lazy val prop = filterProp(param, initParams, prefix) - - private val outputTopic = prop.getProperty("topic") - - /** Sink to kafka as a string */ - override def sink(rdd: RDD[T], time: Time = Time(System.currentTimeMillis())): Unit = { - rdd.writeToKafka( - prop, - x => new ProducerRecord[String, String](outputTopic, UUID.randomUUID().toString, x.toString)) - } -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/source/KafkaSource.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/source/KafkaSource.scala deleted file mode 100644 index e83eb70ad1..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/source/KafkaSource.scala +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.kafka.source - -import org.apache.streampark.spark.connector.kafka.offset.KafkaClient -import org.apache.streampark.spark.connector.source.Source - -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.apache.spark.streaming.{StreamingContext, Time} -import org.apache.spark.streaming.dstream.DStream -import org.apache.spark.streaming.kafka010.{HasOffsetRanges, OffsetRange} - -import java.util.concurrent.ConcurrentHashMap - -import scala.language.postfixOps -import scala.reflect.ClassTag -import scala.util.Try - -/** - * Wrapped Kafka Direct Api - * - * @param ssc - * StreamingContext - * @param overrideParams - * specific kafka params - */ -class KafkaSource[K: ClassTag, V: ClassTag]( - @transient val ssc: StreamingContext, - overrideParams: Map[String, String] = Map.empty[String, String]) - extends Source { - - override val prefix: String = "spark.source.kafka.consume." - - // partition nums - lazy val repartition: Int = - sparkConf.get("spark.source.kafka.consume.repartition", "0").toInt - - // kafka consume topic - private lazy val topicSet: Set[String] = - overrideParams - .getOrElse("consume.topics", sparkConf.get("spark.source.kafka.consume.topics")) - .split(",") - .map(_.trim) - .toSet - - // assemble kafka params - private lazy val kafkaParams: Map[String, String] = { - sparkConf.getAll.flatMap { - case (k, v) if k.startsWith(prefix) && Try(v.nonEmpty).getOrElse(false) => - Some(k.substring(prefix.length) -> v) - case _ => None - } toMap - } ++ overrideParams ++ Map("enable.auto.commit" -> "false") - - lazy val groupId: Option[String] = kafkaParams.get("group.id") - - override type SourceType = ConsumerRecord[K, V] - - val kafkaClient = new KafkaClient(ssc.sparkContext.getConf) - - private lazy val offsetRanges: java.util.Map[Long, Array[OffsetRange]] = - new ConcurrentHashMap[Long, Array[OffsetRange]] - - /** Get DStream */ - override def getDStream[R: ClassTag](recordHandler: ConsumerRecord[K, V] => R): DStream[R] = { - val stream = - kafkaClient.createDirectStream[K, V](ssc, kafkaParams, topicSet) - stream - .transform((rdd, time) => { - offsetRanges.put(time.milliseconds, rdd.asInstanceOf[HasOffsetRanges].offsetRanges) - rdd - }) - .map(recordHandler) - } - - /** - * Update offset Note that: Must be placed at the end of all logical code, it ensures that the - * offset is updated only after the action is executed successfully. - */ - def updateOffset(time: Time): Unit = { - val milliseconds = time.milliseconds - if (groupId.isDefined) { - logInfo( - s"updateOffset with ${kafkaClient.offsetStoreType} for time $milliseconds offsetRanges: $offsetRanges") - val offsetRange = offsetRanges.get(milliseconds) - kafkaClient.updateOffset(groupId.get, offsetRange) - } - offsetRanges.remove(milliseconds) - } - -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/DStreamKafkaWriter.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/DStreamKafkaWriter.scala deleted file mode 100644 index c9800482f1..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/DStreamKafkaWriter.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.kafka.writer - -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.spark.streaming.dstream.DStream - -import java.util.Properties - -import scala.annotation.meta.param -import scala.reflect.ClassTag - -class DStreamKafkaWriter[T: ClassTag](@(transient @param) dstream: DStream[T]) - extends KafkaWriter[T] { - - /** - * @param producerConfig - * The configuration that can be used to connect to Kafka - * @param serializerFunc - * The function to convert the data from the stream into Kafka [[ProducerRecord]]s. - * @tparam K - * The type of the key - * @tparam V - * The type of the value - */ - override def writeToKafka[K, V]( - producerConfig: Properties, - serializerFunc: T => ProducerRecord[K, V]): Unit = { - dstream.foreachRDD { - rdd => - val rddWriter = new RDDKafkaWriter[T](rdd) - rddWriter.writeToKafka(producerConfig, serializerFunc) - } - } -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/IterKafkaWriter.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/IterKafkaWriter.scala deleted file mode 100644 index 9b38c3ed9a..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/IterKafkaWriter.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.kafka.writer - -import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} - -import java.util.Properties - -import scala.annotation.meta.param -import scala.reflect.ClassTag - -/** A simple Kafka producers */ -class IterKafkaWriter[T: ClassTag](@(transient @param) msg: Iterator[T]) extends KafkaWriter[T] { - - /** - * @param producerConfig - * The configuration that can be used to connect to Kafka - * @param serializerFunc - * The function to convert the data from the stream into Kafka [[ProducerRecord]]s. - * @tparam K - * The type of the key - * @tparam V - * The type of the value - */ - override def writeToKafka[K, V]( - producerConfig: Properties, - serializerFunc: (T) => ProducerRecord[K, V]): Unit = { - val producer: KafkaProducer[K, V] = KafkaWriter.getProducer(producerConfig) - msg.map(serializerFunc).foreach(producer.send) - } -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/KafkaWriter.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/KafkaWriter.scala deleted file mode 100644 index 26e35ee11c..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/KafkaWriter.scala +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.kafka.writer - -import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} -import org.apache.spark.rdd.RDD -import org.apache.spark.streaming.dstream.DStream - -import java.util.Properties - -import scala.collection.mutable -import scala.language.implicitConversions -import scala.reflect.ClassTag - -/** - * Import this object in this form: - * {{{ - * import org.cloudera.spark.streaming.kafka.KafkaWriter._ - * }}} - * - * Once this is done, the `writeToKafka` can be called on the [[DStream]] object in this form: - * {{{ - * dstream.writeToKafka(producerConfig, f) - * }}} - */ -private[kafka] object KafkaWriter { - - private val producers = new mutable.HashMap[Properties, Any]() - - def getProducer[K, V](config: Properties): KafkaProducer[K, V] = { - producers - .getOrElse( - config, { - val producer = new KafkaProducer[K, V](config) - producers(config) = producer - producer - }) - .asInstanceOf[KafkaProducer[K, V]] - } - - /** - * This implicit method allows the user to call dstream.writeToKafka(..) - * - * @param dstream - * \- DStream to write to Kafka - * @tparam T - * \- The type of the DStream - * @tparam K - * \- The type of the key to serialize to - * @tparam V - * \- The type of the value to serialize to - * @return - */ - implicit def createKafkaOutputWriter[T: ClassTag, K, V](dstream: DStream[T]): KafkaWriter[T] = { - new DStreamKafkaWriter[T](dstream) - } - - implicit def createKafkaOutputWriter[T: ClassTag, K, V](rdd: RDD[T]): KafkaWriter[T] = { - new RDDKafkaWriter[T](rdd) - } - - implicit def createKafkaOutputWriter[T: ClassTag, K, V](msg: Iterator[T]): KafkaWriter[T] = { - new IterKafkaWriter[T](msg) - } - - implicit def createKafkaOutputWriter[T: ClassTag, K, V](msg: T): KafkaWriter[T] = { - new SimpleKafkaWriter[T](msg) - } -} - -/** - * This class can be used to write data to Kafka from Spark Streaming. To write data to Kafka simply - * `import org.cloudera.spark.streaming.kafka.KafkaWriter._` in your application and call - * `dstream.writeToKafka(producerConf, func)` - * - * Here is an example: - * {{{ - * // Adding this line allows the user to call dstream.writeDStreamToKafka(..) - * import org.apache.spark.streaming.kafka.KafkaWriter._ - * - * class ExampleWriter { - * val instream = ssc.queueStream(toBe) - * - * val props = new Properties() - * props.put("bootstrap.servers", "localhost:9092") - * props.put("acks", "all") - * props.put("retries", 0) - * props.put("batch.size", 16384) - * props.put("linger.ms", 1) - * props.put("buffer.memory", 33554432) - * props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") - * props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") - * - * instream.writeToKafka(props, - * x => new ProducerRecord[String, String](sinkTopic, UUID.randomUUID().toString, x.toString)) - * - * ssc.start() - * } - * - * }}} - */ -abstract class KafkaWriter[T: ClassTag]() extends Serializable { - - /** - * To write data from a DStream to Kafka, call this function after creating the DStream. Once the - * DStream is passed into this function, all data coming from the DStream is written out to Kafka. - * The properties instance takes the configuration required to connect to the Kafka brokers in the - * standard Kafka format. The serializerFunc is a function that converts each element of the RDD - * to a Kafka [[ProducerRecord]]. This closure should be serializable - so it should use only - * instances of Serializables. - * - * @param producerConfig - * The configuration that can be used to connect to Kafka - * @param serializerFunc - * The function to convert the data from the stream into Kafka [[ProducerRecord]]s. - * @tparam K - * The type of the key - * @tparam V - * The type of the value - */ - def writeToKafka[K, V]( - producerConfig: Properties, - serializerFunc: T => ProducerRecord[K, V]): Unit -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/RDDKafkaWriter.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/RDDKafkaWriter.scala deleted file mode 100644 index f5fee024b4..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/RDDKafkaWriter.scala +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.kafka.writer - -import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} -import org.apache.spark.rdd.RDD - -import java.util.Properties - -import scala.annotation.meta.param -import scala.reflect.ClassTag - -class RDDKafkaWriter[T: ClassTag](@(transient @param) rdd: RDD[T]) extends KafkaWriter[T] { - - /** - * @param producerConfig - * The configuration that can be used to connect to Kafka - * @param serializerFunc - * The function to convert the data from the stream into Kafka [[ProducerRecord]]s. - * @tparam K - * The type of the key - * @tparam V - * The type of the value - */ - override def writeToKafka[K, V]( - producerConfig: Properties, - serializerFunc: (T) => ProducerRecord[K, V]): Unit = { - rdd.foreachPartition(events => { - val producer: KafkaProducer[K, V] = - KafkaWriter.getProducer(producerConfig) - events.map(serializerFunc).foreach(producer.send) - }) - } -} diff --git a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/SimpleKafkaWriter.scala b/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/SimpleKafkaWriter.scala deleted file mode 100644 index de43a0a174..0000000000 --- a/streampark-spark/streampark-spark-connector/streampark-spark-connector-kafka/src/main/scala/org/apache/streampark/spark/connector/kafka/writer/SimpleKafkaWriter.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.connector.kafka.writer - -import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} - -import java.util.Properties - -import scala.annotation.meta.param -import scala.reflect.ClassTag - -/** A simple Kafka producers */ -class SimpleKafkaWriter[T: ClassTag](@(transient @param) msg: T) extends KafkaWriter[T] { - - /** - * @param producerConfig - * The configuration that can be used to connect to Kafka - * @param serializerFunc - * The function to convert the data from the stream into Kafka [[ProducerRecord]]s. - * @tparam K - * The type of the key - * @tparam V - * The type of the value - */ - override def writeToKafka[K, V]( - producerConfig: Properties, - serializerFunc: (T) => ProducerRecord[K, V]): Unit = { - val producer: KafkaProducer[K, V] = KafkaWriter.getProducer(producerConfig) - producer.send(serializerFunc(msg)) - } -} diff --git a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/SparkBatch.scala b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/SparkBatch.scala deleted file mode 100644 index 1f8a6494ca..0000000000 --- a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/SparkBatch.scala +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.core - -import org.apache.spark.SparkContext - -import scala.annotation.meta.getter - -/** - * SparkBatch

Spark batch processing entry

Creation Time: - * 2022/8/8 20:44. - */ -trait SparkBatch extends Spark { - - @(transient @getter) - protected lazy val context: SparkContext = sparkSession.sparkContext - - override def destroy(): Unit = { - context.stop() - } - -} diff --git a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/SparkStreaming.scala b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/SparkStreaming.scala deleted file mode 100644 index 99a0540d8d..0000000000 --- a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/SparkStreaming.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.core - -import org.apache.streampark.common.conf.ConfigKeys._ - -import org.apache.spark.SparkConf -import org.apache.spark.streaming.{Seconds, StreamingContext} - -import scala.annotation.meta.getter - -/** SparkBatch

Spark streaming handle entrance

*/ -trait SparkStreaming extends Spark { - - @(transient @getter) - protected lazy val context: StreamingContext = { - - /** Construct StreamingContext */ - def _context(): StreamingContext = { - val duration = sparkConf.get(KEY_SPARK_BATCH_DURATION).toInt - new StreamingContext(sparkSession.sparkContext, Seconds(duration)) - } - - checkpoint match { - case "" => _context() - case checkpointPath => - val tmpContext = - StreamingContext.getOrCreate(checkpointPath, _context, createOnError = createOnError) - tmpContext.checkpoint(checkpointPath) - tmpContext - } - } - - final override def start(): Unit = { - context.start() - context.awaitTermination() - } - - /** - * The purpose of the config phase is to allow the developer to set more parameters (other than - * the agreed configuration file) by means of hooks. Such as, conf.set("spark.serializer", - * "org.apache.spark.serializer.KryoSerializer") conf.registerKryoClasses(Array(classOf[User], - * classOf[Order],...)) - */ - override def config(sparkConf: SparkConf): Unit = {} - - /** - * The ready phase is an entry point for the developer to do other actions after the parameters - * have been set, and is done after initialization and before the program starts. - */ - override def ready(): Unit = {} - - /** - * The destroy phase, is the last phase before jvm exits after the program has finished running, - * and is generally used to wrap up the work. - */ - override def destroy(): Unit = {} - -} diff --git a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleAvroOutputsFormat.scala b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleAvroOutputsFormat.scala deleted file mode 100644 index dbdd23ac32..0000000000 --- a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleAvroOutputsFormat.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.core.serializable - -import org.apache.avro.generic.GenericContainer -import org.apache.avro.mapred.AvroKey -import org.apache.avro.mapreduce.{AvroKeyOutputFormat, AvroMultipleOutputs} -import org.apache.hadoop.io.NullWritable -import org.apache.hadoop.mapreduce.TaskInputOutputContext - -object MultipleAvroOutputsFormat { - // This seems to be an unfortunate limitation of type inference of lambda defaults within constructor params. - // If it would work I would just inline this function - def amoMaker[T](io: TaskInputOutputContext[_, _, AvroKey[T], NullWritable]): MultipleOutputer[AvroKey[T], NullWritable] = new AvroMultipleOutputs(io) -} - -class MultipleAvroOutputsFormat[T <: GenericContainer] - extends MultipleOutputsFormat( - new AvroKeyOutputFormat[T], - (io: TaskInputOutputContext[_, _, AvroKey[T], NullWritable]) => - MultipleAvroOutputsFormat.amoMaker(io)) {} diff --git a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleOutputer.scala b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleOutputer.scala deleted file mode 100644 index 7223b0a60b..0000000000 --- a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleOutputer.scala +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.core.serializable - -import org.apache.avro.mapreduce.AvroMultipleOutputs -import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs - -/** - * Multiple dir output - * - * @tparam K - * key - * @tparam V - * value - */ -trait MultipleOutputer[K, V] { - def write(key: K, value: V, path: String): Unit - - def close(): Unit -} - -object MultipleOutputer { - - /** - * Avro multi files output - * - * @param mo - * @tparam K - * @tparam V - */ - implicit class AvroMultipleOutputer[K, V](mo: AvroMultipleOutputs) - extends MultipleOutputer[K, V] { - def write(key: K, value: V, path: String): Unit = mo.write(key, value, path) - - def close(): Unit = mo.close() - } - - /** - * Unformatted multipath output - * - * @param mo - * @tparam K - * @tparam V - */ - implicit class PlainMultipleOutputer[K, V](mo: MultipleOutputs[K, V]) - extends MultipleOutputer[K, V] { - def write(key: K, value: V, path: String): Unit = mo.write(key, value, path) - - def close(): Unit = mo.close() - } - -} diff --git a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleOutputsFormat.scala b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleOutputsFormat.scala deleted file mode 100644 index ca521ab6f6..0000000000 --- a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleOutputsFormat.scala +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.core.serializable - -import org.apache.hadoop.io.{DataInputBuffer, NullWritable} -import org.apache.hadoop.mapred.RawKeyValueIterator -import org.apache.hadoop.mapreduce.{Job, _} -import org.apache.hadoop.mapreduce.counters.GenericCounter -import org.apache.hadoop.mapreduce.lib.output.{LazyOutputFormat, MultipleOutputs} -import org.apache.hadoop.mapreduce.task.ReduceContextImpl -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl.DummyReporter -import org.apache.hadoop.util.Progress - -object MultipleOutputsFormat { - // Type inference fails with this inlined in constructor parameters - private def defaultMultipleOutputsMaker[K, V]( - io: TaskInputOutputContext[_, _, K, V]): MultipleOutputer[K, V] = - new MultipleOutputs[K, V](io) -} - -/** - * Use this abstract class to create multiple outputs of OutputFormat. The output format requires a - * two-part key (outputPath and actualKey), and the outputPath will be used to output into different - * directories ('/' separated by filenames). - * - * @param outputFormat - * OutputFormat - * @param multipleOutputsMaker - * Methods that implement the MultipleOutputer trait - * @tparam K - * Basic OutputFormat's key type - * @tparam V - * Basic OutputFormat's value type - */ -abstract class MultipleOutputsFormat[K, V]( - outputFormat: OutputFormat[K, V], - multipleOutputsMaker: TaskInputOutputContext[_, _, K, V] => MultipleOutputer[K, V] = - (r: TaskInputOutputContext[_, _, K, V]) => - MultipleOutputsFormat.defaultMultipleOutputsMaker[K, V](r)) - extends OutputFormat[(String, K), V] { - - /** - * Check for validity of the output-specification for the job. - * - *

This is to validate the output specification for the job when it is a job is submitted. - * Typically checks that it does not already exist, throwing an exception when it already exists, - * so that output is not overwritten.

- * - * @param context - * information about the job - * @throws IOException - * when output should not be attempted - */ - override def checkOutputSpecs(context: JobContext): Unit = - outputFormat.checkOutputSpecs(context) - - /** - * Get the output committer for this output format. This is responsible for ensuring the output is - * committed correctly. - * - * @param context - * the task context - * @return - * an output committer - * @throws IOException - * @throws InterruptedException - */ - override def getOutputCommitter(context: TaskAttemptContext): OutputCommitter = outputFormat - .getOutputCommitter(context) - - /** - * Get the {@link RecordWriter} for the given task. - * - * @param context - * the information about the current task. - * @return - * a { @link RecordWriter} to write the output for the job. - * @throws IOException - */ - override def getRecordWriter(context: TaskAttemptContext): RecordWriter[(String, K), V] = - new RecordWriter[(String, K), V] { - - val job: Job = Job.getInstance(context.getConfiguration) - LazyOutputFormat.setOutputFormatClass(job, outputFormat.getClass) - // of Spark's saveAs*Hadoop* methods - val ioContext = new ReduceContextImpl( - job.getConfiguration, - context.getTaskAttemptID, - new DummyIterator, - new GenericCounter, - new GenericCounter, - new DummyRecordWriter, - new DummyOutputCommitter, - new DummyReporter, - null, - classOf[NullWritable], - classOf[NullWritable]) - - val multipleOutputs: MultipleOutputer[K, V] = multipleOutputsMaker(ioContext) - - /** - * Writes a keys/value pair. - * - * @param keys - * the key to write. - * @param value - * the value to write. - * @throws IOException - */ - override def write(keys: (String, K), value: V): Unit = { - keys match { - case (path, key) => - multipleOutputs.write(key, value, path) - } - } - - override def close(context: TaskAttemptContext): Unit = - multipleOutputs.close() - } - - private class DummyOutputCommitter extends OutputCommitter { - override def setupJob(jobContext: JobContext): Unit = () - - override def needsTaskCommit(taskContext: TaskAttemptContext): Boolean = - false - - override def setupTask(taskContext: TaskAttemptContext): Unit = () - - override def commitTask(taskContext: TaskAttemptContext): Unit = () - - override def abortTask(taskContext: TaskAttemptContext): Unit = () - } - - private class DummyRecordWriter extends RecordWriter[K, V] { - override def write(key: K, value: V): Unit = () - - override def close(context: TaskAttemptContext): Unit = () - } - - private class DummyIterator extends RawKeyValueIterator { - override def getKey: DataInputBuffer = null - - override def getValue: DataInputBuffer = null - - override def getProgress: Progress = null - - override def close(): Unit = () - - override def next: Boolean = true - } - -} diff --git a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleTextOutputsFormat.scala b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleTextOutputsFormat.scala deleted file mode 100644 index d816c3e29b..0000000000 --- a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/MultipleTextOutputsFormat.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.core.serializable - -import org.apache.hadoop.io.{NullWritable, Text} -import org.apache.hadoop.mapreduce.lib.output.{TextOutputFormat => OutFormat} - -class MultipleTextOutputsFormat extends MultipleOutputsFormat(new OutFormat[NullWritable, Text]) {} diff --git a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/SerializationExtensions.scala b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/SerializationExtensions.scala deleted file mode 100644 index 597a5e97b8..0000000000 --- a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/SerializationExtensions.scala +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.core.serializable - -import org.apache.avro.Schema -import org.apache.avro.generic.GenericRecord -import org.apache.avro.mapred.AvroKey -import org.apache.avro.mapreduce.{AvroJob, AvroKeyInputFormat, AvroKeyOutputFormat} -import org.apache.avro.specific.SpecificData -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.io.NullWritable -import org.apache.hadoop.mapreduce.Job -import org.apache.log4j.Logger -import org.apache.spark.SparkContext -import org.apache.spark.rdd.RDD - -import scala.reflect.{classTag, ClassTag} - -object SerializationExtensions { - - private val logger = Logger.getLogger(getClass) - - /** - * Build a AvroJob - * - * @param job - * @tparam T - * @return - */ - def avroJob[T <: GenericRecord: ClassTag]( - job: Job = Job.getInstance(new Configuration())): Job = { - val schema: Schema = SpecificData.get.getSchema(classTag[T].runtimeClass) - AvroJob.setInputKeySchema(job, schema) - AvroJob.setOutputKeySchema(job, schema) - job - } - - def isDefined(record: GenericRecord, field: String): Boolean = { - if (record.get(field) != null) return true - logger.warn( - s"Expected field '$field' to be defined, but it was not on record of type '${record.getClass}'") - false - } - - implicit class AvroRDDSparkContext(val sparkContext: SparkContext) extends AnyVal { - - /** - * Read the Avro file in the specified path - * - * @param path - * @tparam T - * @return - */ - def avroFile[T: ClassTag](path: String): RDD[T] = { - sparkContext - .newAPIHadoopFile( - path, - classOf[AvroKeyInputFormat[T]], - classOf[AvroKey[T]], - classOf[NullWritable], - avroJob().getConfiguration) - .map[T](_._1.datum()) - } - } - - implicit class AvroRDD[T <: GenericRecord: ClassTag](val avroRDD: RDD[T]) { - def filterIfUnexpectedNull(fields: String*): RDD[T] = { - avroRDD.filter(r => fields.forall(isDefined(r, _))) - } - - /** Save as avro file */ - def saveAsAvroFile(outputPath: String): Unit = { - avroRDD - .map(r => (new AvroKey[T](r), NullWritable.get)) - .saveAsNewAPIHadoopFile( - outputPath, - classOf[AvroKey[T]], - classOf[NullWritable], - classOf[AvroKeyOutputFormat[T]], - avroJob[T]().getConfiguration) - } - - /** - * Save the specified avro files according to the Key value - * - * @param outputKeyFun - * outputKeyFunction - * @param outputPath - * outputPath - */ - def saveAsMultipleAvroFiles(outputKeyFun: (T) => String, outputPath: String): Unit = { - avroRDD - .map(r => ((outputKeyFun(r), new AvroKey(r)), NullWritable.get)) - .saveAsNewAPIHadoopFile( - outputPath, - classOf[AvroKey[(String, T)]], - classOf[NullWritable], - classOf[MultipleAvroOutputsFormat[T]], - avroJob[T]().getConfiguration) - } - } - -} diff --git a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/TextOutputFormat.scala b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/TextOutputFormat.scala deleted file mode 100644 index 6d8438800f..0000000000 --- a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/serializable/TextOutputFormat.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.core.serializable - -import org.apache.hadoop.mapred.lib.MultipleTextOutputFormat - -class TextOutputFormat extends MultipleTextOutputFormat[Any, Any] { - - override def generateFileNameForKeyValue(key: Any, value: Any, name: String): String = { - s"${key}_$name" - } - - override def generateActualKey(key: Any, value: Any): AnyRef = { - null - } -} diff --git a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/Spark.scala b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SparkSqlExecutor.scala similarity index 51% rename from streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/Spark.scala rename to streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SparkSqlExecutor.scala index 26aaeaad9f..4e207bb124 100644 --- a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/Spark.scala +++ b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SparkSqlExecutor.scala @@ -15,78 +15,73 @@ * limitations under the License. */ -package org.apache.streampark.spark.core +package org.apache.streampark.spark.core.util import org.apache.streampark.common.conf.ConfigKeys._ import org.apache.streampark.common.util.{DeflaterUtils, Logger, PropertiesUtils} -import org.apache.streampark.spark.core.util.{ParameterTool, SqlCommandParser} import org.apache.commons.lang3.StringUtils import org.apache.spark.SparkConf -import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.spark.sql.SparkSession +import org.apache.spark.streaming.{Seconds, StreamingContext} import java.util.concurrent.locks.ReentrantReadWriteLock -import scala.annotation.meta.getter import scala.collection.mutable.ArrayBuffer import scala.util.{Failure, Success, Try} -/** Spark

Spark Basic Traits

*/ -trait Spark extends Logger { +object SparkSqlExecutor extends Logger { - @(transient @getter) - final protected lazy val sparkConf: SparkConf = new SparkConf() + private val lock = new ReentrantReadWriteLock().writeLock - @(transient @getter) - final private[this] val sparkListeners = new ArrayBuffer[String]() - - @(transient @getter) - final protected var sparkSession: SparkSession = _ - - // Directory of checkpoint - final protected var checkpoint: String = "" - - // If recovery from checkpoint fails, recreate - final protected var createOnError: Boolean = true - - private[this] val lock = new ReentrantReadWriteLock().writeLock - - /** Entrance */ - def main(args: Array[String]): Unit = { - - init(args) + def runBatch(args: Array[String]): Unit = { + val sparkSession = createSparkSession(args) + try { + executeSql(args, sparkSession) + } finally { + sparkSession.sparkContext.stop() + } + } - config(sparkConf) + def runStreaming(args: Array[String]): Unit = { + val sparkSession = createSparkSession(args) + val sparkConf = sparkSession.sparkContext.getConf + val checkpoint = resolveCheckpoint(args) + val createOnError = resolveCreateOnError(args) + + val context = checkpoint match { + case "" => + new StreamingContext(sparkSession.sparkContext, Seconds(sparkConf.get(KEY_SPARK_BATCH_DURATION).toInt)) + case checkpointPath => + def createContext(): StreamingContext = + new StreamingContext(sparkSession.sparkContext, Seconds(sparkConf.get(KEY_SPARK_BATCH_DURATION).toInt)) + val tmpContext = + StreamingContext.getOrCreate(checkpointPath, createContext _, createOnError = createOnError) + tmpContext.checkpoint(checkpointPath) + tmpContext + } - // 1) system.properties - val sysProps = sparkConf.getAllWithPrefix("spark.config.system.properties") - if (sysProps != null) { - sysProps.foreach(x => { - System.getProperties.setProperty(x._1.drop(1), x._2) - }) + try { + executeSql(args, sparkSession) + context.start() + context.awaitTermination() + } finally { + context.stop(stopSparkContext = false, stopGracefully = true) } + } + private def createSparkSession(args: Array[String]): SparkSession = { + val sparkConf = initSparkConf(args) val builder = SparkSession.builder().config(sparkConf) - val enableHive = sparkConf.getBoolean("spark.config.enable.hive.support", defaultValue = false) - if (enableHive) { + if (sparkConf.getBoolean("spark.config.enable.hive.support", defaultValue = false)) { builder.enableHiveSupport() } + builder.getOrCreate() + } - sparkSession = builder.getOrCreate() - - // 2) hive - val sparkSql = sparkConf.getAllWithPrefix("spark.config.spark.sql") - if (sparkSql != null) { - sparkSql.foreach(x => { - sparkSession.sparkContext.getConf.set(x._1.drop(1), x._2) - }) - } - - ready() - + private def executeSql(args: Array[String], sparkSession: SparkSession): Unit = { val parameterTool = ParameterTool.fromArgs(args) - - val sparkSqls = { + val sparkSql = { val sql = parameterTool.get(KEY_SPARK_SQL()) require(StringUtils.isNotBlank(sql), "Usage: spark sql cannot be null") Try(DeflaterUtils.unzipString(sql)) match { @@ -96,32 +91,22 @@ trait Spark extends Logger { } } - SqlCommandParser - .parseSQL(sparkSqls) - .foreach(x => { - val args = if (x.operands.isEmpty) null else x.operands.head - val command = x.command.name - x.command match { - case _ => - try { - lock.lock() - val dataFrame: DataFrame = handle(x.originSql) - logInfo(s"$command:$args") - } finally { - if (lock.isHeldByCurrentThread) { - lock.unlock() - } - } + SqlCommandParser.parseSQL(sparkSql).foreach { command => + val operands = if (command.operands.isEmpty) null else command.operands.head + try { + lock.lock() + sparkSession.sql(command.originSql) + logInfo(s"${command.command.name}:$operands") + } finally { + if (lock.isHeldByCurrentThread) { + lock.unlock() } - }) - -// handle(sparkSqls) - start() - destroy() + } + } } - /** Initialize sparkConf according to user parameters */ - final private def init(args: Array[String]): Unit = { + private def initSparkConf(args: Array[String]): SparkConf = { + val sparkConf = new SparkConf() var argv = args.toList var conf: String = null val userArgs = ArrayBuffer[(String, String)]() @@ -131,11 +116,9 @@ trait Spark extends Logger { case "--conf" :: value :: tail => conf = value argv = tail - case "--checkpoint" :: value :: tail => - checkpoint = value + case "--checkpoint" :: _ :: tail => argv = tail - case "--createOnError" :: value :: tail => - createOnError = value.toBoolean + case "--createOnError" :: _ :: tail => argv = tail case Nil => case other :: value :: tail if other.startsWith(PARAM_PREFIX) => @@ -171,53 +154,51 @@ trait Spark extends Logger { case name => name } - // debug mode - val localMode = sparkConf.get("spark.master", null) == "local" - if (localMode) { + if (sparkConf.get("spark.master", null) == "local") { sparkConf.setAppName(s"[LocalDebug] $appName").setMaster("local[*]") sparkConf.set("spark.streaming.kafka.maxRatePerPartition", "10") } - // stop... sparkConf.set("spark.streaming.stopGracefullyOnShutdown", "true") + sparkConf + } + + private def resolveCheckpoint(args: Array[String]): String = { + var argv = args.toList + var checkpoint = "" + while (argv.nonEmpty) { + argv match { + case "--checkpoint" :: value :: tail => + checkpoint = value + argv = tail + case _ :: _ :: tail => + argv = tail + case _ :: Nil => + argv = Nil + case Nil => + } + } + checkpoint + } - val extraListeners = - sparkListeners.mkString(",") + "," + sparkConf.get("spark.extraListeners", "") - if (extraListeners != "") { - sparkConf.set("spark.extraListeners", extraListeners) + private def resolveCreateOnError(args: Array[String]): Boolean = { + var argv = args.toList + var createOnError = true + while (argv.nonEmpty) { + argv match { + case "--createOnError" :: value :: tail => + createOnError = value.toBoolean + argv = tail + case _ :: _ :: tail => + argv = tail + case _ :: Nil => + argv = Nil + case Nil => + } } + createOnError } - /** - * The purpose of the config phase is to allow the developer to set more parameters (other than - * the agreed configuration file) by means of hooks. Such as, conf.set("spark.serializer", - * "org.apache.spark.serializer.KryoSerializer") conf.registerKryoClasses(Array(classOf[User], - * classOf[Order],...)) - */ - def config(sparkConf: SparkConf): Unit = {} - - /** - * The ready phase is an entry point for the developer to do other actions after the parameters - * have been set, and is done after initialization and before the program starts. - */ - def ready(): Unit = {} - - /** - * The handle phase is the entry point to the code written by the developer and is the most - * important phase. - */ - def handle(sql: String = null): DataFrame = sparkSession.sql(sql) - - /** The start phase starts the task, which is executed automatically by the framework. */ - def start(): Unit = {} - - /** - * The destroy phase, is the last phase before jvm exits after the program has finished running, - * and is generally used to wrap up the work. - */ - def destroy(): Unit - - /** printUsageAndExit */ - private[this] def printUsageAndExit(): Unit = { + private def printUsageAndExit(): Unit = { logError( """ |"Usage: Streaming [options] diff --git a/streampark-spark/streampark-spark-sqlclient/src/main/scala/org/apache/streampark/spark/cli/SqlClient.scala b/streampark-spark/streampark-spark-sqlclient/src/main/scala/org/apache/streampark/spark/cli/SqlClient.scala index b8075c4b1a..8faee1cc07 100644 --- a/streampark-spark/streampark-spark-sqlclient/src/main/scala/org/apache/streampark/spark/cli/SqlClient.scala +++ b/streampark-spark/streampark-spark-sqlclient/src/main/scala/org/apache/streampark/spark/cli/SqlClient.scala @@ -17,51 +17,22 @@ package org.apache.streampark.spark.cli -import org.apache.streampark.common.conf.ConfigKeys.KEY_SPARK_SQL -import org.apache.streampark.common.util.DeflaterUtils -import org.apache.streampark.spark.core.{SparkBatch, SparkStreaming} -import org.apache.streampark.spark.core.util.ParameterTool - -import org.apache.commons.lang3.StringUtils -import org.apache.spark.sql.DataFrame +import org.apache.streampark.spark.core.util.SparkSqlExecutor import scala.collection.mutable.ArrayBuffer -import scala.util.{Failure, Success, Try} object SqlClient extends App { val arguments = ArrayBuffer(args: _*) - private[this] val parameterTool = ParameterTool.fromArgs(args) - - private[this] val sparkSql = { - val sql = parameterTool.get(KEY_SPARK_SQL()) - require(StringUtils.isNotBlank(sql), "Usage: spark sql cannot be null") - Try(DeflaterUtils.unzipString(sql)) match { - case Success(value) => value - case Failure(_) => - throw new IllegalArgumentException("Usage: spark sql is invalid or null, please check") - } - } - - private[this] val defaultMode = "BATCH" - - private[this] val mode = defaultMode + private[this] val mode = "BATCH" mode match { - case "STREAMING" | "AUTOMATIC" => StreamSqlApp.main(arguments.toArray) - case "BATCH" => BatchSqlApp.main(arguments.toArray) + case "STREAMING" | "AUTOMATIC" => SparkSqlExecutor.runStreaming(arguments.toArray) + case "BATCH" => SparkSqlExecutor.runBatch(arguments.toArray) case _ => throw new IllegalArgumentException( "Usage: runtime execution-mode invalid, optional [STREAMING|BATCH|AUTOMATIC]") } - private[this] object BatchSqlApp extends SparkBatch { - override def handle(sql: String): DataFrame = super.handle(sql) - } - - private[this] object StreamSqlApp extends SparkStreaming { - override def handle(sql: String): DataFrame = super.handle(sql) - } - }