Skip to content
This repository was archived by the owner on Dec 15, 2025. It is now read-only.

Commit e7e5675

Browse files
carsonwangcarsonwang
committed
Add streamingbench conf templates
1 parent 4c025b8 commit e7e5675

5 files changed

Lines changed: 116 additions & 10 deletions

File tree

‎conf/flink.conf.template‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
hibench.streambench.flink.home /PATH/TO/YOUR/FLINK/HOME
2+
3+
hibench.flink.master HOSTNAME:PORT
4+
5+
# Default parallelism of flink job
6+
hibench.streambench.flink.parallelism 20
7+
hibench.streambench.flink.bufferTimeout 10
8+
hibench.streambench.flink.checkpointDuration 1000

‎conf/gearpump.conf.template‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
hibench.streambench.gearpump.home /PATH/TO/YOUR/GEARPUMP/HOME
2+
3+
hibench.streambench.gearpump.parallelism 12
4+
5+
hibench.streambench.gearpump.executors 12

‎conf/hibench.conf‎

Lines changed: 52 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,11 @@
1-
21
hibench.scale.profile small
32

43
hibench.default.map.parallelism 12
54
hibench.default.shuffle.parallelism 12
65

7-
8-
96
#======================================================
107
# Report files
118
#======================================================
12-
139
# default report formats
1410
hibench.report.formats "%-12s %-10s %-8s %-20s %-20s %-20s %-20s\n"
1511

@@ -39,7 +35,6 @@ hibench.streambench.flinkbench.jar ${hibench.home}/flinkbench/streaming/tar
3935
#======================================================
4036
# workload home/input/ouput path
4137
#======================================================
42-
4338
hibench.hive.home ${hibench.home}/hadoopbench/sql/target/${hibench.hive.release}
4439
hibench.hive.release hive-0.12.0-bin
4540
hibench.hivebench.template.dir ${hibench.home}/hadoopbench/sql/hive_template
@@ -52,6 +47,10 @@ hibench.mahout.release.cdh5 mahout-0.9-cdh5.1.0
5247
hibench.mahout.release ${hibench.mahout.release.${hibench.hadoop.release}}
5348
hibench.mahout.home ${hibench.home}/hadoopbench/mahout/target/${hibench.mahout.release}
5449

50+
hibench.workload.input
51+
hibench.workload.output
52+
hibench.workload.dir.name.input Input
53+
hibench.workload.dir.name.output Output
5554

5655
hibench.nutch.dir.name.input ${hibench.workload.dir.name.input}
5756
hibench.nutch.dir.name.output ${hibench.workload.dir.name.output}
@@ -62,9 +61,53 @@ hibench.nutch.home ${hibench.home}/hadoopbench/nutchindexing/target/${hibench.n
6261
hibench.dfsioe.dir.name.input ${hibench.workload.dir.name.input}
6362
hibench.dfsioe.dir.name.output ${hibench.workload.dir.name.output}
6463

65-
hibench.workload.dir.name.input Input
66-
hibench.workload.dir.name.output Output
6764

65+
#======================================================
66+
# Streaming General
67+
#======================================================
68+
# Indicate whether in debug mode for correctness verfication (default: false)
69+
hibench.streambench.debugMode false
70+
hibench.streambench.sampleProbability 0.1
71+
hibench.streambench.fixWindowDuration 10000
72+
hibench.streambench.fixWindowSlideStep 10000
6873

69-
hibench.workload.input
70-
hibench.workload.output
74+
75+
#======================================================
76+
# Kafka for streaming benchmarks
77+
#======================================================
78+
hibench.streambench.kafka.home /PATH/TO/YOUR/KAFKA/HOME
79+
# zookeeper host:port of kafka cluster, host1:port1,host2:port2...
80+
hibench.streambench.zkHost
81+
# Kafka broker lists, written in mode host:port,host:port,..
82+
hibench.streambench.kafka.brokerList
83+
hibench.streambench.kafka.consumerGroup HiBench
84+
# number of partitions of generated topic (default 20)
85+
hibench.streambench.kafka.topicPartitions 20
86+
# consumer group of the consumer for kafka (default: HiBench)
87+
hibench.streambench.kafka.consumerGroup HiBench
88+
# Set the starting offset of kafkaConsumer (default: largest)
89+
hibench.streambench.kafka.offsetReset largest
90+
91+
92+
#======================================================
93+
# Data generator for streaming benchmarks
94+
#======================================================
95+
# Interval span in millisecond (default: 50)
96+
hibench.streambench.datagen.intervalSpan 50
97+
# Number of records to generate per interval span (default: 5)
98+
hibench.streambench.datagen.recordsPerInterval 5
99+
# fixed length of record (default: 200)
100+
hibench.streambench.datagen.recordLength 200
101+
# Number of KafkaProducer running on different thread (default: 1)
102+
hibench.streambench.datagen.producerNumber 1
103+
# Total round count of data send (default: -1 means infinity)
104+
hibench.streambench.datagen.totalRounds -1
105+
# Number of total records that will be generated (default: -1 means infinity)
106+
hibench.streambench.datagen.totalRecords -1
107+
# default path to store seed files (default: ${hibench.hdfs.data.dir}/Streaming)
108+
hibench.streambench.datagen.dir ${hibench.hdfs.data.dir}/Streaming
109+
# default path setting for genearate data1 & data2
110+
hibench.streambench.datagen.data1.name Seed
111+
hibench.streambench.datagen.data1.dir ${hibench.streambench.datagen.dir}/${hibench.streambench.datagen.data1.name}
112+
hibench.streambench.datagen.data2_cluster.dir ${hibench.streambench.datagen.dir}/Kmeans/Cluster
113+
hibench.streambench.datagen.data2_samples.dir ${hibench.streambench.datagen.dir}/Kmeans/Samples

‎conf/spark.conf.template‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Spark home
2-
hibench.spark.home /PATH/TO/YOUR/SPARK/ROOT
2+
hibench.spark.home /PATH/TO/YOUR/SPARK/HOME
33

44
# Spark version. Supported value: spark1.6, spark2.0
55
hibench.spark.version spark1.6
@@ -23,3 +23,28 @@ spark.default.parallelism ${hibench.default.map.parallelism}
2323

2424
# set spark sql's default shuffle partitions according to hibench's parallelism value
2525
spark.sql.shuffle.partitions ${hibench.default.map.parallelism}
26+
27+
28+
#======================================================
29+
# Spark Streaming
30+
#======================================================
31+
# Spark streaming Batchnterval in millisecond (default 100)
32+
hibench.streambench.spark.batchInterval 100
33+
34+
# Number of nodes that will receive kafka input (default: 4)
35+
hibench.streambench.spark.receiverNumber 4
36+
37+
# Indicate RDD storage level. (default: 2)
38+
# 0 = StorageLevel.MEMORY_ONLY
39+
# 1 = StorageLevel.MEMORY_AND_DISK_SER
40+
# other = StorageLevel.MEMORY_AND_DISK_SER_2
41+
hibench.streambench.spark.storageLevel 2
42+
43+
# indicate whether to test the write ahead log new feature (default: false)
44+
hibench.streambench.spark.enableWAL false
45+
46+
# if testWAL is true, this path to store stream context in hdfs shall be specified. If false, it can be empty (default: /var/tmp)
47+
hibench.streambench.spark.checkpointPath /var/tmp
48+
49+
# whether to use direct approach or not (dafault: true)
50+
hibench.streambench.spark.useDirectMode true

‎conf/storm.conf.template‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# nimbus of storm cluster
2+
hibench.streambench.storm.nimbus HOSTNAME_OF_STORM_NIMBUS
3+
hibench.streambench.storm.nimbusAPIPort 6627
4+
5+
hibench.streambench.storm.home /PATH/TO/YOUR/STORM/HOME
6+
7+
# number of workers of Storm. Number of most bolt threads is also equal to this param.
8+
hibench.streambench.storm.worker_count 12
9+
10+
# number of kafka spout threads of Storm
11+
hibench.streambench.storm.spout_threads 12
12+
13+
# number of bolt threads altogether
14+
hibench.streambench.storm.bolt_threads 12
15+
16+
hibench.streambench.storm.localshuffle true
17+
18+
# time interval to contact nimbus to judge if finished
19+
hibench.streambench.storm.nimbusContactInterval 10
20+
21+
# kafka arg indicating whether to read data from kafka from the start or go on to read from last position
22+
hibench.streambench.storm.read_from_start true
23+
24+
# whether to turn on ack
25+
hibench.streambench.storm.ackon true

0 commit comments

Comments
 (0)