Skip to content
This repository was archived by the owner on Dec 15, 2025. It is now read-only.

Commit 612f012

Browse files
carsonwangcarsonwang
committed
add scripts for other streaming workloads
1 parent f36ea0b commit 612f012

21 files changed

Lines changed: 786 additions & 0 deletions

File tree

‎bin/workloads/streaming/fixwindow/flink/run.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
current_dir=`dirname "$0"`
18+
current_dir=`cd "$current_dir"; pwd`
19+
root_dir=${current_dir}/../../../../..
20+
workload_config=${root_dir}/conf/workloads/streaming/fixwindow.conf
21+
. "${root_dir}/bin/functions/load-bench-config.sh"
22+
23+
enter_bench FlinkFixWindow ${workload_config} ${current_dir}
24+
show_bannar start
25+
26+
START_TIME=`timestamp`
27+
printFullLog
28+
run-flink-job -c com.intel.hibench.flinkbench.RunBench
29+
END_TIME=`timestamp`
30+
31+
gen_report ${START_TIME} ${END_TIME} 0
32+
show_bannar finish
33+
leave_bench

‎bin/workloads/streaming/fixwindow/gearpump/run.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
current_dir=`dirname "$0"`
18+
current_dir=`cd "$current_dir"; pwd`
19+
root_dir=${current_dir}/../../../../..
20+
workload_config=${root_dir}/conf/workloads/streaming/fixwindow.conf
21+
. "${root_dir}/bin/functions/load-bench-config.sh"
22+
23+
enter_bench GearpumpFixWindow ${workload_config} ${current_dir}
24+
show_bannar start
25+
26+
START_TIME=`timestamp`
27+
printFullLog
28+
run-gearpump-app com.intel.hibench.gearpumpbench.RunBench ${SPARKBENCH_PROPERTIES_FILES}
29+
END_TIME=`timestamp`
30+
31+
gen_report ${START_TIME} ${END_TIME} 0
32+
show_bannar finish
33+
leave_bench

‎bin/workloads/streaming/fixwindow/prepare/dataGen.sh‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
current_dir=`dirname "$0"`
18+
current_dir=`cd "$current_dir"; pwd`
19+
root_dir=${current_dir}/../../../../..
20+
workload_config=${root_dir}/conf/workloads/streaming/fixwindow.conf
21+
. "${root_dir}/bin/functions/load-bench-config.sh"
22+
23+
enter_bench StreamingFixWindowPrepare ${workload_config} ${current_dir}
24+
show_bannar start
25+
26+
DATA_FILE1=${STREAMING_DATA1_DIR}/uservisits
27+
DATA_FILE2=${STREAMING_DATA2_SAMPLE_DIR}
28+
29+
JVM_OPTS="-Xmx1024M -server -XX:+UseCompressedOops -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled -XX:+CMSScavengeBeforeRemark -XX:+DisableExplicitGC -Djava.awt.headless=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dkafka.logs.dir=bin/../logs -cp ${DATATOOLS}"
30+
31+
printFullLog
32+
33+
CMD="$JAVA_BIN $JVM_OPTS com.intel.hibench.datagen.streaming.DataGenerator $SPARKBENCH_PROPERTIES_FILES $DATA_FILE1 0 $DATA_FILE2 0"
34+
echo -e "${BGreen}Sending streaming data to kafka, periodically: ${Green}$CMD${Color_Off}"
35+
execute_withlog $CMD
36+
37+
show_bannar finish

‎bin/workloads/streaming/fixwindow/prepare/genSeedDataset.sh‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
current_dir=`dirname "$0"`
18+
current_dir=`cd "$current_dir"; pwd`
19+
root_dir=${current_dir}/../../../../..
20+
workload_config=${root_dir}/conf/workloads/streaming/fixwindow.conf
21+
. "${root_dir}/bin/functions/load-bench-config.sh"
22+
23+
24+
# generate seed data1 by hive
25+
enter_bench HadoopPrepareDatafile1 ${workload_config} ${current_dir}
26+
show_bannar start
27+
28+
PAGES=120000
29+
USERVISITS=1000000
30+
31+
rmr-hdfs $STREAMING_DATA_DIR || true
32+
echo -e "${On_Blue}Pages:${PAGES}, USERVISITS:${USERVISITS}${Color_Off}"
33+
34+
OPTION="-t hive \
35+
-b ${STREAMING_DATA_DIR} \
36+
-n ${STREAMING_DATA1_NAME} \
37+
-m ${NUM_MAPS} \
38+
-r ${NUM_REDS} \
39+
-p ${PAGES} \
40+
-v ${USERVISITS}"
41+
42+
START_TIME=`timestamp`
43+
run-hadoop-job ${DATATOOLS} HiBench.DataGen ${OPTION}
44+
END_TIME=`timestamp`
45+
SIZE="0"
46+
47+
show_bannar finish
48+
leave_bench
49+
50+
# generate seed data2 by kmeans
51+
enter_bench HadoopPrepareDatafile2 ${workload_config} ${current_dir}
52+
show_bannar start
53+
54+
rmr-hdfs $STREAMING_DATA2_SAMPLE_DIR || true
55+
OPTION="-sampleDir ${STREAMING_DATA2_SAMPLE_DIR} -clusterDir ${STREAMING_DATA2_CLUSTER_DIR} -numClusters 5 -numSamples 3000000 -samplesPerFile 600000 -sampleDimension 20 -textOutput"
56+
run-hadoop-job ${DATATOOLS} org.apache.mahout.clustering.kmeans.GenKMeansDataset -D hadoop.job.history.user.location=${STREAMING_DATA2_SAMPLE_DIR} ${OPTION}
57+
END_TIME=`timestamp`
58+
59+
show_bannar finish
60+
leave_bench

‎bin/workloads/streaming/fixwindow/spark/run.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
current_dir=`dirname "$0"`
18+
current_dir=`cd "$current_dir"; pwd`
19+
root_dir=${current_dir}/../../../../..
20+
workload_config=${root_dir}/conf/workloads/streaming/fixwindow.conf
21+
. "${root_dir}/bin/functions/load-bench-config.sh"
22+
23+
enter_bench SparkStreamingFixWindow ${workload_config} ${current_dir}
24+
show_bannar start
25+
26+
START_TIME=`timestamp`
27+
printFullLog
28+
run-spark-job com.intel.hibench.sparkbench.streaming.RunBench $SPARKBENCH_PROPERTIES_FILES
29+
END_TIME=`timestamp`
30+
31+
gen_report ${START_TIME} ${END_TIME} 0
32+
show_bannar finish
33+
leave_bench

‎bin/workloads/streaming/fixwindow/storm/run.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
current_dir=`dirname "$0"`
18+
current_dir=`cd "$current_dir"; pwd`
19+
root_dir=${current_dir}/../../../../..
20+
workload_config=${root_dir}/conf/workloads/streaming/fixwindow.conf
21+
. "${root_dir}/bin/functions/load-bench-config.sh"
22+
23+
enter_bench StormFixWindow ${workload_config} ${current_dir}
24+
show_bannar start
25+
26+
START_TIME=`timestamp`
27+
printFullLog
28+
run-storm-job com.intel.hibench.stormbench.RunBench ${SPARKBENCH_PROPERTIES_FILES} storm
29+
END_TIME=`timestamp`
30+
31+
gen_report ${START_TIME} ${END_TIME} 0
32+
show_bannar finish
33+
leave_bench

‎bin/workloads/streaming/fixwindow/trident/run.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
current_dir=`dirname "$0"`
18+
current_dir=`cd "$current_dir"; pwd`
19+
root_dir=${current_dir}/../../../../..
20+
workload_config=${root_dir}/conf/workloads/streaming/fixwindow.conf
21+
. "${root_dir}/bin/functions/load-bench-config.sh"
22+
23+
enter_bench StormTridentFixWindow ${workload_config} ${current_dir}
24+
show_bannar start
25+
26+
START_TIME=`timestamp`
27+
printFullLog
28+
run-storm-job com.intel.hibench.stormbench.RunBench ${SPARKBENCH_PROPERTIES_FILES} trident
29+
END_TIME=`timestamp`
30+
31+
gen_report ${START_TIME} ${END_TIME} 0
32+
show_bannar finish
33+
leave_bench

‎bin/workloads/streaming/identity/flink/run.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
current_dir=`dirname "$0"`
18+
current_dir=`cd "$current_dir"; pwd`
19+
root_dir=${current_dir}/../../../../..
20+
workload_config=${root_dir}/conf/workloads/streaming/identity.conf
21+
. "${root_dir}/bin/functions/load-bench-config.sh"
22+
23+
enter_bench FlinkIdentity ${workload_config} ${current_dir}
24+
show_bannar start
25+
26+
START_TIME=`timestamp`
27+
printFullLog
28+
run-flink-job -c com.intel.hibench.flinkbench.RunBench
29+
END_TIME=`timestamp`
30+
31+
gen_report ${START_TIME} ${END_TIME} 0
32+
show_bannar finish
33+
leave_bench

‎bin/workloads/streaming/identity/gearpump/run.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
current_dir=`dirname "$0"`
18+
current_dir=`cd "$current_dir"; pwd`
19+
root_dir=${current_dir}/../../../../..
20+
workload_config=${root_dir}/conf/workloads/streaming/identity.conf
21+
. "${root_dir}/bin/functions/load-bench-config.sh"
22+
23+
enter_bench GearpumpIdentity ${workload_config} ${current_dir}
24+
show_bannar start
25+
26+
START_TIME=`timestamp`
27+
printFullLog
28+
run-gearpump-app com.intel.hibench.gearpumpbench.RunBench ${SPARKBENCH_PROPERTIES_FILES}
29+
END_TIME=`timestamp`
30+
31+
gen_report ${START_TIME} ${END_TIME} 0
32+
show_bannar finish
33+
leave_bench

‎bin/workloads/streaming/identity/prepare/dataGen.sh‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
current_dir=`dirname "$0"`
18+
current_dir=`cd "$current_dir"; pwd`
19+
root_dir=${current_dir}/../../../../..
20+
workload_config=${root_dir}/conf/workloads/streaming/identity.conf
21+
. "${root_dir}/bin/functions/load-bench-config.sh"
22+
23+
enter_bench StreamingIdentityPrepare ${workload_config} ${current_dir}
24+
show_bannar start
25+
26+
DATA_FILE1=${STREAMING_DATA1_DIR}/uservisits
27+
DATA_FILE2=${STREAMING_DATA2_SAMPLE_DIR}
28+
29+
JVM_OPTS="-Xmx1024M -server -XX:+UseCompressedOops -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled -XX:+CMSScavengeBeforeRemark -XX:+DisableExplicitGC -Djava.awt.headless=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dkafka.logs.dir=bin/../logs -cp ${DATATOOLS}"
30+
31+
printFullLog
32+
33+
CMD="$JAVA_BIN $JVM_OPTS com.intel.hibench.datagen.streaming.DataGenerator $SPARKBENCH_PROPERTIES_FILES $DATA_FILE1 0 $DATA_FILE2 0"
34+
echo -e "${BGreen}Sending streaming data to kafka, periodically: ${Green}$CMD${Color_Off}"
35+
execute_withlog $CMD
36+
37+
show_bannar finish

0 commit comments

Comments
 (0)