Merge branch 'master5.0' into mergeStreamingWithMaster

Author: carsonwang

.gitignore

@@ -11,3 +11,6 @@ target/
 derby.log
 metastore_db/
 report/
+.classpath
+.project
+.settings/

.travis.yml

@@ -0,0 +1,14 @@
+language: java
+jdk:
+  - openjdk7
+  - oraclejdk8
+install: 
+  - cd src
+before_script:
+  - "echo $JAVA_OPTS"
+  - "export JAVA_OPTS=-Xmx512m"
+env:
+  - SPARK_VERSION=1.5
+  - SPARK_VERSION=1.6
+script:
+  - mvn clean package -q -Dmaven.javadoc.skip=true -Dspark${SPARK_VERSION} -DMR2

README.md

@@ -1,4 +1,4 @@
-# HiBench Suite #
+# HiBench Suite [![Build Status](https://travis-ci.org/intel-hadoop/HiBench.svg?branch=master)](https://travis-ci.org/intel-hadoop/HiBench)
 ## The bigdata micro benchmark suite ##
 
 
@@ -121,4 +121,3 @@ Note:
 
 ### [Advanced Configurations](https://github.com/intel-hadoop/HiBench/wiki/Advanced-Configurations) ###
 ### [Possible issues](https://github.com/intel-hadoop/HiBench/wiki/Possible-issues) ###
-

bin/functions/hibench_prop_env_mapping.py

@@ -112,6 +112,15 @@
     WT_FILE_SIZE="hibench.dfsioe.write.file_size",
     MAP_JAVA_OPTS="hibench.dfsioe.map.java_opts",
     RED_JAVA_OPTS="hibench.dfsioe.red.java_opts",
+    # For NWeight
+    MODEL_INPUT="hibench.nweight.model_path",
+    EDGES="hibench.workload.edges",
+    DEGREE="hibench.nweight.degree",
+    MAX_OUT_EDGES="hibench.nweight.max_out_edges",
+    NUM_PARTITION="hibench.nweight.partitions",
+    STORAGE_LEVEL="hibench.nweight.storage_level",
+    DISABLE_KRYO="hibench.nweight.disable_kryo",
+    MODEL="hibench.nweight.model",
 
     # For streaming bench
     STREAMING_TESTCASE="hibench.streambench.testCase",

bin/functions/workload-functions.sh

@@ -408,8 +408,9 @@ function ensure-nutchindexing-release () {
         mkdir $NUTCH_HOME_WORKLOAD/temp
         unzip -q $NUTCH_HOME_WORKLOAD/nutch-1.2.job -d $NUTCH_HOME_WORKLOAD/temp
         rm -f $NUTCH_HOME_WORKLOAD/temp/lib/jcl-over-slf4j-*.jar
+        rm -f $NUTCH_HOME_WORKLOAD/temp/lib/slf4j-log4j*.jar
         cp ${NUTCH_DIR}/target/dependency/jcl-over-slf4j-*.jar $NUTCH_HOME_WORKLOAD/temp/lib
-        rm -f $NUTCH_ROOT/nutch-1.2.job
+        rm -f $NUTCH_HOME_WORKLOAD/nutch-1.2.job
         cd $NUTCH_HOME_WORKLOAD/temp
         zip -qr $NUTCH_HOME_WORKLOAD/nutch-1.2.job *
         rm -rf $NUTCH_HOME_WORKLOAD/temp
@@ -431,8 +432,9 @@ set ${MAP_CONFIG_NAME}=$NUM_MAPS;
 set ${REDUCER_CONFIG_NAME}=$NUM_REDS;
 set hive.stats.autogather=false;
 ${HIVE_SQL_COMPRESS_OPTS}
-
+DROP TABLE IF EXISTS uservisits;
 CREATE EXTERNAL TABLE uservisits (sourceIP STRING,destURL STRING,visitDate STRING,adRevenue DOUBLE,userAgent STRING,countryCode STRING,languageCode STRING,searchWord STRING,duration INT ) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS SEQUENCEFILE LOCATION '$INPUT_HDFS/uservisits';
+DROP TABLE IF EXISTS uservisits_aggre;
 CREATE EXTERNAL TABLE uservisits_aggre ( sourceIP STRING, sumAdRevenue DOUBLE) STORED AS SEQUENCEFILE LOCATION '$OUTPUT_HDFS/uservisits_aggre';
 INSERT OVERWRITE TABLE uservisits_aggre SELECT sourceIP, SUM(adRevenue) FROM uservisits GROUP BY sourceIP;
 EOF
@@ -453,8 +455,11 @@ set hive.stats.autogather=false;
 
 ${HIVE_SQL_COMPRESS_OPTS}
 
+DROP TABLE IF EXISTS rankings;
 CREATE EXTERNAL TABLE rankings (pageURL STRING, pageRank INT, avgDuration INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS SEQUENCEFILE LOCATION '$INPUT_HDFS/rankings';
+DROP TABLE IF EXISTS uservisits_copy;
 CREATE EXTERNAL TABLE uservisits_copy (sourceIP STRING,destURL STRING,visitDate STRING,adRevenue DOUBLE,userAgent STRING,countryCode STRING,languageCode STRING,searchWord STRING,duration INT ) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS SEQUENCEFILE LOCATION '$INPUT_HDFS/uservisits';
+DROP TABLE IF EXISTS rankings_uservisits_join;
 CREATE EXTERNAL TABLE rankings_uservisits_join ( sourceIP STRING, avgPageRank DOUBLE, totalRevenue DOUBLE) STORED AS SEQUENCEFILE LOCATION '$OUTPUT_HDFS/rankings_uservisits_join';
 INSERT OVERWRITE TABLE rankings_uservisits_join SELECT sourceIP, avg(pageRank), sum(adRevenue) as totalRevenue FROM rankings R JOIN (SELECT sourceIP, destURL, adRevenue FROM uservisits_copy UV WHERE (datediff(UV.visitDate, '1999-01-01')>=0 AND datediff(UV.visitDate, '2000-01-01')<=0)) NUV ON (R.pageURL = NUV.destURL) group by sourceIP order by totalRevenue DESC;
 EOF
@@ -475,7 +480,9 @@ set hive.stats.autogather=false;
 
 ${HIVE_SQL_COMPRESS_OPTS}
 
+DROP TABLE IF EXISTS uservisits;
 CREATE EXTERNAL TABLE uservisits (sourceIP STRING,destURL STRING,visitDate STRING,adRevenue DOUBLE,userAgent STRING,countryCode STRING,languageCode STRING,searchWord STRING,duration INT ) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS SEQUENCEFILE LOCATION '$INPUT_HDFS/uservisits';
+DROP TABLE IF EXISTS uservisits_copy;
 CREATE EXTERNAL TABLE uservisits_copy (sourceIP STRING,destURL STRING,visitDate STRING,adRevenue DOUBLE,userAgent STRING,countryCode STRING,languageCode STRING,searchWord STRING,duration INT ) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS SEQUENCEFILE LOCATION '$OUTPUT_HDFS/uservisits_copy';
 INSERT OVERWRITE TABLE uservisits_copy SELECT * FROM uservisits;
 EOF

conf/00-default-properties.conf

@@ -150,6 +150,8 @@ hibench.randomtextwriter.bytestotal.hadoop1.name   test.randomtextwrite.total_by
 hibench.randomtextwriter.bytestotal.hadoop2.name   mapreduce.randomtextwriter.totalbytes
 hibench.randomtextwriter.bytestotal.name	   ${hibench.randomtextwriter.bytestotal.${hibench.hadoop.version}.name}
 
+hibench.nweight.model_path	${hibench.dependency.dir}/sparkbench/src/main/scala/com/intel/sparkbench/nweight/model/user-features
+
 # Workload Input/Output name setting for compress/uncompress mode
 hibench.workload.dir.name.compress_disable.input	Input
 hibench.workload.dir.name.compress_disable.output	Output
@@ -260,4 +262,4 @@ spark.sql.shuffle.partitions  ${hibench.default.map.parallelism}
 #=======================================================
 # Flink
 #=======================================================
-hibench.flink.master FLINK_JM_HOST:PORT
+hibench.flink.master FLINK_JM_HOST:PORT

conf/10-data-scale-profile.conf

@@ -212,4 +212,24 @@ hibench.dfsioe.gigantic.write.file_size		400
 hibench.dfsioe.bigdata.read.number_of_files	2048
 hibench.dfsioe.bigdata.read.file_size		1000
 hibench.dfsioe.bigdata.write.number_of_files	2048
-hibench.dfsioe.bigdata.write.file_size		1000
+hibench.dfsioe.bigdata.write.file_size		1000
+
+#NWeight
+hibench.nweight.tiny.edges			100000
+hibench.nweight.tiny.degree                     3
+hibench.nweight.tiny.max_out_edges              30
+hibench.nweight.small.edges                     1000000
+hibench.nweight.small.degree                    3
+hibench.nweight.small.max_out_edges             30
+hibench.nweight.large.edges                     10000000
+hibench.nweight.large.degree                    3
+hibench.nweight.large.max_out_edges             30
+hibench.nweight.huge.edges                      100000000
+hibench.nweight.huge.degree                     3
+hibench.nweight.huge.max_out_edges              30
+hibench.nweight.gigantic.edges                  425000000
+hibench.nweight.gigantic.degree                 3
+hibench.nweight.gigantic.max_out_edges          30
+hibench.nweight.bigdata.edges                   4250000000
+hibench.nweight.bigdata.degree                  3
+hibench.nweight.bigdata.max_out_edges           30

docker/base/base-core

@@ -2,7 +2,6 @@
 USER root
 
 ## add proxy config inside FIREWALL
-
 #==============================
 # System Basic Tools  Installation
 #==============================
@@ -54,15 +53,12 @@ RUN apt-get install -y python-numpy python-matplotlib
 
 # Install Java
 RUN \
-  echo oracle-java${JDK_VERSION}-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \
-  add-apt-repository -y ppa:webupd8team/java && \
+  add-apt-repository -y ppa:openjdk-r/ppa && \
   apt-get update && \
-  apt-get install -y oracle-java${JDK_VERSION}-installer && \
-  rm -rf /var/lib/apt/lists/* && \
-  rm -rf /var/cache/oracle-jdk${JDK_VERSION}-installer
+  apt-get install -y openjdk-${JDK_VERSION}-jdk
 
 # Define commonly used JAVA_HOME variable
-ENV JAVA_HOME /usr/lib/jvm/java-${JDK_VERSION}-oracle
+ENV JAVA_HOME /usr/lib/jvm/java-${JDK_VERSION}-openjdk-amd64
 ENV PATH $PATH:$JAVA_HOME/bin
 
 
@@ -102,6 +98,9 @@ RUN mv apache-maven-* /usr/local/apache-maven
 # define environment variables for maven
 ENV M2_HOME /usr/local/apache-maven
 ENV PATH $PATH:/usr/local/apache-maven/bin
+# copy local maven repository to docker image
+#RUN rm -rf /root/.m2
+#ADD .m2 /root/.m2
 
 
 #==============================
@@ -119,5 +118,9 @@ RUN mv /root/HiBench* ${HIBENCH_HOME}
 RUN rm -f HiBench-${HIBENCH_VERSION}.zip
 COPY conf/99-user_defined_properties.conf ${HIBENCH_HOME}/conf/
 # start building HiBench
+RUN apt-get update && apt-get install -y thrift-compiler
+#RUN cd ${HIBENCH_HOME}/src && \
+#mvn clean package -D spark1.5 -D MR2
+# /bin/build-all.sh can be used to built hibench for all known Spark and MR versions
 RUN ${HIBENCH_HOME}/bin/build-all.sh
 

docker/cdh-docker/Dockerfile

@@ -60,6 +60,6 @@ COPY scripts/hadoop-env.sh /etc/hadoop/conf/hadoop-env.sh
 #Format HDFS
 COPY scripts/restart-hadoop-spark.sh /usr/bin/restart-hadoop-spark.sh
 RUN chmod +x /usr/bin/restart-hadoop-spark.sh
-
-# start HADOOP/SPARK
-CMD bash -C '/usr/bin/restart-hadoop-spark.sh'; 'bash'
+#Copy RunExample File
+COPY scripts/runexample.sh /root/runexample.sh
+RUN chmod +x /root/runexample.sh

docker/cdh-docker/conf/core-site.xml

@@ -1,3 +1,4 @@
+<!--
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,11 +13,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+-->
 
 
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
 <configuration>
 
     <!--