Modify docker module errors and deploy docker image on openjdk (#288)

zybing · carsonwang · commit 6b3d95220df8 · 2016-09-06T14:51:57.000+08:00
* docker on openjdk

* hibench docker on openjdk

* modify docker script

* add runexample.sh for opensource

* change directory error

* change spark version

* build all

* set opensource docker and cdh docker different name

* modify run-container

* support jdk 8

* remove bin/bash for some file
diff --git a/docker/base/base-core b/docker/base/base-core
@@ -2,7 +2,6 @@
 USER root
 
 ## add proxy config inside FIREWALL
-
 #==============================
 # System Basic Tools  Installation
 #==============================
@@ -54,15 +53,12 @@ RUN apt-get install -y python-numpy python-matplotlib
 
 # Install Java
 RUN \
-  echo oracle-java${JDK_VERSION}-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \
-  add-apt-repository -y ppa:webupd8team/java && \
+  add-apt-repository -y ppa:openjdk-r/ppa && \
   apt-get update && \
-  apt-get install -y oracle-java${JDK_VERSION}-installer && \
-  rm -rf /var/lib/apt/lists/* && \
-  rm -rf /var/cache/oracle-jdk${JDK_VERSION}-installer
+  apt-get install -y openjdk-${JDK_VERSION}-jdk
 
 # Define commonly used JAVA_HOME variable
-ENV JAVA_HOME /usr/lib/jvm/java-${JDK_VERSION}-oracle
+ENV JAVA_HOME /usr/lib/jvm/java-${JDK_VERSION}-openjdk-amd64
 ENV PATH $PATH:$JAVA_HOME/bin
 
 
@@ -102,6 +98,9 @@ RUN mv apache-maven-* /usr/local/apache-maven
 # define environment variables for maven
 ENV M2_HOME /usr/local/apache-maven
 ENV PATH $PATH:/usr/local/apache-maven/bin
+# copy local maven repository to docker image
+#RUN rm -rf /root/.m2
+#ADD .m2 /root/.m2
 
 
 #==============================
@@ -119,5 +118,9 @@ RUN mv /root/HiBench* ${HIBENCH_HOME}
 RUN rm -f HiBench-${HIBENCH_VERSION}.zip
 COPY conf/99-user_defined_properties.conf ${HIBENCH_HOME}/conf/
 # start building HiBench
+RUN apt-get update && apt-get install -y thrift-compiler
+#RUN cd ${HIBENCH_HOME}/src && \
+#mvn clean package -D spark1.5 -D MR2
+# /bin/build-all.sh can be used to built hibench for all known Spark and MR versions
 RUN ${HIBENCH_HOME}/bin/build-all.sh
 
diff --git a/docker/cdh-docker/Dockerfile b/docker/cdh-docker/Dockerfile
@@ -60,6 +60,6 @@ COPY scripts/hadoop-env.sh /etc/hadoop/conf/hadoop-env.sh
 #Format HDFS
 COPY scripts/restart-hadoop-spark.sh /usr/bin/restart-hadoop-spark.sh
 RUN chmod +x /usr/bin/restart-hadoop-spark.sh
-
-# start HADOOP/SPARK
-CMD bash -C '/usr/bin/restart-hadoop-spark.sh'; 'bash'
+#Copy RunExample File
+COPY scripts/runexample.sh /root/runexample.sh
+RUN chmod +x /root/runexample.sh
diff --git a/docker/cdh-docker/conf/core-site.xml b/docker/cdh-docker/conf/core-site.xml
@@ -1,3 +1,4 @@
+<!--
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,11 +13,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+-->
 
 
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
 <configuration>
 
     <!--
diff --git a/docker/cdh-docker/conf/hdfs-site.xml b/docker/cdh-docker/conf/hdfs-site.xml
@@ -1,3 +1,4 @@
+<!--
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,11 +13,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+-->
 
 
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
 <configuration>
 
   <property>
diff --git a/docker/cdh-docker/conf/mapred-site.xml b/docker/cdh-docker/conf/mapred-site.xml
@@ -1,3 +1,4 @@
+<!--
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,11 +13,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+-->
 
 
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
 <configuration>
 
   <property>
diff --git a/docker/cdh-docker/conf/yarn-site.xml b/docker/cdh-docker/conf/yarn-site.xml
@@ -1,3 +1,4 @@
+<!--
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,11 +13,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+-->
 
 
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
 <configuration>
   	
   <property>
diff --git a/docker/cdh-docker/scripts/hadoop-env.sh b/docker/cdh-docker/scripts/hadoop-env.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -14,10 +15,9 @@
 # limitations under the License.
 
 
-#!/bin/bash
 
 export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true $HADOOP_CLIENT_OPTS"
-export JAVA_HOME="/usr/lib/jvm/java-7-oracle"
+export JAVA_HOME="/usr/lib/jvm/java-7-openjdk-amd64"
 
 export HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_PREFIX}/lib/native
 export HADOOP_OPTS="-Djava.library.path=$HADOOP_PREFIX/lib"
diff --git a/docker/cdh-docker/scripts/restart-hadoop-spark.sh b/docker/cdh-docker/scripts/restart-hadoop-spark.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -14,7 +15,6 @@
 # limitations under the License.
 
 
-#!/bin/bash
 
 # start ssh service
 service ssh restart
diff --git a/docker/cdh-docker/scripts/runexample.sh b/docker/cdh-docker/scripts/runexample.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+#modify etc/hosts file to avoid connection error
+cp /etc/hosts /etc/hostsbak
+sed -i 's/::1/#::1/' /etc/hostsbak
+cat /etc/hostsbak > /etc/hosts
+rm -rf /etc/hostsbak
+#run wordcount example
+/usr/bin/restart-hadoop-spark.sh
+${HIBENCH_HOME}/workloads/wordcount/prepare/prepare.sh
+${HIBENCH_HOME}/workloads/wordcount/mapreduce/bin/run.sh
diff --git a/docker/hibench-docker.conf b/docker/hibench-docker.conf
@@ -31,7 +31,7 @@ OS_VERSION                       14.04
 PYTHON_VERSION                   2.7
 JDK_VERSION                      7
 SCALA_VERSION                    2.10.5
-MAVEN_VERSION                    3.1.0
+MAVEN_VERSION                    3.0.5
 HIBENCH_VERSION                  5.0
 
 
@@ -40,8 +40,8 @@ HIBENCH_VERSION                  5.0
 #=========================#
 HADOOP_VERSION                   hadoop2
 HADOOP_VERSION_DETAIL            2.5.0
-SPARK_VERSION                    spark1.3
-SPARK_VERSION_DETAIL             1.3.0
+SPARK_VERSION                    spark1.5
+SPARK_VERSION_DETAIL             1.5.2
 HADOOP_FOR_SPARK_VERSION         2.4
 
 
diff --git a/docker/opensource-docker/Dockerfile b/docker/opensource-docker/Dockerfile
@@ -64,8 +64,8 @@ COPY conf/hdfs-site.xml ${HADOOP_CONF_DIR}/
 COPY conf/mapred-site.xml ${HADOOP_CONF_DIR}/
 COPY conf/yarn-site.xml ${HADOOP_CONF_DIR}/
 COPY scripts/hadoop-env.sh ${HADOOP_CONF_DIR}/
-COPY scripts/restart-hadoop-spark.sh /root/
-RUN chmod +x /root/restart-hadoop-spark.sh
-
-# start HADOOP/SPARK
-CMD bash -C '/root/restart-hadoop-spark.sh'; 'bash'
+COPY scripts/restart-hadoop-spark.sh /usr/bin
+RUN chmod +x /usr/bin/restart-hadoop-spark.sh
+#Copy RunExample File
+COPY scripts/runexample.sh /root/runexample.sh
+RUN chmod +x /root/runexample.sh
diff --git a/docker/opensource-docker/conf/core-site.xml b/docker/opensource-docker/conf/core-site.xml
@@ -1,3 +1,4 @@
+<!--
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+-->
 
 <configuration>
 
diff --git a/docker/opensource-docker/conf/hdfs-site.xml b/docker/opensource-docker/conf/hdfs-site.xml
@@ -1,3 +1,4 @@
+<!--
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+-->
 
 <configuration>
 	<property>
diff --git a/docker/opensource-docker/conf/mapred-site.xml b/docker/opensource-docker/conf/mapred-site.xml
@@ -1,3 +1,4 @@
+<!--
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+-->
 
 <configuration>
 	<property>
diff --git a/docker/opensource-docker/conf/yarn-site.xml b/docker/opensource-docker/conf/yarn-site.xml
@@ -1,3 +1,4 @@
+<!--
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+-->
 
 <configuration>
 	<property>
diff --git a/docker/opensource-docker/scripts/hadoop-env.sh b/docker/opensource-docker/scripts/hadoop-env.sh
@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export JAVA_HOME=/usr/lib/jvm/java-7-oracle
+export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
diff --git a/docker/opensource-docker/scripts/runexample.sh b/docker/opensource-docker/scripts/runexample.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+#modify etc/hosts file to avoid connection error
+cp /etc/hosts /etc/hostsbak
+sed -i 's/::1/#::1/' /etc/hostsbak
+cat /etc/hostsbak > /etc/hosts
+rm -rf /etc/hostsbak
+#run wordcount example
+/usr/bin/restart-hadoop-spark.sh
+${HIBENCH_HOME}/workloads/wordcount/prepare/prepare.sh
+${HIBENCH_HOME}/workloads/wordcount/mapreduce/bin/run.sh
diff --git a/docker/scripts/build-docker.sh b/docker/scripts/build-docker.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -14,7 +15,6 @@
 # limitations under the License.
 
 
-#!/bin/bash
 
 CUR_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 HOME_DIR=${CUR_DIR}/..
@@ -38,12 +38,12 @@ case "$1" in
   "cdh")
      build-base
      second-step
-     sudo docker build -t hibench-docker ${HOME_DIR}/cdh-docker/
+     sudo docker build -t hibench-docker-cdh ${HOME_DIR}/cdh-docker/
      ;;
   "open-source")
      build-base
      second-step
-     sudo docker build -t hibench-docker ${HOME_DIR}/opensource-docker/
+     sudo docker build -t hibench-docker-opensource ${HOME_DIR}/opensource-docker/
      ;;
   *)
      exit 1
diff --git a/docker/scripts/gen_base_dockerfile.sh b/docker/scripts/gen_base_dockerfile.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -14,8 +15,6 @@
 # limitations under the License.
 
 
-#!/bin/bash
-
 CUR_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 HOME_DIR=${CUR_DIR}/..
 
@@ -56,9 +55,9 @@ while IFS='' read -r line || [[ -n "$line" ]]; do
         echo "ENV $line"  >> ${DOCKERFILE_ADDR}
 
         JDK_VERSION=`echo $line | cut -d ' ' -f 2`
-        sed -i 's/java-[0-9]/java-${JDK_VERSION}/' CDH_HADOOP_ENV_FILE
-        sed -i 's/java-[0-9]/java-${JDK_VERSION}/' OPENSOURCE_HADOOP_ENV_FILE
-        ;;
+        sed -i 's/java-[0-9]/java-'"$JDK_VERSION"'/' $CDH_HADOOP_ENV_FILE
+        sed -i 's/java-[0-9]/java-'"$JDK_VERSION"'/' $OPENSOURCE_HADOOP_ENV_FILE
+	;;
       *)
         echo "ENV $line"  >> ${DOCKERFILE_ADDR}
         ;;
diff --git a/docker/scripts/run-container.sh b/docker/scripts/run-container.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -14,7 +15,6 @@
 # limitations under the License.
 
 
-#!/bin/bash
 
 DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 
@@ -39,4 +39,10 @@ fi
 
 # run HiBench workload wordcount as an example
 # Format: sudo docker run (-v "LocalLargeDiskDir:/usr/loal"-it) hibench-hadoop-spark /bin/bash /root/HiBench/workloads/<workload-name>/prepare/prepare.sh
-sudo docker run -ti hibench-docker /bin/bash -c '${HIBENCH_HOME}/workloads/wordcount/prepare/prepare.sh && ${HIBENCH_HOME}/workloads/wordcount/mapreduce/bin/run.sh'
+if [ "$1" == "cdh" ]
+then
+   sudo docker run -ti hibench-docker-cdh /bin/bash -c '/root/runexample.sh'
+elif [ "$1" == "open-source" ]
+then
+   sudo docker run -ti hibench-docker-opensource /bin/bash -c '/root/runexample.sh'
+fi
