Merge remote-tracking branch 'origin/6.0/stage' into 6.0/release

rasantel · rasantel · commit 262dd13379b6 · 2021-04-19T09:09:34.000-04:00
diff --git a/default-package-config.sh b/default-package-config.sh
@@ -61,7 +61,8 @@ function kernel_build() {
 	#       debian/rules command, the rightmost declaration
 	#       is the one that is actually used.
 	#
-	local debian_rules_extra_args="$2"
+	shift
+	local debian_rules_extra_args=("$@")
 
 	logmust cd "$WORKDIR/repo"
 
@@ -91,7 +92,7 @@ function kernel_build() {
 	#
 	local canonical_abinum delphix_abinum kernel_release kernel_version
 	canonical_abinum=$(fakeroot debian/rules printenv | grep -E '^abinum ' | cut -d= -f2 | tr -d '[:space:]')
-	delphix_abinum="dlpx-$(date -u +"%Y%m%dt%H%M%S")-$(git rev-parse --short HEAD)-${canonical_abinum}"
+	delphix_abinum="${canonical_abinum}-$(date -u +"dx%Y%m%d%H")-$(git rev-parse --short HEAD)"
 	kernel_release=$(fakeroot debian/rules printenv | grep -E '^release ' | cut -d= -f2 | tr -d '[:space:]')
 
 	#
@@ -113,20 +114,35 @@ function kernel_build() {
 	# disable_d_i=true
 	#   This prevents udeb packages from being built as they are
 	#   not consumed by the Delphix Appliance.
-	#
-	local debian_rules_args="skipdbg=false uefi_signed=false disable_d_i=true flavours=$platform abinum=${delphix_abinum} ${debian_rules_extra_args}"
+	# do_dkms_*=false
+	#   This disables the build of various out-of-tree kernel modules
+	#   that we do not use in our product or that we provide separately.
+	#
+	local debian_rules_args=(
+		"skipdbg=false"
+		"uefi_signed=false"
+		"disable_d_i=true"
+		"do_zfs=false"
+		"do_dkms_nvidia=false"
+		"do_dkms_nvidia_server=false"
+		"do_dkms_vbox=false"
+		"do_dkms_wireguard=false"
+		"flavours=$platform"
+		"abinum=${delphix_abinum}"
+	)
+	debian_rules_args+=("${debian_rules_extra_args[@]}")
 
 	#
 	# Clean up everything generated so far and recreate the
 	# final control file with the arguments that we want.
 	#
-	logmust fakeroot debian/rules clean ${debian_rules_args}
+	logmust fakeroot debian/rules clean "${debian_rules_args[@]}"
 
 	#
 	# Print the environment configuration solely for
 	# debugging purposes.
 	#
-	logmust fakeroot debian/rules printenv ${debian_rules_args}
+	logmust fakeroot debian/rules printenv "${debian_rules_args[@]}"
 
 	#
 	# The default value of the tool argument for mk-build-deps
@@ -139,7 +155,7 @@ function kernel_build() {
 	local build_deps_tool="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes"
 	logmust sudo mk-build-deps --install debian/control --tool "${build_deps_tool}"
 
-	logmust fakeroot debian/rules "binary" ${debian_rules_args}
+	logmust fakeroot debian/rules "binary" "${debian_rules_args[@]}"
 
 	logmust cd "$WORKDIR"
 	logmust mv ./*deb "artifacts/"
@@ -220,24 +236,24 @@ function kernel_update_upstream() {
 	# Note that "generic" (used mainly ESX) is a special
 	# case as we are currently using the HWE kernel image.
 	#
-	local tag_prefix
+	local tag_prefix_flavour
 	if [[ "${platform}" == generic ]] &&
 		[[ "$UBUNTU_DISTRIBUTION" == bionic ]]; then
-		tag_prefix="Ubuntu-hwe-${kernel_version}-${abinum}"
+		tag_prefix_flavour="Ubuntu-hwe"
 	elif [[ "${platform}" == aws ]] ||
 		[[ "${platform}" == azure ]] ||
 		[[ "${platform}" == gcp ]] ||
 		[[ "${platform}" == oracle ]]; then
-
-		local kvers_major kvers_minor short_kvers
-		kvers_major=$(echo "${kernel_version}" | cut -d '.' -f 1)
-		kvers_minor=$(echo "${kernel_version}" | cut -d '.' -f 2)
-		short_kvers="${kvers_major}.${kvers_minor}"
-
-		tag_prefix="Ubuntu-${platform}-${short_kvers}-${kernel_version}-${abinum}"
+		tag_prefix_flavour="Ubuntu-${platform}"
 	else
 		die "assertion: unexpected platform: ${platform}"
 	fi
+
+	local tag_prefix kvers_major kvers_minor short_kvers
+	kvers_major=$(echo "${kernel_version}" | cut -d '.' -f 1)
+	kvers_minor=$(echo "${kernel_version}" | cut -d '.' -f 2)
+	short_kvers="${kvers_major}.${kvers_minor}"
+	tag_prefix="${tag_prefix_flavour}-${short_kvers}-${kernel_version}-${abinum}"
 	echo "note: upstream tag prefix used: ${tag_prefix}"
 
 	#
@@ -274,8 +290,18 @@ function kernel_update_upstream() {
 
 	logmust git fetch upstream "+refs/tags/${upstream_tag}:refs/tags/${upstream_tag}"
 
+	#
+	# Note that we add '^{}' at the end to dereference the tag recursively
+	# until it arrives to an actual commit. This is needed in case the
+	# tag points to an anotated tag object which contains extra information
+	# such as a PGP signature. That annoted tag will in turn reference the
+	# actual commit, which will be returned when appending ^{}.
+	# upstream-HEAD will be pointing to the commit directly rather than the
+	# annoted tag object, so if we want to compare the two we need to query
+	# for the dereferenced commit. See 'git help gitrevisions' for more info.
+	#
 	local upstream_tag_commit
-	upstream_tag_commit="$(git rev-parse "refs/tags/${upstream_tag}")" ||
+	upstream_tag_commit="$(git rev-parse "refs/tags/${upstream_tag}^{}")" ||
 		die "couldn't get commit of tag ${upstream_tag}"
 	echo "note: upstream tag: ${upstream_tag}, commit ${upstream_tag_commit}"
 
diff --git a/lib/common.sh b/lib/common.sh
@@ -35,7 +35,7 @@ export UBUNTU_DISTRIBUTION="bionic"
 #  2. "archive": dowloading from apt
 #  3. "prebuilt": pre-built kernel stored in artifactory
 #
-export DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE="prebuilt"
+export DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE="delphix"
 
 # shellcheck disable=SC2086
 function enable_colors() {
@@ -89,7 +89,7 @@ function die() {
 }
 
 function logmust() {
-	echo Running: "$@"
+	[[ "$LOGGING" == "false" ]] || echo Running: "$@" >&2
 	"$@" || die "failed command '$*'"
 }
 
@@ -470,7 +470,7 @@ function install_pkgs() {
 		echo "Running: sudo env DEBIAN_FRONTEND=noninteractive " \
 			"apt-get install -y $*"
 		sudo env DEBIAN_FRONTEND=noninteractive apt-get install \
-			-y "$@" && return
+			-y --allow-downgrades "$@" && return
 		echo "apt-get install failed, retrying."
 		sleep 10
 	done
diff --git a/package-lists/build/main.pkgs b/package-lists/build/main.pkgs
@@ -18,6 +18,7 @@ grub2
 libkdumpfile
 make-jpkg
 makedumpfile
+masking
 misc-debs
 nfs-utils
 performance-diagnostics
@@ -27,3 +28,4 @@ recovery-environment
 savedump
 sdb
 targetcli-fb
+virtualization
diff --git a/package-lists/update/main.pkgs b/package-lists/update/main.pkgs
@@ -17,6 +17,11 @@ crash
 drgn
 grub2
 libkdumpfile
+linux-kernel-aws
+linux-kernel-azure
+linux-kernel-gcp
+linux-kernel-generic
+linux-kernel-oracle
 makedumpfile
 nfs-utils
 python-rtslib-fb
diff --git a/packages/masking/config.sh b/packages/masking/config.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Copyright 2021 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# shellcheck disable=SC2034
+
+DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/masking/dms-core-gate.git"
+PACKAGE_DEPENDENCIES="adoptopenjdk"
+
+function prepare() {
+	logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb
+}
+
+function build() {
+	export JAVA_HOME
+	JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") ||
+		die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH"
+
+	logmust cd "$WORKDIR/repo"
+
+	#
+	# The "appliance-build-stage0" Jenkins job consumes this file,
+	# along with various other files (e.g. licensing metadata).
+	# Thus, if we don't generate it here, the Jenkins job that
+	# builds the appliance will fail.
+	#
+	# shellcheck disable=SC2016
+	logmust jq -n \
+		--arg h "$(git rev-parse HEAD)" \
+		--arg d "$(date --utc --iso-8601=seconds)" \
+		'{ "dms-core-gate" : { "git-hash" : $h, "date": $d }}' \
+		>"$WORKDIR/artifacts/metadata.json"
+
+	logmust ./gradlew --no-daemon --stacktrace \
+		-Porg.gradle.configureondemand=false \
+		-PenvironmentName=linuxappliance \
+		clean \
+		generateLicenseReport \
+		:dist:distDeb \
+		:dist:distLicenseReport
+
+	logmust rsync -av dist/build/distributions/ "$WORKDIR/artifacts/"
+	logmust cp -v \
+		dist/build/reports/dependency-license/* "$WORKDIR/artifacts/"
+}
diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+#
+# Copyright 2021 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# shellcheck disable=SC2034
+
+DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/app/dlpx-app-gate.git"
+PACKAGE_DEPENDENCIES="adoptopenjdk crypt-blowfish misc-debs"
+
+function prepare() {
+	logmust install_pkgs \
+		ant \
+		gcc \
+		libcairo2 \
+		libcurl4-openssl-dev \
+		libjbig0 \
+		libnss3-dev \
+		libnss3-dbg \
+		libnss3-tools \
+		libpam0g-dev \
+		libpixman-1-0 \
+		libssl-dev \
+		libtiff5 \
+		libxcb-render0 \
+		libxcb-shm0 \
+		python-jira \
+		python-requests \
+		rsync
+
+	logmust install_pkgs \
+		"$DEPDIR"/adoptopenjdk/*.deb \
+		"$DEPDIR"/crypt-blowfish/*.deb \
+		"$DEPDIR"/misc-debs/unzip_6.0-21ubuntu1_amd64.deb
+}
+
+function build() {
+	export JAVA_HOME
+	JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") ||
+		die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH"
+
+	export LANG
+	LANG=en_US.UTF-8
+
+	logmust cd "$WORKDIR/repo"
+
+	#
+	# The "appliance-build-stage0" Jenkins job consumes this file,
+	# along with various other files (e.g. licensing metadata).
+	# Thus, if we don't generate it here, the Jenkins job that
+	# builds the appliance will fail.
+	#
+	# shellcheck disable=SC2016
+	logmust jq -n \
+		--arg h "$(git rev-parse HEAD)" \
+		--arg d "$(date --utc --iso-8601=seconds)" \
+		'{ "dlpx-app-gate" : { "git-hash" : $h, "date": $d }}' \
+		>"$WORKDIR/artifacts/metadata.json"
+
+	#
+	# Build the virtualization package
+	#
+	logmust cd "$WORKDIR/repo/appliance"
+	if [[ -n "$DELPHIX_RELEASE_VERSION" ]]; then
+		logmust ant -Ddockerize=true -DbuildJni=true \
+			-DhotfixGenDlpxVersion="$DELPHIX_RELEASE_VERSION" \
+			all package
+	else
+		logmust ant -Ddockerize=true -DbuildJni=true all package
+	fi
+
+	#
+	# Publish the virtualization package artifacts
+	#
+	logmust cd "$WORKDIR/repo/appliance"
+	logmust rsync -av packaging/build/distributions/ "$WORKDIR/artifacts/"
+	logmust cp -vr \
+		bin/out/common/com.delphix.common/uem/tars \
+		"$WORKDIR/artifacts/hostchecker2"
+	logmust cp -v \
+		server/api/build/api/json-schemas/delphix.json \
+		"$WORKDIR/artifacts"
+	logmust cp -v \
+		dist/server/opt/delphix/client/etc/api.ini \
+		"$WORKDIR/artifacts"
+	logmust cp -v \
+		packaging/build/reports/dependency-license/* \
+		"$WORKDIR/artifacts/"
+
+	#
+	# Build the "toolkit-devkit" artifacts
+	#
+	logmust cd "$WORKDIR/repo/appliance/toolkit"
+	if [[ -n "$DELPHIX_RELEASE_VERSION" ]]; then
+		logmust ant \
+			-Dversion.number="$DELPHIX_RELEASE_VERSION" \
+			toolkit-devkit
+	else
+		logmust ant \
+			"-Dversion.number=$(date --utc +%Y-%m-%d-%H-%m)" \
+			toolkit-devkit
+	fi
+
+	#
+	# Publish the "toolkit-devkit" artifacts
+	#
+	logmust cd "$WORKDIR/repo/appliance"
+	logmust mkdir -p "$WORKDIR/artifacts/hostchecker2"
+	logmust cp -v toolkit/toolkit-devkit.tar "$WORKDIR/artifacts"
+}
diff --git a/query-packages.sh b/query-packages.sh
@@ -19,6 +19,12 @@ cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
 TOP="$PWD"
 source "$TOP/lib/common.sh"
 
+#
+# Disable "logmust" output as that is unnecessary here, and actually
+# trips up some of our testing (e.g. verify-query-packages.sh).
+#
+export LOGGING=false
+
 ALL_OUTPUT_FIELDS=(name git-url dependencies can-update)
 
 function usage() {
diff --git a/setup.sh b/setup.sh
@@ -93,12 +93,14 @@ logmust sudo apt-get update
 #   build dependencies from a control file.
 # - install_shfmt and shellcheck are needed for - make check - to be able to
 #   make sure style checks are fine.
+# - jq is used to generate a JSON formatted metadata file by some packages.
 #
 logmust install_pkgs \
 	debhelper \
 	devscripts \
 	equivs \
-	shellcheck
+	shellcheck \
+	jq
 
 logmust install_shfmt
 
