Merge remote-tracking branch 'origin/6.0/release' into 6.0/patch

Author: rasantel

default-package-config.sh

@@ -61,7 +61,8 @@ function kernel_build() {
 	#       debian/rules command, the rightmost declaration
 	#       is the one that is actually used.
 	#
-	local debian_rules_extra_args="$2"
+	shift
+	local debian_rules_extra_args=("$@")
 
 	logmust cd "$WORKDIR/repo"
 
@@ -91,7 +92,7 @@ function kernel_build() {
 	#
 	local canonical_abinum delphix_abinum kernel_release kernel_version
 	canonical_abinum=$(fakeroot debian/rules printenv | grep -E '^abinum ' | cut -d= -f2 | tr -d '[:space:]')
-	delphix_abinum="dlpx-$(date -u +"%Y%m%dt%H%M%S")-$(git rev-parse --short HEAD)-${canonical_abinum}"
+	delphix_abinum="${canonical_abinum}-$(date -u +"dx%Y%m%d%H")-$(git rev-parse --short HEAD)"
 	kernel_release=$(fakeroot debian/rules printenv | grep -E '^release ' | cut -d= -f2 | tr -d '[:space:]')
 
 	#
@@ -113,20 +114,35 @@ function kernel_build() {
 	# disable_d_i=true
 	#   This prevents udeb packages from being built as they are
 	#   not consumed by the Delphix Appliance.
-	#
-	local debian_rules_args="skipdbg=false uefi_signed=false disable_d_i=true flavours=$platform abinum=${delphix_abinum} ${debian_rules_extra_args}"
+	# do_dkms_*=false
+	#   This disables the build of various out-of-tree kernel modules
+	#   that we do not use in our product or that we provide separately.
+	#
+	local debian_rules_args=(
+		"skipdbg=false"
+		"uefi_signed=false"
+		"disable_d_i=true"
+		"do_zfs=false"
+		"do_dkms_nvidia=false"
+		"do_dkms_nvidia_server=false"
+		"do_dkms_vbox=false"
+		"do_dkms_wireguard=false"
+		"flavours=$platform"
+		"abinum=${delphix_abinum}"
+	)
+	debian_rules_args+=("${debian_rules_extra_args[@]}")
 
 	#
 	# Clean up everything generated so far and recreate the
 	# final control file with the arguments that we want.
 	#
-	logmust fakeroot debian/rules clean ${debian_rules_args}
+	logmust fakeroot debian/rules clean "${debian_rules_args[@]}"
 
 	#
 	# Print the environment configuration solely for
 	# debugging purposes.
 	#
-	logmust fakeroot debian/rules printenv ${debian_rules_args}
+	logmust fakeroot debian/rules printenv "${debian_rules_args[@]}"
 
 	#
 	# The default value of the tool argument for mk-build-deps
@@ -139,7 +155,7 @@ function kernel_build() {
 	local build_deps_tool="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes"
 	logmust sudo mk-build-deps --install debian/control --tool "${build_deps_tool}"
 
-	logmust fakeroot debian/rules "binary" ${debian_rules_args}
+	logmust fakeroot debian/rules "binary" "${debian_rules_args[@]}"
 
 	logmust cd "$WORKDIR"
 	logmust mv ./*deb "artifacts/"
@@ -220,24 +236,24 @@ function kernel_update_upstream() {
 	# Note that "generic" (used mainly ESX) is a special
 	# case as we are currently using the HWE kernel image.
 	#
-	local tag_prefix
+	local tag_prefix_flavour
 	if [[ "${platform}" == generic ]] &&
 		[[ "$UBUNTU_DISTRIBUTION" == bionic ]]; then
-		tag_prefix="Ubuntu-hwe-${kernel_version}-${abinum}"
+		tag_prefix_flavour="Ubuntu-hwe"
 	elif [[ "${platform}" == aws ]] ||
 		[[ "${platform}" == azure ]] ||
 		[[ "${platform}" == gcp ]] ||
 		[[ "${platform}" == oracle ]]; then
-
-		local kvers_major kvers_minor short_kvers
-		kvers_major=$(echo "${kernel_version}" | cut -d '.' -f 1)
-		kvers_minor=$(echo "${kernel_version}" | cut -d '.' -f 2)
-		short_kvers="${kvers_major}.${kvers_minor}"
-
-		tag_prefix="Ubuntu-${platform}-${short_kvers}-${kernel_version}-${abinum}"
+		tag_prefix_flavour="Ubuntu-${platform}"
 	else
 		die "assertion: unexpected platform: ${platform}"
 	fi
+
+	local tag_prefix kvers_major kvers_minor short_kvers
+	kvers_major=$(echo "${kernel_version}" | cut -d '.' -f 1)
+	kvers_minor=$(echo "${kernel_version}" | cut -d '.' -f 2)
+	short_kvers="${kvers_major}.${kvers_minor}"
+	tag_prefix="${tag_prefix_flavour}-${short_kvers}-${kernel_version}-${abinum}"
 	echo "note: upstream tag prefix used: ${tag_prefix}"
 
 	#
@@ -274,8 +290,18 @@ function kernel_update_upstream() {
 
 	logmust git fetch upstream "+refs/tags/${upstream_tag}:refs/tags/${upstream_tag}"
 
+	#
+	# Note that we add '^{}' at the end to dereference the tag recursively
+	# until it arrives to an actual commit. This is needed in case the
+	# tag points to an anotated tag object which contains extra information
+	# such as a PGP signature. That annoted tag will in turn reference the
+	# actual commit, which will be returned when appending ^{}.
+	# upstream-HEAD will be pointing to the commit directly rather than the
+	# annoted tag object, so if we want to compare the two we need to query
+	# for the dereferenced commit. See 'git help gitrevisions' for more info.
+	#
 	local upstream_tag_commit
-	upstream_tag_commit="$(git rev-parse "refs/tags/${upstream_tag}")" ||
+	upstream_tag_commit="$(git rev-parse "refs/tags/${upstream_tag}^{}")" ||
 		die "couldn't get commit of tag ${upstream_tag}"
 	echo "note: upstream tag: ${upstream_tag}, commit ${upstream_tag_commit}"
 

lib/common.sh

@@ -35,7 +35,7 @@ export UBUNTU_DISTRIBUTION="bionic"
 #  2. "archive": dowloading from apt
 #  3. "prebuilt": pre-built kernel stored in artifactory
 #
-export DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE="prebuilt"
+export DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE="delphix"
 
 # shellcheck disable=SC2086
 function enable_colors() {
@@ -89,7 +89,7 @@ function die() {
 }
 
 function logmust() {
-	echo Running: "$@"
+	[[ "$LOGGING" == "false" ]] || echo Running: "$@" >&2
 	"$@" || die "failed command '$*'"
 }
 
@@ -470,7 +470,7 @@ function install_pkgs() {
 		echo "Running: sudo env DEBIAN_FRONTEND=noninteractive " \
 			"apt-get install -y $*"
 		sudo env DEBIAN_FRONTEND=noninteractive apt-get install \
-			-y "$@" && return
+			-y --allow-downgrades "$@" && return
 		echo "apt-get install failed, retrying."
 		sleep 10
 	done
@@ -1116,7 +1116,26 @@ function fetch_kernel_from_artifactory() {
 	url="$url/linux-pkg/linux-prebuilt/${artifactory_deb}"
 
 	logmust cd "$WORKDIR/artifacts"
-	logmust wget -nv "$url"
+	fetch_file_from_artifactory "$url"
+}
+
+function fetch_file_from_artifactory() {
+	local url="$1"
+	local sha256_expected="$2"
+	local file sha256_actual
+
+	[[ -n "$url" ]] || die "url argument is missing."
+	file="$(basename "$url")"
+
+	logmust wget -nv "$url" -O "$file"
+
+	if [[ -n "$sha256_expected" ]]; then
+		sha256_actual="$(sha256sum "$file" | awk '{print $1}')"
+		if [[ "$sha256_expected" != "$sha256_actual" ]]; then
+			die "SHA256 mismatch. Expected: $sha256_expected," \
+				"Actual: $sha256_actual"
+		fi
+	fi
 }
 
 #

package-lists/build/main.pkgs

@@ -18,6 +18,8 @@ grub2
 libkdumpfile
 make-jpkg
 makedumpfile
+masking
+misc-debs
 nfs-utils
 performance-diagnostics
 ptools
@@ -26,4 +28,4 @@ recovery-environment
 savedump
 sdb
 targetcli-fb
-td-agent-prebuilt
+virtualization

package-lists/update/main.pkgs

@@ -17,6 +17,11 @@ crash
 drgn
 grub2
 libkdumpfile
+linux-kernel-aws
+linux-kernel-azure
+linux-kernel-gcp
+linux-kernel-generic
+linux-kernel-oracle
 makedumpfile
 nfs-utils
 python-rtslib-fb

packages/adoptopenjdk/config.sh

@@ -19,8 +19,9 @@
 DEFAULT_PACKAGE_GIT_URL=none
 PACKAGE_DEPENDENCIES="make-jpkg"
 
-tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz"
-jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"
+_tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz"
+_tarfile_sha256="733755fd649fad6ae91fc083f7e5a5a0b56410fb6ac1815cff29f744b128b1b1"
+_jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"
 
 function prepare() {
 	logmust install_pkgs "$DEPDIR"/make-jpkg/*.deb
@@ -29,15 +30,15 @@ function prepare() {
 function fetch() {
 	logmust cd "$WORKDIR/"
 
-	local url="http://artifactory.delphix.com/artifactory"
+	local url="http://artifactory.delphix.com/artifactory/java-binaries/linux/jdk/8/$_tarfile"
 
-	logmust wget -nv "$url/java-binaries/linux/jdk/8/$tarfile" -O "$tarfile"
+	logmust fetch_file_from_artifactory "$url" "$_tarfile_sha256"
 }
 
 function build() {
 	logmust cd "$WORKDIR/"
 
-	logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$tarfile" <<<y
+	logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$_tarfile" <<<y
 
 	logmust mv ./*deb "$WORKDIR/artifacts/"
 	#
@@ -49,7 +50,7 @@ function build() {
 	# packages, such as the app-gate, decide to fetch and install Java from
 	# the Linux-pkg bundle.
 	#
-	logmust bash -c "echo $jdk_path >'$WORKDIR/artifacts/JDK_PATH'"
+	logmust bash -c "echo $_jdk_path >'$WORKDIR/artifacts/JDK_PATH'"
 
-	echo "Tar file: $tarfile" >"$WORKDIR/artifacts/BUILD_INFO"
+	echo "Tar file: $_tarfile" >"$WORKDIR/artifacts/BUILD_INFO"
 }

packages/java8/config.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Copyright 2021 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# shellcheck disable=SC2034
+
+DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/masking/dms-core-gate.git"
+PACKAGE_DEPENDENCIES="adoptopenjdk"
+
+function prepare() {
+	logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb
+}
+
+function build() {
+	export JAVA_HOME
+	JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") ||
+		die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH"
+
+	logmust cd "$WORKDIR/repo"
+
+	#
+	# The "appliance-build-stage0" Jenkins job consumes this file,
+	# along with various other files (e.g. licensing metadata).
+	# Thus, if we don't generate it here, the Jenkins job that
+	# builds the appliance will fail.
+	#
+	# shellcheck disable=SC2016
+	logmust jq -n \
+		--arg h "$(git rev-parse HEAD)" \
+		--arg d "$(date --utc --iso-8601=seconds)" \
+		'{ "dms-core-gate" : { "git-hash" : $h, "date": $d }}' \
+		>"$WORKDIR/artifacts/metadata.json"
+
+	logmust ./gradlew --no-daemon --stacktrace \
+		-Porg.gradle.configureondemand=false \
+		-PenvironmentName=linuxappliance \
+		clean \
+		generateLicenseReport \
+		:dist:distDeb \
+		:dist:distLicenseReport
+
+	logmust rsync -av dist/build/distributions/ "$WORKDIR/artifacts/"
+	logmust cp -v \
+		dist/build/reports/dependency-license/* "$WORKDIR/artifacts/"
+}