Merge remote-tracking branch 'origin/6.0/stage' into 6.0/release

Author: smeda-delphix

buildpkg.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright 2018, 2021 Delphix
 #

checkupdates.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright 2020 Delphix
 #

default-package-config.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright 2018, 2020 Delphix
 #
@@ -234,20 +234,24 @@ function kernel_update_upstream() {
 	# version and the ABI num that we got above.
 	#
 	# Note that "generic" (used mainly ESX) is a special
-	# case as we are currently using the HWE kernel image.
+	# case on bionic where we are using the HWE kernel image.
 	#
 	local tag_prefix_flavour
-	if [[ "${platform}" == generic ]] &&
-		[[ "$UBUNTU_DISTRIBUTION" == bionic ]]; then
-		tag_prefix_flavour="Ubuntu-hwe"
-	elif [[ "${platform}" == aws ]] ||
-		[[ "${platform}" == azure ]] ||
-		[[ "${platform}" == gcp ]] ||
-		[[ "${platform}" == oracle ]]; then
+	case "${platform}" in
+	generic)
+		if [[ "$UBUNTU_DISTRIBUTION" == bionic ]]; then
+			tag_prefix_flavour="Ubuntu-hwe"
+		else
+			tag_prefix_flavour="Ubuntu"
+		fi
+		;;
+	aws | azure | gcp | oracle)
 		tag_prefix_flavour="Ubuntu-${platform}"
-	else
+		;;
+	*)
 		die "assertion: unexpected platform: ${platform}"
-	fi
+		;;
+	esac
 
 	local tag_prefix kvers_major kvers_minor short_kvers
 	kvers_major=$(echo "${kernel_version}" | cut -d '.' -f 1)
@@ -282,6 +286,26 @@ function kernel_update_upstream() {
 	#
 	local upstream_tag_info
 	upstream_tag_info=$(git ls-remote --tags --ref upstream | grep "${tag_prefix}" | tail -n 1)
+
+	if [[ -z "${upstream_tag_info}" ]]; then
+		echo "tag with prefix ${tag_prefix} not found."
+		tag_prefix="${tag_prefix_flavour}-${kernel_version}-${abinum}"
+		#
+		# Canonical has 2 ways of naming their tags:
+		# - Ubuntu-gcp-5.4.0-1046.49
+		# - Ubuntu-gcp-5.4-5.4.0-1046.49_18.04.1
+		#
+		# For a given kernel version and a given distribution, only one
+		# naming scheme is being used. When two distributions (such
+		# as 20.04 and 18.04) use the same major kernel version,
+		# Canonical first applies the patches to one distribution and
+		# then creates a base tag. It then backports those changes to
+		# the older distribution, and then creates a longer tag name.
+		#
+		echo "trying tag prefix: ${tag_prefix}."
+		upstream_tag_info=$(git ls-remote --tags --ref upstream | grep "${tag_prefix}" | tail -n 1)
+	fi
+
 	[[ -z "${upstream_tag_info}" ]] && die "could not find upstream tag for tag prefix: ${tag_prefix}"
 
 	local upstream_tag
@@ -347,17 +371,12 @@ function kernel_merge_with_upstream() {
 
 	check_git_ref "$upstream_ref" "$repo_ref"
 
-	if git merge-base --is-ancestor "$upstream_ref" "$repo_ref"; then
-		echo "NOTE: $PACKAGE is already up-to-date with upstream."
-		return 0
-	fi
-
 	#
 	# Ensure that there is a commit marking the start of
 	# the Delphix set of patches. Then get the hash of
 	# the commit right before it.
 	#
-	local dlpx_patch_end dlpx_patch_start current_ubuntu_commit
+	local dlpx_patch_end dlpx_patch_start current_ubuntu_commit upstream_head_commit
 	dlpx_patch_start=$(git log --pretty=oneline repo-HEAD | grep @@DELPHIX_PATCHSET_START@@ | awk '{ print $1 }')
 	[[ -z "${dlpx_patch_start}" ]] && die "could not find DELPHIX_PATCHSET_START"
 	[[ $(wc -l <<<"${dlpx_patch_start}") != 1 ]] && die "multiple DELPHIX_PATCHSET_START commits - ${dlpx_patch_start}"
@@ -366,6 +385,13 @@ function kernel_merge_with_upstream() {
 	dlpx_patch_end=$(git rev-parse repo-HEAD)
 	[[ -z "${dlpx_patch_end}" ]] && die "could not find repo-HEAD's head commit"
 
+	upstream_head_commit="$(git rev-parse "$upstream_ref")"
+
+	if [[ "$current_ubuntu_commit" == "$upstream_head_commit" ]]; then
+		echo "NOTE: $PACKAGE is already up-to-date with upstream."
+		return 0
+	fi
+
 	#
 	# We rebase all the Delphix commits on top of the new upstream-HEAD
 	# by using git cherry-pick. Note that we also save the previous

lib/common.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright 2018, 2020 Delphix
 #
@@ -444,9 +444,22 @@ function create_workdir() {
 function install_pkgs() {
 	for attempt in {1..3}; do
 		echo "Running: sudo env DEBIAN_FRONTEND=noninteractive " \
-			"apt-get install -y $*"
+			"apt-get install -y --allow-downgrades $*"
+
+		#
+		# We use the "--allow-downgrades" for the case of a
+		# package needing to install the "unzip" debian package
+		# that we build via the "misc-debs" linux-pkg package.
+		# The "misc-debs" based "unzip" package may have an
+		# older version than what's already installed on the
+		# system, so we need to "--allow-downgrades" in order to
+		# install that specific package; further, that specific
+		# package is required to build the "virtualization"
+		# debian packages.
+		#
 		sudo env DEBIAN_FRONTEND=noninteractive apt-get install \
 			-y --allow-downgrades "$@" && return
+
 		echo "apt-get install failed, retrying."
 		sleep 10
 	done
@@ -481,12 +494,13 @@ function list_all_packages() {
 }
 
 #
-# Read a package-list file and return listed packages in _RET_LIST.
+# Read a new-line separated list from a file, removing extra whitespace
+# and comments. Return items in _RET_LIST.
 #
-function read_package_list() {
+function read_list() {
 	local file="$1"
 
-	local pkg
+	local item
 	local line
 
 	_RET_LIST=()
@@ -495,13 +509,26 @@ function read_package_list() {
 
 	while read -r line; do
 		# trim whitespace
-		pkg=$(echo "$line" | tr -d '[:space:]')
-		[[ -z "$pkg" ]] && continue
+		item=$(echo "$line" | tr -d '[:space:]')
+		[[ -z "$item" ]] && continue
 		# ignore comments
-		[[ ${pkg:0:1} == "#" ]] && continue
+		[[ ${item:0:1} == "#" ]] && continue
+		_RET_LIST+=("$item")
+	done <"$file" || die "Failed to read list: $file"
+}
+
+#
+# Read a package-list file and return listed packages in _RET_LIST.
+#
+function read_package_list() {
+	local file="$1"
+
+	local pkg
+
+	logmust read_list "$file"
+	for pkg in "${_RET_LIST[@]}"; do
 		check_package_exists "$pkg"
-		_RET_LIST+=("$pkg")
-	done <"$file" || die "Failed to read package list: $file"
+	done
 }
 
 #

packages/adoptopenjdk/config.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright 2018, 2020 Delphix
 #
@@ -19,8 +19,8 @@
 DEFAULT_PACKAGE_GIT_URL=none
 PACKAGE_DEPENDENCIES="make-jpkg"
 
-_tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz"
-_tarfile_sha256="733755fd649fad6ae91fc083f7e5a5a0b56410fb6ac1815cff29f744b128b1b1"
+_tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u282b08.tar.gz"
+_tarfile_sha256="e6e6e0356649b9696fa5082cfcb0663d4bef159fc22d406e3a012e71fce83a5c"
 _jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"
 
 function prepare() {

packages/bcc/config.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright 2018, 2020 Delphix
 #

packages/bpftrace/config.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright 2018, 2020 Delphix
 #

packages/challenge-response/config.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright 2019 Delphix
 #

packages/cloud-init/config.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright 2018, 2019 Delphix
 #

packages/connstat/config.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Copyright 2018, 2020 Delphix
 #