Merge remote-tracking branch 'origin/6.0/stage' into 6.0/release

Author: smeda-delphix

.github/scripts/verify-query-packages.sh

@@ -72,7 +72,7 @@ test "$(TARGET_KERNEL_FLAVORS=generic ./query-packages.sh list linux-kernel)" ==
 # Check that when a package has multiple dependencies they are printed in the
 # expected format.
 test "$(TARGET_KERNEL_FLAVORS="generic aws" ./query-packages.sh single -o dependencies zfs)" == \
-	"linux-kernel-generic,linux-kernel-aws"
+	"linux-kernel-generic,linux-kernel-aws,delphix-rust"
 
 # Check that the output from the appliance list contains zfs and
 # delphix-platform packages. Note, we explicitly do not use grep -q here as it

README.md

@@ -50,21 +50,14 @@ This quick tutorial shows how to build the packages managed by this framework.
 You need a system that meets the requirements above. For Delphix developers, you
 should clone the `bootstrap-18-04` group on DCoA.
 
-### Step 2. Clone this repository and run the setup script
+### Step 2. Clone this repository
 
 Clone this repository on the build VM.
 
 ```
 git clone https://github.com/delphix/linux-pkg.git
 ```
 
-Run the setup script. It only needs to be run once after cloning the VM.
-
-```
-cd linux-pkg
-./setup.sh
-```
-
 ### Step 3. Build a package
 
 We can now build an arbitrary package. Any package in the

buildpkg.sh

@@ -19,6 +19,7 @@ TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
 source "$TOP/lib/common.sh"
 
 logmust check_running_system
+logmust run_setup_if_needed
 
 function usage() {
 	[[ $# != 0 ]] && echo "$(basename "$0"): $*"

checkupdates.sh

@@ -19,6 +19,7 @@ TOP="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
 source "$TOP/lib/common.sh"
 
 logmust check_running_system
+logmust run_setup_if_needed
 
 function usage() {
 	[[ $# != 0 ]] && echo "$(basename "$0"): $*"

lib/common.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright 2018, 2020 Delphix
+# Copyright 2018, 2021 Delphix
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,6 +26,7 @@ export SUPPORTED_KERNEL_FLAVORS="generic aws gcp azure oracle"
 # for testing purposes to use jenkins-ops.<developer> instead.
 #
 export JENKINS_OPS_DIR="${JENKINS_OPS_DIR:-jenkins-ops}"
+export S3_DEVOPS_BRANCH="${S3_DEVOPS_BRANCH:-master}"
 
 export UBUNTU_DISTRIBUTION="bionic"
 
@@ -100,23 +101,50 @@ function logmust() {
 # scripts on their work system and changing its configuration.
 #
 function check_running_system() {
+	local msg
+
 	if [[ "$DISABLE_SYSTEM_CHECK" == "true" ]]; then
 		echo "WARNING: System check disabled."
 		return 0
 	fi
 
+	msg="Note that you can bypass this check by setting environment"
+	msg="${msg} variable DISABLE_SYSTEM_CHECK=true. Use this at your"
+	msg="${msg} own risk as running this command may modify your system."
+
 	if ! (command -v lsb_release >/dev/null &&
 		[[ $(lsb_release -cs) == "$UBUNTU_DISTRIBUTION" ]]); then
-		die "Script can only be ran on an ubuntu-${UBUNTU_DISTRIBUTION} system."
+		echo_error "Script can only be run on an ubuntu-${UBUNTU_DISTRIBUTION} system."
+		echo_bold "$msg"
+		exit 1
 	fi
 
 	if ! curl "http://169.254.169.254/latest/meta-datas" \
 		>/dev/null 2>&1; then
-		die "Not running in AWS, are you sure you are on the" \
+		echo_error "Not running in AWS, are you sure you are on the" \
 			"right system?"
+		echo_bold "$msg"
+		exit 1
 	fi
 }
 
+#
+# We need to have run setup.sh before running most linux-pkg commands.
+# This checks if setup has been run before. Note that if the system was
+# rebooted we want to rerun setup as cloud-init will reset apt sources on
+# boot.
+#
+function run_setup_if_needed() {
+	[[ -f /run/linux-pkg-setup ]] && return
+
+	check_env TOP
+	echo_bold "------------------------------------------------------------"
+	echo_bold "Running setup script"
+	echo_bold "------------------------------------------------------------"
+	logmust "$TOP/setup.sh"
+	echo_bold "------------------------------------------------------------"
+}
+
 #
 # Determine DEFAULT_GIT_BRANCH. If it is unset, default to the branch set in
 # branch.config.
@@ -632,7 +660,7 @@ function determine_dependencies_base_url() {
 		[[ -n "$suv" ]] || die "No artifacts found at $url"
 		DEPENDENCIES_BASE_URL="$url/$suv/input-artifacts/combined-packages/packages"
 	else
-		DEPENDENCIES_BASE_URL="s3://snapshot-de-images/builds/$JENKINS_OPS_DIR/devops-gate/master/linux-pkg/$DEFAULT_GIT_BRANCH/build-package"
+		DEPENDENCIES_BASE_URL="s3://snapshot-de-images/builds/$JENKINS_OPS_DIR/devops-gate/$S3_DEVOPS_BRANCH/linux-pkg/$DEFAULT_GIT_BRANCH/build-package"
 	fi
 
 	#
@@ -715,6 +743,26 @@ function fetch_dependencies() {
 	done
 }
 
+#
+# Run git fetch with the passed arguments. Git url must be passed as first
+# argument. If FETCH_GIT_TOKEN is set and this is a github repository
+# then pass-in the token when fetching.
+#
+function git_fetch_helper() {
+	local orig_url="$1"
+	local git_url="$1"
+	local label=''
+	shift
+
+	if [[ -n "$FETCH_GIT_TOKEN" ]] &&
+		[[ "$git_url" == https://github.com/* ]]; then
+		git_url="${git_url/https:\/\//https:\/\/${FETCH_GIT_TOKEN}@}"
+		label='[token passed]'
+	fi
+	echo "Running: $label git fetch $orig_url $*"
+	git fetch "$git_url" "$@" || die "git fetch failed"
+}
+
 #
 # Fetch package repository into $WORKDIR/repo
 #
@@ -731,14 +779,14 @@ function fetch_repo_from_git() {
 	# Otherwise just get the latest commit of the main branch.
 	#
 	if [[ "$DO_UPDATE_PACKAGE" == "true" ]]; then
-		logmust git fetch --no-tags "$PACKAGE_GIT_URL" \
+		logmust git_fetch_helper "$PACKAGE_GIT_URL" --no-tags \
 			"+$PACKAGE_GIT_BRANCH:repo-HEAD"
-		logmust git fetch --no-tags "$PACKAGE_GIT_URL" \
+		logmust git_fetch_helper "$PACKAGE_GIT_URL" --no-tags \
 			"+upstreams/$DEFAULT_GIT_BRANCH:upstream-HEAD"
 		logmust git show-ref repo-HEAD
 		logmust git show-ref upstream-HEAD
 	else
-		logmust git fetch --no-tags "$PACKAGE_GIT_URL" \
+		logmust git_fetch_helper "$PACKAGE_GIT_URL" --no-tags \
 			"+$PACKAGE_GIT_BRANCH:repo-HEAD" --depth=1
 		logmust git show-ref repo-HEAD
 	fi
@@ -826,8 +874,7 @@ function update_upstream_from_git() {
 	#
 	# Fetch updates from third-party upstream repository.
 	#
-	logmust git remote add upstream "$UPSTREAM_GIT_URL"
-	logmust git fetch upstream "$UPSTREAM_GIT_BRANCH"
+	logmust git_fetch_helper "$UPSTREAM_GIT_URL" "$UPSTREAM_GIT_BRANCH"
 
 	#
 	# Compare third-party upstream repository to our local snapshot of the

package-lists/build/main.pkgs

@@ -10,6 +10,7 @@ cloud-init
 crash-python
 crypt-blowfish
 delphix-platform
+delphix-rust
 delphix-sso-app
 drgn
 docker-python-image

packages/adoptopenjdk/config.sh

@@ -19,8 +19,8 @@
 DEFAULT_PACKAGE_GIT_URL=none
 PACKAGE_DEPENDENCIES="make-jpkg"
 
-_tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u282b08.tar.gz"
-_tarfile_sha256="e6e6e0356649b9696fa5082cfcb0663d4bef159fc22d406e3a012e71fce83a5c"
+_tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u302b08.tar.gz"
+_tarfile_sha256="cc13f274becf9dd5517b6be583632819dfd4dd81e524b5c1b4f406bdaf0e063a"
 _jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"
 
 function prepare() {

packages/delphix-rust/config.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# Copyright 2021 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# shellcheck disable=SC2034
+
+DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-rust.git"
+
+function build() {
+	logmust mkdir -p "$WORKDIR/repo"
+	logmust dpkg_buildpackage_default
+}

packages/td-agent/config.sh

@@ -17,7 +17,7 @@
 # shellcheck disable=SC2034
 
 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/zfs.git"
-PACKAGE_DEPENDENCIES="@linux-kernel"
+PACKAGE_DEPENDENCIES="@linux-kernel delphix-rust"
 
 UPSTREAM_GIT_URL="https://github.com/zfsonlinux/zfs.git"
 UPSTREAM_GIT_BRANCH="master"
@@ -29,7 +29,6 @@ function prepare() {
 		autogen \
 		autotools-dev \
 		build-essential \
-		cargo \
 		debhelper \
 		devscripts \
 		dh-autoreconf \
@@ -52,10 +51,10 @@ function prepare() {
 		pkg-config \
 		po-debconf \
 		python3 \
-		rustc \
 		uuid-dev \
 		zlib1g-dev
 	logmust install_kernel_headers
+	logmust install_pkgs "$DEPDIR"/delphix-rust/*.deb
 }
 
 function checkstyle() {