Merge remote-tracking branch 'origin/6.0/release' into 6.0/patch

Author: smeda-delphix

.github/scripts/verify-query-packages.sh

@@ -63,7 +63,6 @@ done
 ./query-packages.sh list build/main.pkgs >/dev/null
 ./query-packages.sh list build/kernel-modules.pkgs >/dev/null
 ./query-packages.sh list linux-kernel >/dev/null
-./query-packages.sh list update/main.pkgs >/dev/null
 
 # Check that overriding TARGET_KERNEL_FLAVORS changes which kernel packages are
 # returned.
@@ -86,3 +85,5 @@ test "$(TARGET_KERNEL_FLAVORS="generic aws" ./query-packages.sh single -o depend
 # This redoes the "list all" test from above
 cd packages
 diff <(ls -1 | sort) <(../query-packages.sh list all 2>&1 | sort)
+
+echo "All tests passed"

README.md

@@ -346,30 +346,6 @@ Here is a list of variables that can be defined for a package:
   "dep" is the dependency's name. A special value can be passed for packages
   that target all the supported flavours of the linux-kernel: `@linux-kernel`.
 
-* **DEFAULT_PACKAGE_GIT_BRANCH**: (DEPRECATED) Default git branch to use when
-  fetching from or pushing to `DEFAULT_PACKAGE_GIT_URL`. This should be
-  typically left unset. The branch to fetch the package from defaults
-  to the value of the environment variable `DEFAULT_BRANCH`, which itself
-  defaults to "master".
-  WARNING: do not set this parameter unless you know exactly what you are doing,
-  as our current versioning convention is to use DEFAULT_BRANCH for each
-  package. This parameter may be removed in the future.
-
-* **DEFAULT_PACKAGE_VERSION**: (Optional) The version of the package is set to
-  this value when it is built. **Note:** If this field is not set, then you
-  should provide a mechanism in the [build](#build-hook) hook to auto-determine
-  the version from the source code.
-  WARNING: This parameter will be removed in the near future, as we will rely on
-  the changelog contained in the package's repository to get the package version
-  in the future.
-
-* **DEFAULT_PACKAGE_REVISION**: (Optional) The revision of the package is set to
-  this value when it is built (note that the full version of a package is
-  "_VERSION-REVISION_"). If unset, it defaults to value of environment variable
-  DEFAULT_REVISION.
-  WARNING: This parameter is currently unused and will be removed in the near
-  future.
-
 * **UPSTREAM_SOURCE_PACKAGE**: (Optional) Third-party packages that have an
   [update_upstream](#update-upstream-hook) hook and are updated from an Ubuntu
   source package should set this to the name of the source package.
@@ -644,15 +620,6 @@ use the `install_pkgs()` lib function to install packages.
 Next step is to add a [build()](#build) hook. It is recommended to use the
 `dpkg_buildpackage_default()` function.
 
-Then you'll need to provide the version of the package. For packages created
-from an Ubuntu source package, it is advised to use the same version as was set
-in the source package. For other packages you can use any version you like (e.g.
-`1.0.0`). The version must be provided with:
-
-```
-DEFAULT_PACKAGE_VERSION="<version>"
-```
-
 Note that if you are using an Ubuntu source package, you should now be ready to
 build the package.
 
@@ -717,14 +684,10 @@ In `config.sh`, you'll need to define two variables:
 * `DEFAULT_PACKAGE_GIT_URL`: the `https://` git URL for the source code of the
   package.
 
-* `DEFAULT_PACKAGE_VERSION`: the version of the package. If unsure, just
-  use `1.0.0`.
-
 e.g.:
 
 ```
 DEFAULT_PACKAGE_GIT_URL="https://delphix.gitlab.com/<user>/<package>"
-DEFAULT_PACKAGE_VERSION="1.0.0"
 ```
 
 #### Step 2. Add package hooks

buildpkg.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2018, 2020 Delphix
+# Copyright 2018, 2021 Delphix
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@ logmust check_running_system
 function usage() {
 	[[ $# != 0 ]] && echo "$(basename "$0"): $*"
 	echo "Usage: $(basename "$0") [-ch] [-g pkg_git_url]"
-	echo "         [-b pkg_git_branch] [-v pkg_version] [-r pkg_revision]"
+	echo "         [-b pkg_git_branch] [-r pkg_revision]"
 	echo "         package"
 	echo ""
 	echo "  This script builds a package based on its config.sh. If '-u'"
@@ -36,7 +36,6 @@ function usage() {
 	echo "    -g  override default git url for the package."
 	echo "    -b  override default git branch for the package."
 	echo "    -c  also run package's checkstyle hook."
-	echo "    -v  override default version for package."
 	echo "    -r  override default revision for package."
 	echo "    -h  display this message and exit."
 	echo ""
@@ -45,15 +44,13 @@ function usage() {
 
 unset PARAM_PACKAGE_GIT_URL
 unset PARAM_PACKAGE_GIT_BRANCH
-unset PARAM_PACKAGE_VERSION
 unset PARAM_PACKAGE_REVISION
 
 do_checkstyle=false
-while getopts ':b:cg:hr:v:' c; do
+while getopts ':b:cg:hr:' c; do
 	case "$c" in
 	g) export PARAM_PACKAGE_GIT_URL="$OPTARG" ;;
 	b) export PARAM_PACKAGE_GIT_BRANCH="$OPTARG" ;;
-	v) export PARAM_PACKAGE_VERSION="$OPTARG" ;;
 	r) export PARAM_PACKAGE_REVISION="$OPTARG" ;;
 	c) do_checkstyle=true ;;
 	h) usage >&2 ;;

lib/common.sh

@@ -223,12 +223,9 @@ function reset_package_config_variables() {
 	local vars="
 	PACKAGE_GIT_URL
 	PACKAGE_GIT_BRANCH
-	PACKAGE_GIT_VERSION
-	PACKAGE_GIT_REVISION
+	PACKAGE_VERSION
+	PACKAGE_REVISION
 	DEFAULT_PACKAGE_GIT_URL
-	DEFAULT_PACKAGE_GIT_BRANCH
-	DEFAULT_PACKAGE_GIT_VERSION
-	DEFAULT_PACKAGE_GIT_REVISION
 	PACKAGE_DEPENDENCIES
 	UPSTREAM_SOURCE_PACKAGE
 	UPSTREAM_GIT_URL
@@ -364,7 +361,7 @@ function load_package_config() {
 
 #
 # Use different config sources to determine the values for:
-#   PACKAGE_GIT_URL, PACKAGE_GIT_BRANCH, PACKAGE_VERSION, PACKAGE_REVISION
+#   PACKAGE_GIT_URL, PACKAGE_GIT_BRANCH, PACKAGE_REVISION
 #
 # The sources for the config, in decreasing order of priority, are:
 #   1. Command line parameters passed to build script.
@@ -401,39 +398,20 @@ function get_package_config_from_env() {
 	elif [[ -n "${!var}" ]]; then
 		PACKAGE_GIT_BRANCH="${!var}"
 		echo "PACKAGE_GIT_BRANCH set to value of ${var}"
-	elif [[ -n "$DEFAULT_PACKAGE_GIT_BRANCH" ]]; then
-		PACKAGE_GIT_BRANCH="$DEFAULT_PACKAGE_GIT_BRANCH"
-		echo "PACKAGE_GIT_BRANCH set to value of" \
-			"DEFAULT_PACKAGE_GIT_BRANCH"
 	fi
 
 	if [[ -z "$PACKAGE_GIT_BRANCH" ]]; then
 		PACKAGE_GIT_BRANCH="$DEFAULT_GIT_BRANCH"
 		echo "PACKAGE_GIT_BRANCH set to value of DEFAULT_GIT_BRANCH"
 	fi
 
-	var="${PACKAGE_PREFIX}_VERSION"
-	if [[ -n "$PARAM_PACKAGE_VERSION" ]]; then
-		PACKAGE_VERSION="$PARAM_PACKAGE_VERSION"
-		echo "PACKAGE_VERSION passed from '-v'"
-	elif [[ -n "${!var}" ]]; then
-		PACKAGE_VERSION="${!var}"
-		echo "PACKAGE_VERSION set to value of ${var}"
-	elif [[ -n "$DEFAULT_PACKAGE_VERSION" ]]; then
-		PACKAGE_VERSION="$DEFAULT_PACKAGE_VERSION"
-		echo "PACKAGE_VERSION set to value of DEFAULT_PACKAGE_VERSION"
-	fi
-
 	var="${PACKAGE_PREFIX}_REVISION"
 	if [[ -n "$PARAM_PACKAGE_REVISION" ]]; then
 		PACKAGE_REVISION="$PARAM_PACKAGE_REVISION"
 		echo "PACKAGE_REVISION passed from '-r'"
 	elif [[ -n "${!var}" ]]; then
 		PACKAGE_REVISION="${!var}"
 		echo "PACKAGE_REVISION set to value of ${var}"
-	elif [[ -n "$DEFAULT_PACKAGE_REVISION" ]]; then
-		PACKAGE_REVISION="$DEFAULT_PACKAGE_REVISION"
-		echo "PACKAGE_REVISION set to value of DEFAULT_PACKAGE_REVISION"
 	fi
 
 	if [[ -z "$PACKAGE_REVISION" ]]; then
@@ -443,13 +421,11 @@ function get_package_config_from_env() {
 
 	export PACKAGE_GIT_URL
 	export PACKAGE_GIT_BRANCH
-	export PACKAGE_VERSION
 	export PACKAGE_REVISION
 
 	echo_bold "------------------------------------------------------------"
 	echo_bold "PACKAGE_GIT_URL:      $PACKAGE_GIT_URL"
 	echo_bold "PACKAGE_GIT_BRANCH:   $PACKAGE_GIT_BRANCH"
-	echo_bold "PACKAGE_VERSION:      $PACKAGE_VERSION"
 	echo_bold "PACKAGE_REVISION:     $PACKAGE_REVISION"
 	echo_bold "------------------------------------------------------------"
 }
@@ -971,9 +947,23 @@ function push_to_remote() {
 # If no changelog file exists, source package name can be passed in first arg.
 #
 function set_changelog() {
-	check_env PACKAGE_VERSION PACKAGE_REVISION
+	check_env PACKAGE_REVISION
 	local src_package="${1:-$PACKAGE}"
 
+	#
+	# If PACKAGE_VERSION hasn't been set already, then retrieve it from
+	# The changelog file. If the changelog file doesn't exist, which
+	# could be the case for in-house packages where we do not care about
+	# the version, then default to "1.0.0".
+	#
+	if [[ -z "$PACKAGE_VERSION" ]]; then
+		if [[ -f debian/changelog ]]; then
+			PACKAGE_VERSION="$(logmust dpkg-parsechangelog -S Version | awk -F'-' '{print $1}')"
+		else
+			PACKAGE_VERSION=1.0.0
+		fi
+	fi
+
 	logmust export DEBEMAIL="Delphix Engineering <eng@delphix.com>"
 	if [[ -f debian/changelog ]]; then
 		# update existing changelog

package-lists/build/main.pkgs

@@ -7,14 +7,15 @@ bcc
 bpftrace
 challenge-response
 cloud-init
-crash
 crash-python
 crypt-blowfish
 delphix-platform
 delphix-sso-app
 drgn
+docker-python-image
 gdb-python
 grub2
+host-jdks
 libkdumpfile
 make-jpkg
 makedumpfile

package-lists/update/main.pkgs

@@ -17,7 +17,6 @@
 # shellcheck disable=SC2034
 
 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/bpftrace.git"
-DEFAULT_PACKAGE_VERSION=1.0.0
 PACKAGE_DEPENDENCIES="bcc"
 
 UPSTREAM_GIT_URL="https://github.com/iovisor/bpftrace.git"

packages/bpftrace/config.sh

@@ -17,7 +17,6 @@
 # shellcheck disable=SC2034
 
 DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/os-platform/linux-dlpx-pkgs.git"
-DEFAULT_PACKAGE_VERSION="1.0.0"
 
 function prepare() {
 	logmust install_pkgs \

packages/challenge-response/config.sh

@@ -17,7 +17,6 @@
 # shellcheck disable=SC2034
 
 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/cloud-init.git"
-# Note: we get the package version programatically in build()
 
 UPSTREAM_SOURCE_PACKAGE=cloud-init
 
@@ -31,10 +30,6 @@ function checkstyle() {
 }
 
 function build() {
-	logmust cd "$WORKDIR/repo"
-	if [[ -z "$PACKAGE_VERSION" ]]; then
-		logmust eval PACKAGE_VERSION="$(python3 tools/read-version)"
-	fi
 	logmust dpkg_buildpackage_default
 }
 

packages/cloud-init/config.sh

@@ -17,7 +17,6 @@
 # shellcheck disable=SC2034
 
 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/connstat.git"
-DEFAULT_PACKAGE_VERSION="1.0.0"
 PACKAGE_DEPENDENCIES="@linux-kernel"
 
 function prepare() {