Merge pull request #106 from buildkite-plugins/toote_expand_vars

Expand vars option

Author: mcncl

README.md

@@ -31,7 +31,7 @@ steps:
   - command: ...
     plugins:
     - artifacts#v1.9.4:
-        upload: 
+        upload:
           - from: log1.log
             to: log2.log
 ```
@@ -91,7 +91,7 @@ steps:
   - command: ...
     plugins:
       - artifacts#v1.9.4:
-          download: 
+          download:
             - from: log1.log
               to: log2.log
 ```
@@ -105,7 +105,7 @@ steps:
       - artifacts#v1.9.4:
           step: UUID-DEFAULT
           build: UUID-DEFAULT-2
-          download: 
+          download:
             - from: log1.log
               to: log2.log
               step: UUID-1
@@ -114,25 +114,25 @@ steps:
               build: UUID-2
 ```
 
-## Configuration
+## Mandatory Configuration
 
-### `upload` (string, array of strings, {from,to}, array of {from,to})
-
-A glob pattern, or array of glob patterns, for files to upload.
+You must specify at least one of the following
 
 ### `download` (string, array of strings, {from,to}, array of {from,to[,step][,build]})
 
-A glob pattern, or array of glob patterns, for files to download.
+A glob pattern, or array of glob patterns, for files to download. Alternatively you can specify `from` and `to` to rename the artifact after downloading. If you do so, you can also specify `step` and/or `build` to override those options for that particular artifact.
+
+### `upload` (string, array of strings, {from,to}, array of {from,to})
 
-### `step` (optional, string)
+A glob pattern, or array of glob patterns, for files to upload as-is. Alternatively, you can specify `from` and `to` to rename the artifact before uploading.
 
-The job UUID or name to download the artifacts from unless specified otherwise in the `download` array specification.
+## Other Configuration
 
-### `build` (optional, string)
+### `build` (string, download only)
 
-The build UUID to download the artifact from unless specificed otherwise in the `download` array specification.
+The build UUID to download all artifacts from. Note that you can override it for specific artifacts when using the verbose format of the `download` element.
 
-### `compressed` (optional, string)
+### `compressed` (string)
 
 ⚠️ Limitations:
 * filename needs to end with `.zip` or `.tgz` and that will determine the compression executable to use
@@ -161,11 +161,23 @@ steps:
           compressed: logs.tgz
 ```
 
-### `ignore-missing` (optional, boolean)
+### `expand-download-vars` (boolean, **unsafe**, download only)
+
+When set to true, it will activate interpolation of variables in the elements of the `download` path-related configurations as well as `compressed` option if present. When turned off (the default), attempting to use variables will fail as the literal `$VARIABLE_NAME` string will be used.
+
+⚠️ Important: this is considered an unsafe option as the most compatible way to achieve this is to run the strings through `eval` which could lead to arbitrary code execution or information leaking if you don't have complete control of the pipeline.
+
+### `expand-upload-vars` (boolean, **unsafe**, download only)
+
+When set to true, it will activate interpolation of variables in the elements of the `upload` path-related configurations as well as `compressed` option if present. When turned off (the default), attempting to use variables will fail as the literal `$VARIABLE_NAME` string will be used.
+
+⚠️ Important: this is considered an unsafe option as the most compatible way to achieve this is to run the strings through `eval` which could lead to arbitrary code execution or information leaking if you don't have complete control of the pipeline.
+
+### `ignore-missing` (boolean)
 
 If set to `true`, it will ignore errors caused when calling `buildkite-agent artifact` to prevent failures if you expect artifacts not to be present in some situations. When using the `compressed` property, it will ignore compressing the artifacts that are not present.
 
-### `skip-on-status` (optional, integer or array of integers, uploads only)
+### `skip-on-status` (integer or array of integers, upload only)
 
 You can set this to the exit codes or array of exit codes of the command step (as defined by the `BUILDKITE_COMMAND_EXIT_STATUS` variable) that will cause the plugin to avoid trying to upload artifacts.
 
@@ -195,6 +207,10 @@ steps:
           - 5
 ```
 
+### `step` (string, download only)
+
+The job UUID or name to download all artifacts from. Note that you can override it for specific artifacts when using the verbose format of the `download` element.
+
 ## Developing
 
 To run testing, shellchecks and plugin linting use use `bk run` with the [Buildkite CLI](https://github.com/buildkite/cli).

hooks/post-command

@@ -6,6 +6,19 @@ if [[ "${BUILDKITE_PLUGIN_ARTIFACTS_DEBUG:-false}" =~ (true|on|1) ]] ; then
   set -x
 fi
 
+
+# expand variables in the variable name referenced
+# DANGEROUS due to the use of eval
+# assumes that the environment variable name exists
+expand_vars() {
+  if [[ ${BUILDKITE_PLUGIN_ARTIFACTS_EXPAND_UPLOAD_VARS:-false} != "false" ]]; then
+    # expand the variable
+    eval echo "${!1}"
+  else
+    echo "${!1}"
+  fi
+}
+
 while IFS='=' read -r EXIT_VAR _ ; do
   if [[ $EXIT_VAR =~ ^(BUILDKITE_PLUGIN_ARTIFACTS_SKIP_ON_STATUS(|_[0-9]+))$ ]]; then
     if [ "${BUILDKITE_COMMAND_EXIT_STATUS}" -eq "${!EXIT_VAR}" ]; then
@@ -55,9 +68,9 @@ if [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED:-}" ]]; then
   COMPRESSED="true"
 
   if [[ "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}" =~ .*\.zip ]]; then
-    compress+=("zip" "-r" "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}")
+    compress+=("zip" "-r")
   elif [[ "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}" =~ .*\.tgz ]]; then
-    compress+=("tar" "czf" "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}")
+    compress+=("tar" "czf")
   else
     echo "+++ 🚨 The inferred compression file format for the artifact is not currently supported"
     exit 1
@@ -66,8 +79,8 @@ fi
 
 workdir=${BUILDKITE_PLUGIN_ARTIFACTS_WORKDIR:-.}
 
-pushd "${workdir}"
-trap popd EXIT
+pushd "${workdir}" > /dev/null
+trap "popd > /dev/null" EXIT
 
 bk_agent() {
   if ! buildkite-agent artifact "${@}"; then
@@ -91,11 +104,14 @@ handle_relocation() {
   dest_env_var="BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_${var_string}TO"
 
   if [[ -n "${!source_env_var:-}" ]] && [[ -n "${!dest_env_var:-}" ]]; then
-    if [[ -e ${!source_env_var} ]]; then
-      echo "~~~ Moving [${!source_env_var}] to [${!dest_env_var}]..."
-      mv "${!source_env_var}" "${!dest_env_var}"
+    SOURCE_FILE="$(expand_vars "${source_env_var}")"
+    DEST_FILE="$(expand_vars "${dest_env_var}")"
+
+    if [[ -e "${SOURCE_FILE}" ]]; then
+      echo "~~~ Moving [${SOURCE_FILE}] to [${DEST_FILE}]..."
+      mv "${SOURCE_FILE}" "${DEST_FILE}"
     elif [[ ${BUILDKITE_PLUGIN_ARTIFACTS_IGNORE_MISSING:-"false"} == "true" ]]; then
-      echo "Ignoring missing file ${!source_env_var} for relocation"
+      echo "Ignoring missing file ${SOURCE_FILE} for relocation"
     fi
   fi
 }
@@ -121,42 +137,49 @@ fi
 if [[ "${SINGULAR_UPLOAD_OBJECT}" == "true" ]]; then
   if [[ "${RELOCATION}" == "true" ]]; then
     handle_relocation ""
-    path="${BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_TO}"
+    path="$(expand_vars BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_TO)"
   else
-    path="${paths[0]}"
+    export FILENAME="${paths[0]}"
+    path="$(expand_vars FILENAME)"
   fi
 
   if [[ "${COMPRESSED}" == "true" ]]; then
-    echo "~~~ Compressing ${path} to ${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}"
+    COMPRESSED_FILENAME="$(expand_vars BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED)"
+    echo "~~~ Compressing ${path} to ${COMPRESSED_FILENAME}"
     if [[ ! -e "${path}" ]]; then
       echo "+++ 🚨 Unable to compress artifact, '${path}' may not exist or is an empty directory"
     else
-      "${compress[@]}" "${path}"
-      path=" ${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}"
+      "${compress[@]}" "${COMPRESSED_FILENAME}" "${path}"
+      path="${COMPRESSED_FILENAME}"
     fi
   fi
 
   echo "~~~ Uploading artifacts ${EXTRA_MESSAGE}"
   bk_agent "${args[@]}" "${path}"
 elif [[ "${COMPRESSED}" == "true" ]]; then
+  COMPRESSED_FILENAME="$(expand_vars BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED)"
   final_paths=()
   index=0
   for path in "${paths[@]}"; do
     handle_relocation "${index}"
     dest_env_var="BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_${index}_TO"
     if [[ -n "${!dest_env_var:-}" ]]; then
-      path="${!dest_env_var}"
+      path="$(expand_vars "${dest_env_var}")"
+    else
+      # could use $path directly, but would break the ordering because 0,1,10,11,2,20,3 :(
+      # may be solved by using sort -V but it is GNU-specific
+      path="$(expand_vars "BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_${index}")"
     fi
 
     final_paths+=("$path")
     ((index+=1))
   done
 
-  echo "~~~ Compressing ${final_paths[*]} to ${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}"
+  echo "~~~ Compressing ${final_paths[*]} to ${COMPRESSED_FILENAME}"
   "${compress[@]}" "${final_paths[@]}"
 
   echo "~~~ Uploading artifacts ${EXTRA_MESSAGE}"
-  bk_agent "${args[@]}" "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}"
+  bk_agent "${args[@]}" "${COMPRESSED_FILENAME}"
 elif [[ "${MULTIPLE_UPLOADS}" == "true" ]]; then
   index=0
   echo "~~~ Uploading artifacts ${EXTRA_MESSAGE}"
@@ -165,7 +188,9 @@ elif [[ "${MULTIPLE_UPLOADS}" == "true" ]]; then
     handle_relocation "${index}"
     dest_env_var="BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_${index}_TO"
     if [[ -n "${!dest_env_var:-}" ]]; then
-      path="${!dest_env_var}"
+      path="$(expand_vars "${dest_env_var}")"
+    else
+      path="$(expand_vars path)"
     fi
 
     bk_agent "${args[@]}" "$path"

hooks/pre-command

@@ -14,23 +14,33 @@ compress=()
 
 COMPRESSED="false"
 SINGULAR_DOWNLOAD_OBJECT="false"
-RELOCATION="false"
 MULTIPLE_DOWNLOADS="false"
 
+# expand variables in the variable name referenced
+# DANGEROUS due to the use of eval
+# assumes that the environment variable name exists
+expand_vars() {
+  if [[ ${BUILDKITE_PLUGIN_ARTIFACTS_EXPAND_DOWNLOAD_VARS:-false} != "false" ]]; then
+    # expand the variable
+    eval echo "${!1}"
+  else
+    echo "${!1}"
+  fi
+}
+
 if [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD:-}" ]] || { [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_FROM:-}" ]] && [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO:-}" ]]; }; then
     SINGULAR_DOWNLOAD_OBJECT="true"
     if [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD:-}" ]] ; then
-      paths+=("$BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD")
+      paths+=("$(expand_vars BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD)")
     elif [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_FROM:-}" ]] && [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO:-}" ]] ; then
-      RELOCATION="true"
-      paths+=("$BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_FROM")
+      paths+=("$(expand_vars BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_FROM)")
     fi
 fi
 
 while IFS='=' read -r path _ ; do
   if [[ $path =~ ^(BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_[0-9]+)$ ]] || [[ $path =~ ^(BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_[0-9]+_FROM)$ ]]; then
     MULTIPLE_DOWNLOADS="true"
-    paths+=("${!path}")
+    paths+=("$(expand_vars "${path}")")
   fi
 done < <(env | sort)
 
@@ -87,25 +97,28 @@ handle_relocation() {
   source_env_var="BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_${var_string}FROM"
   dest_env_var="BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_${var_string}TO"
   if [[ -n "${!dest_env_var:-}" ]]; then
-    if ! [[ -d $(dirname "${!dest_env_var}") ]]; then
-      mkdir -p "$(dirname "${!dest_env_var}")"
+    SOURCE_FILE="$(expand_vars "${source_env_var}")"
+    DEST_FILE="$(expand_vars "${dest_env_var}")"
+    if ! [[ -d "$(dirname "${DEST_FILE}")" ]]; then
+      mkdir -p "$(dirname "${DEST_FILE}")"
     fi
 
-    if [[ ! -e "${!source_env_var}" ]] && [[ ${BUILDKITE_PLUGIN_ARTIFACTS_IGNORE_MISSING:-"false"} != "false" ]]; then
-      echo "Ignoring missing file ${!source_env_var} for relocation"
+    if [[ ! -e "${SOURCE_FILE}" ]] && [[ ${BUILDKITE_PLUGIN_ARTIFACTS_IGNORE_MISSING:-"false"} != "false" ]]; then
+      echo "Ignoring missing file ${SOURCE_FILE} for relocation"
     else
-      echo "~~~ Moving [${!source_env_var}] to [${!dest_env_var}]..."
-      mv "${!source_env_var}" "${!dest_env_var}"
+      echo "~~~ Moving [${SOURCE_FILE}] to [${DEST_FILE}]..."
+      mv "${SOURCE_FILE}" "${DEST_FILE}"
     fi
   fi
 }
 
 if [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED:-}" ]]; then
   COMPRESSED="true"
-  if [[ "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}" =~ .*\.zip ]]; then
-    compress+=("unzip" "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}")
-  elif [[ "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}" =~ .*\.tgz ]]; then
-    compress+=("tar" "xzf" "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}")
+  COMPRESSED_FILENAME="$(expand_vars BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED)"
+  if [[ "${COMPRESSED_FILENAME}" =~ .*\.zip ]]; then
+    compress+=("unzip" "${COMPRESSED_FILENAME}")
+  elif [[ "${COMPRESSED_FILENAME}" =~ .*\.tgz ]]; then
+    compress+=("tar" "xzf" "${COMPRESSED_FILENAME}")
   else
     echo "+++ 🚨 The inferred compression file format for the artifact is not currently supported"
     exit 1
@@ -115,16 +128,14 @@ fi
 workdir="${BUILDKITE_PLUGIN_ARTIFACTS_WORKDIR:-.}"
 
 if [[ "${COMPRESSED}" == "true" ]]; then
-  if ! bk_agent "${step_option}" "${build_option}" "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}" "${workdir}"; then
-    handle_bk_error "${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}"
+  if ! bk_agent "${step_option}" "${build_option}" "${COMPRESSED_FILENAME}" "${workdir}"; then
+    handle_bk_error "${COMPRESSED_FILENAME}"
   else
-    echo "~~~ Uncompressing ${paths[*]} from ${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}"
+    echo "~~~ Uncompressing ${paths[*]} from ${COMPRESSED_FILENAME}"
     "${compress[@]}" "${paths[@]}"
 
     # single relocation
-    if [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO:-}" ]]; then
-      handle_relocation ""
-    fi
+    handle_relocation ""
 
     # multiple relocations
     index=0
@@ -135,35 +146,32 @@ if [[ "${COMPRESSED}" == "true" ]]; then
   fi
 
 elif [[ "${SINGULAR_DOWNLOAD_OBJECT}" == "true" ]]; then
-  if [[ "${RELOCATION}" == "true" ]]; then
-    source="${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_FROM}"
-  else
-    source="${paths[*]}"
-  fi
+  source="${paths[*]}"
 
   if ! bk_agent "${step_option}" "${build_option}" "${source}" "${workdir}"; then
     handle_bk_error "${source}"
   else
-    if [[ -n "${BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_TO:-}" ]]; then
-      handle_relocation ""
-    fi
+    handle_relocation ""
   fi
 elif [[ "${MULTIPLE_DOWNLOADS}" == "true" ]]; then
   index=0
 
   for path in "${paths[@]}"; do
-    source_env_var="BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_${index}_FROM"
-    dest_env_var="BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_${index}_TO"
-    step_env_var="BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_${index}_STEP"
-    build_env_var="BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_${index}_BUILD"
-
-    # finally get the artifact to download
-    if [ -n "${!source_env_var:-}" ] && [ -n "${!dest_env_var:-}" ]; then
-      source="${!source_env_var}"
+    env_var_base="BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_${index}"
+    dest_env_var="${env_var_base}_TO"
+    step_env_var="${env_var_base}_STEP"
+    build_env_var="${env_var_base}_BUILD"
+
+    # could use $path directly, but would break the ordering because 0,1,10,11,2,20,3 :(
+    # may be solved by using sort -V but it is GNU-specific
+    if [ -z "${!env_var_base:-}" ]; then
+      source_env_var="${env_var_base}_FROM"
     else
-      source="${path}"
+      source_env_var="${env_var_base}"
     fi
 
+    source="$(expand_vars "${source_env_var}")"
+
     if ! bk_agent "${!step_env_var:-${step_option}}" "${!build_env_var:-${build_option}}" "${source}" "${workdir}"; then
       handle_bk_error "${source}"
     else

plugin.yml

@@ -58,11 +58,17 @@ configuration:
         - type: array
           items:
             type: integer
+    expand-download-vars:
+      type: boolean
+    expand-upload-vars:
+      type: boolean
   oneOf:
     - required:
       - upload
     - required:
       - download
   dependencies:
     skip-on-status: [ upload ]
+    expand-download-vars: [ download ]
+    expand-upload-vars: [ upload ]
   additionalProperties: false