diff --git a/README.md b/README.md index b43025e9..d7fffebe 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Git Resource -Tracks the commits in a [git](http://git-scm.com/) repository. +Tracks commits, tags, or branches in a [git](http://git-scm.com/) repository. Build Status @@ -15,20 +15,27 @@ Tracks the commits in a [git](http://git-scm.com/) repository. Description - uri (Required) - The location of the repository. - - - branch (Optional) + version_type (Optional) - The branch to track. This is optional if the resource is only used in - get steps; however, it is required when used in a - put step. If unset, get steps will checkout - the repository's default branch; usually master but could - be different. + + + +The following fields are used by all `version_type`'s. + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + +
Field NameDescription
uri (Required)The location of the repository.
private_key (Optional) Private key to use when using an ssh@ format uri. Example: @@ -60,107 +67,10 @@ private_key: | password (Optional) Password for HTTP(S) auth when pulling/pushing.
paths (Optional) - If specified (as a list of glob patterns), only changes to the specified files will yield new versions from check. - Example: - 
-- name: repo
-  type: git
-  source:
-    paths:
-      - some-folder/*
-      - another/folder/path/*
-
-
sparse_paths (Optional) - If specified (as a list of glob patterns), only these paths will be - checked out. Should be used with paths to only trigger on - desired paths. paths and sparse_paths may be - the same or you can configure sparse_paths to check out - other paths. - Example: - 
-- name: repo
-  type: git
-  source:
-    paths:
-      - some-folder/*
-      - another/folder/path/*
-    sparse_paths:
-      - some-folder/*
-      - another/folder/path/*
-
-
ignore_paths (Optional) - The inverse of paths; changes to the specified files are - ignored.

Note that if you want to push commits that change these - files via a put, the commit will still be "detected", as check - and put both introduce versions. To avoid this you - should define a second resource that you use for commits that change - files that you don't want to feed back into your pipeline - think of one - as read-only (with ignore_paths) and one as write-only - (which shouldn't need it).

- Example: - 
-- name: repo
-  type: git
-  source:
-    ignore_paths:
-      - some-folder/*
-      - another/folder/path/*
-
-
skip_ssl_verification (Optional) Skips git ssl verification by exporting GIT_SSL_NO_VERIFY=true.
tag_filter (Optional) - If specified, the resource will only detect commits that have a tag - matching the expression that have been made against the - branch. Patterns are glob(7) - compatible (as in, bash compatible). -
tag_regex (Optional) - If specified, the resource will only detect commits that have a tag - matching the expression that have been made against the - branch. Patterns are grep - compatible (extended matching enabled, matches entire lines only). - Ignored if tag_filter is also specified. -
tag_behaviour (Optional) - If match_tagged (the default), then the resource will only - detect commits that are tagged with a tag matching - tag_regex and tag_filter, and match all other - filters. If match_tag_ancestors, then the resource will - only detect commits matching all other filters and that are ancestors of - a commit that are tagged with a tag matching tag_regex and - tag_filter. -
fetch_tags (Optional)If true the flag --tags will be used to fetch all tags in the repository. If false no tags will be fetched.
submodule_credentials (Optional) List of credentials for HTTP(s) or SSH auth when pulling git submodules which are not stored in the same git server as the container repository or are protected by a different private key. @@ -261,6 +171,136 @@ git_config:
debug (Optional) + Set to true to enable. Sets the following for check/get/put + steps of the resource. Secrets may not be correctly redacted due the + JSON encoding of longer secret strings. + 
+set -x
+export GIT_TRACE=1
+export GIT_TRACE_PACKFILE=1
+export GIT_CURL_VERBOSE=1
+
+
+ +The following fields are used exclusively by `version_type: commits`. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
branch (Optional) + The branch to track. This is optional if the resource is only used in + get steps; however, it is required when used in a + put step. If unset, get steps will checkout + the repository's default branch; usually master but could + be different. +
paths (Optional) + If specified (as a list of glob patterns), only changes to the specified files will yield new versions from check. + Example: + 
+- name: repo
+  type: git
+  source:
+    paths:
+      - some-folder/*
+      - another/folder/path/*
+
+
sparse_paths (Optional) + If specified (as a list of glob patterns), only these paths will be + checked out. Should be used with paths to only trigger on + desired paths. paths and sparse_paths may be + the same or you can configure sparse_paths to check out + other paths. + Example: + 
+- name: repo
+  type: git
+  source:
+    paths:
+      - some-folder/*
+      - another/folder/path/*
+    sparse_paths:
+      - some-folder/*
+      - another/folder/path/*
+
+
ignore_paths (Optional) + The inverse of paths; changes to the specified files are + ignored.

Note that if you want to push commits that change these + files via a put, the commit will still be "detected", as check + and put both introduce versions. To avoid this you + should define a second resource that you use for commits that change + files that you don't want to feed back into your pipeline - think of one + as read-only (with ignore_paths) and one as write-only + (which shouldn't need it).

+ Example: + 
+- name: repo
+  type: git
+  source:
+    ignore_paths:
+      - some-folder/*
+      - another/folder/path/*
+
+
tag_filter (Optional) + If specified, the resource will only detect commits that have a tag + matching the expression that have been made against the + branch. Patterns are glob(7) + compatible (as in, bash compatible). +
tag_regex (Optional) + If specified, the resource will only detect commits that have a tag + matching the expression that have been made against the + branch. Patterns are grep + compatible (extended matching enabled, matches entire lines only). + Ignored if tag_filter is also specified. +
tag_behaviour (Optional) + If match_tagged (the default), then the resource will only + detect commits that are tagged with a tag matching + tag_regex and tag_filter, and match all other + filters. If match_tag_ancestors, then the resource will + only detect commits matching all other filters and that are ancestors of + a commit that are tagged with a tag matching tag_regex and + tag_filter. +
fetch_tags (Optional)If true the flag --tags will be used to fetch all tags in the repository. If false no tags will be fetched.
commit_filter (Optional) Object containing commit message filters @@ -294,18 +334,64 @@ commit_filter: usually create. See also out params.refs_prefix.
+ +The following fields are used exclusively by `version_type: tags`. + - + + + + + + + + + + + + + + +
debug (Optional)Field NameDescription
tag_filters (Optional) - Set to true to enable. Sets the following for check/get/put - steps of the resource. Secrets may not be correctly redacted due the - JSON encoding of longer secret strings. - 
-set -x
-export GIT_TRACE=1
-export GIT_TRACE_PACKFILE=1
-export GIT_CURL_VERBOSE=1
-
+ A list of glob patterns used to filter tags. Only matching tags will be + returned. Patterns are glob(7) + compatible (as in, bash compatible). If you're only specifying one glob + pattern you can use tag_filter. +
tag_regex (Optional) + Regex pattern used to filter tags. Only matching tags will be returned. + Patterns are grep + compatible (extended matching enabled). + Ignored if tag_filter(s) is also specified. +
tag_sort (Optional) Sorting is applied after filtering. Accepts the following values: +
    +
  • creatordate (Default): Uses git's built-in sorting by the creation date of the tag
    • +
  • semver: Uses sort -V to sort all matching tags.
    • +
+
+ +The following fields are used exclusively by `version_type: branches`. + + + + + + + + + + + +
Field NameDescription
branch_filters (Optional) + A list of glob patterns used to filter branches. Only matching branches will be returned. + Patterns are glob(7) + compatible (as in, bash compatible). +
branch_regex (Optional) + Regex pattern used to filter branches. Only matching branches will be returned. + Patterns are grep + compatible (extended matching enabled). + Ignored if branch_filters is also specified.
@@ -386,6 +472,13 @@ resources: ## Behavior +The behavior of the resource changes based on the specified `version_type`. +Expand the relevant section to learn more about how the `check/get/put` steps +work for each `version_type`. + +
+ version_type: commits + ### `check`: Check for new commits The repository is cloned (or pulled if already present), and any commits @@ -395,15 +488,15 @@ for `HEAD` is returned. Any commits that contain the string `[ci skip]` will be ignored. This allows you to commit to your repository without triggering a new version. -### `in`: Clone the repository, at the given ref +### `get`: Clone the repository, at the given ref Clones the repository to the destination, and locks it down to a given ref. It will return the same given ref as version. `git-crypt` encrypted repositories will automatically be decrypted, when the -correct key is provided set in `git_crypt_key`. +correct key is provided set in `source.git_crypt_key`. -#### Parameters +#### `get` Parameters @@ -576,9 +669,8 @@ the case. * `.git/author_date`: Timestamp when the author originally created the commit. * `.git/committer`: For committer notification on failed builds. This special file `.git/committer` which is populated - with the email address of the author of the last commit. This can be used together with an email resource - like [mdomke/concourse-email-resource](https://github.com/mdomke/concourse-email-resource) to notify the committer in - an on_failure step. + with the email address of the author of the last commit. This can be used + together with a resource to send notifications in an `on_failure` step. * `.git/committer_name`: Name of the commit author. @@ -597,7 +689,7 @@ the case. * `.git/metadata.json`: Complete metadata object in JSON format containing all metadata fields. -### `out`: Push to a repository +### `put`: Push to a repository Push the checked-out reference to the source's URI and branch. All tags are also pushed to the source. If a fast-forward for the branch is not possible @@ -717,6 +809,160 @@ export GIT_CURL_VERBOSE=1
+
+ +
+ version_type: tags + +### `check`: Check for new tags + +Checks for new tags, filtering by `tag_filters` or `tag_regex` if specified. +Each version emitted will be the tag and the ref it points to. + +### `get`: Clone the repository at the given tag + +Shallow clones the repository to the destination, checking out the given +tag in a detached HEAD state. + +`git-crypt` encrypted repositories will automatically be decrypted, when the +correct key is provided set in `source.git_crypt_key`. + +#### `get` Parameters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
submodules
Optional		 + If none, submodules will not be fetched. If specified as a + list of paths, only the given paths will be fetched. If not specified, + or if all is explicitly specified, all submodules are + fetched. +
submodule_recursive
Optional		If false, a flat submodules checkout is performed. If not specified, or if true is explicitly specified, a recursive checkout is performed.
submodule_remote
Optional		 + If true, the submodules are checked out for the specified + remote branch specified in the .gitmodules file of the + repository. If not specified, or if false is explicitly + specified, the tracked sub-module revision of the repository is used to + check out the submodules. +
disable_git_lfs
Optional		If true, will not fetch Git LFS files.
short_ref_format
Optional		When populating .git/short_ref use this printf format. Defaults to %s.
timestamp_format
Optional		 + When populating .git/commit_timestamp use this options to + pass to git + log --date. Defaults to iso8601. +
debug (Optional) + Set to true to enable. Sets the following for check/get/put + steps of the resource. Secrets may not be correctly redacted due the + JSON encoding of longer secret strings. + 
+set -x
+export GIT_TRACE=1
+export GIT_TRACE_PACKFILE=1
+export GIT_CURL_VERBOSE=1
+
+
+ +#### GPG signature verification + +If `commit_verification_keys` or `commit_verification_key_ids` is specified in +the source configuration, it will additionally verify that the resulting commit +has been GPG signed by one of the specified keys. It will error if this is not +the case. + +#### Additional files populated + +* `.git/tag`: Tag detected and checked out. + +* `.git/ref`: Full SHA-1 commit hash. + +* `.git/short_ref`: Short (first seven characters) of the `.git/ref`. Can be templated with `short_ref_format` + parameter. + +* `.git/author`: Commit author name. + +* `.git/author_date`: Timestamp when the author originally created the commit. + +* `.git/committer`: For committer notification on failed builds. This special file `.git/committer` which is populated + with the email address of the author of the last commit. This can be used + together with a resource to send notifications in an `on_failure` step. + +* `.git/committer_name`: Name of the commit author. + +* `.git/committer_date`: Timestamp when the commit was added to the repository. + +* `.git/commit_message`: For publishing the Git commit message on successful builds. + +* `.git/commit_timestamp`: For tagging builds with a timestamp. + +* `.git/url`: Web URL to view the commit (if applicable). + +* `.git/metadata.json`: Complete metadata object in JSON format containing all metadata fields. + +### `put`: No-op + +There is no implementation for this step. It will error if you try to use it. If +you want to push new tags, use `version_type: commits`. + +
+ +
+ version_type: branches + +### `check`: Check for new branches + +The list of remote branches are enumerated, filtered by `branch_filters` or +`branch_regex` if specified, and compared to the existing set of branches. If +any branches are new or removed, a new version is emitted. Branches are sorted +lexicographically using `sort` before comparing. + +If no branches are found a special `NONE` version is emitted. + +### `get`: List the given branches + +Produces a `branches.json` file containing a JSON array of the branches from the +given version. Example of the contents of the file: +```json +[ + "feature/add-button", + "feature/refactor-model", + "fix/ui-bug" +] +``` + +### `put`: No-op + +There is no implementation for this step. It will error if you try to use it. + +
+ ## Development ### Prerequisites diff --git a/assets/check b/assets/check index 282e6dfc..58acfd87 100755 --- a/assets/check +++ b/assets/check @@ -1,19 +1,16 @@ #!/bin/bash -# vim: set ft=sh set -e exec 3>&1 # make stdout available as fd 3 for the result exec 1>&2 # redirect all output to stderr for logging -source $(dirname $0)/common.sh - -# for jq -PATH=/usr/local/bin:$PATH +assets="$(dirname $0)" +source "${assets}/common.sh" payload="$(cat <&0)" -unknown_keys=$(jq --slurpfile schema "$(dirname $0)/source_schema.json" '(.source // [] | keys_unsorted) - ($schema[0] | keys_unsorted)' <<< "$payload") +unknown_keys=$(jq --slurpfile schema "${assets}/source_schema.json" '(.source // [] | keys_unsorted) - ($schema[0] | keys_unsorted)' <<< "$payload") if jq --exit-status 'length > 0' <<< "$unknown_keys" &>/dev/null; then echo "Found unknown keys in source:" @@ -30,312 +27,36 @@ if [[ "$debug" == "true" ]]; then export GIT_CURL_VERBOSE=1 fi +git_config_payload=$(jq -r '.source.git_config // []' <<< "$payload") +configure_git_global "${git_config_payload}" load_pubkey "$payload" configure_https_tunnel "$payload" configure_git_ssl_verification "$payload" configure_credentials "$payload" +# These vars are used by multiple version_type's +destination=$TMPDIR/git-resource-repo-cache uri=$(jq -r '.source.uri // ""' <<< "$payload") uri=${uri# } -branch=$(jq -r '.source.branch // ""' <<< "$payload") -branch=${branch# } -paths="$(jq -r '(.source.paths // ["."])[]' <<< "$payload")" # those "'s are important -ignore_paths="$(jq -r '":!" + (.source.ignore_paths // [])[]' <<< "$payload")" # these ones too -tag_filter=$(jq -r '.source.tag_filter // ""' <<< "$payload") -tag_filter=${tag_filter# } -tag_regex=$(jq -r '.source.tag_regex // ""' <<< "$payload") -tag_regex=${tag_regex# } -tag_behaviour=$(jq -r '.source.tag_behaviour // "match_tagged"' <<< "$payload") -git_config_payload=$(jq -r '.source.git_config // []' <<< "$payload") -ref=$(jq -r '.version.ref // ""' <<< "$payload") -skip_ci_disabled=$(jq -r '.source.disable_ci_skip // false' <<< "$payload") -filter_include=$(jq '.source.commit_filter.include // []' <<< "$payload") -filter_include_all_match=$(jq -r '.source.commit_filter.include_all_match // false' <<< "$payload") -filter_exclude=$(jq '.source.commit_filter.exclude // []' <<< "$payload") -filter_exclude_all_match=$(jq -r '.source.commit_filter.exclude_all_match // false' <<< "$payload") -version_depth=$(jq -r '.source.version_depth // 1' <<< "$payload") -reverse=false - if [[ -z "$uri" ]]; then echo "source.uri is required and must not be empty" exit 1 fi -configure_git_global "${git_config_payload}" - -destination=$TMPDIR/git-resource-repo-cache - -# Optimization when last commit only is checked and skip ci is disabled -# Get the commit id with git ls-remote instead of downloading the whole repo -if [ "$skip_ci_disabled" = "true" ] && \ - [ "$version_depth" = "1" ] && \ - [ "$paths" = "." ] && \ - [ -z "$ignore_paths" ] && \ - [ -z "$tag_filter" ] && \ - [ -z "$tag_regex" ] && \ - jq -e 'length == 0' <<<"$filter_include" &>/dev/null && \ - jq -e 'length == 0' <<<"$filter_exclude" &>/dev/null -then - branchflag="HEAD" - if [ -n "$branch" ]; then - branchflag="$branch" - fi - commit=$(git ls-remote $uri $branchflag | awk 'NR<=1{print $1}') - if [ -z "$commit" ]; then - echo "No commit returned. Invalid branch?" - exit 1 - fi - if [ -z "$ref" ] || [ "$ref" = "$commit" ]; then - echo $commit | jq -R '.' | jq -s "map({ref: .})" >&3 - exit 0 - fi -fi - -tagflag="" -if [ -n "$tag_filter" ] && [ -n "$tag_regex" ] ; then - echo "Cannot provide both tag_filter and tag_regex" - exit 1 -elif [ -n "$tag_filter" ] || [ -n "$tag_regex" ] ; then - tagflag="--tags" -else - tagflag="--no-tags" -fi - -if [ "$tag_behaviour" != "match_tagged" ] && [ "$tag_behaviour" != "match_tag_ancestors" ]; then - echo "Invalid tag_behaviour. Must be one of 'match_tagged' or 'match_tag_ancestors'." - exit 1 -fi - -for filter in "$filter_include" "$filter_exclude" -do - if jq -e 'type != "array"' <<<"$filter" &>/dev/null - then - echo 'invalid commit filter (expected array of strings)' - echo "$filter" - exit 1 - fi -done - -if [ "$version_depth" -le 0 ]; then - echo "Invalid version_depth. Must be <= 0." - exit 1 -fi - -# We're just checking for commits; we don't ever need to fetch LFS files here! -export GIT_LFS_SKIP_SMUDGE=1 - -if [ -d $destination ]; then - cd $destination - if [ "$tagflag" = "--tags" ]; then - git fetch origin 'refs/heads/*:refs/heads/*' --tags -f - else - git fetch origin $tagflag $branch -f - git reset --soft FETCH_HEAD - fi -else - branchflag="" - if [ -n "$branch" ]; then - branchflag="--branch $branch" - fi - - # Determine optimal filter based on whether path filtering is used - if [ "$paths" = "." ] && [ -z "$ignore_paths" ]; then - # No path filtering - use most aggressive filter for maximum performance - filter_flag="--filter=tree:0" - else - # Path filtering needed - keep trees to analyze paths, skip blobs - filter_flag="--filter=blob:none" - fi - - if [ "$tagflag" = "--tags" ]; then - # For tag filtering, don't use --single-branch - tags may be on any branch - git clone --bare $filter_flag --progress $uri $destination --tags - else - git clone --bare $filter_flag --single-branch --progress $uri $branchflag $destination $tagflag - fi - cd $destination - # bare clones don't configure the refspec - if [ -n "$branch" ]; then - git remote set-branches --add origin $branch - fi -fi - -if [ -n "$ref" ] && git cat-file -e "$ref"; then - reverse=true - log_range="${ref}~1..HEAD" - - # if ${ref} does not have parents, ${ref}~1 raises the error: "unknown revision or path not in the working tree" - # the initial commit in a branch will never have parents, but rarely, subsequent commits can also be parentless - orphan_commits=$(git rev-list --max-parents=0 HEAD) - for orphan_commit in ${orphan_commits}; do - if [ "${ref}" = "${orphan_commit}" ]; then - log_range="HEAD" - break - fi - done -else - log_range="" - ref="" -fi - -if [ "$paths" = "." ] && [ -z "$ignore_paths" ]; then - paths_search="" -else - paths_search=`echo "-- $paths $ignore_paths" | tr "\n\r" " "` -fi - -list_command="git rev-list --all --first-parent $log_range $paths_search" -if jq -e 'length > 0' <<<"$filter_include" &>/dev/null -then - list_command+=" | git rev-list --stdin --date-order --first-parent --no-walk=unsorted " - include_items=$(echo $filter_include | jq -r -c '.[]') - for wli in "$include_items" - do - list_command+=" --grep=\"$wli\"" - done - if [ "$filter_include_all_match" == "true" ]; then - list_command+=" --all-match" - fi -fi - -if jq -e 'length > 0' <<<"$filter_exclude" &>/dev/null -then - list_command+=" | git rev-list --stdin --date-order --invert-grep --first-parent --no-walk=unsorted " - exclude_items=$(echo $filter_exclude | jq -r -c '.[]') - for bli in "$exclude_items" - do - list_command+=" --grep=\"$bli\"" - done - if [ "$filter_exclude_all_match" == "true" ]; then - list_command+=" --all-match" - fi -fi - - -if [ "$skip_ci_disabled" != "true" ]; then - list_command+=" | git rev-list --stdin --date-order --grep=\"\\[ci\\sskip\\]\" --grep=\"\\[skip\\sci\\]\" --invert-grep --first-parent --no-walk=unsorted" -fi - -replace_escape_chars() { - sed -e 's/[]\/$*.^[]/\\&/g' <<< $1 -} - -lines_including_and_after() { - local escaped_string=$(replace_escape_chars $1) - sed -ne "/$escaped_string/,$ p" -} - -#if no range is selected just grab the last commit that fits the filter -if [ -z "$log_range" ] && [ -z "$tag_filter" ] && [ -z "$tag_regex" ] -then - list_command+="| git rev-list --stdin --date-order --no-walk=unsorted -$version_depth --reverse" -fi - -if [ "$reverse" == "true" ] && [ -z "$tag_filter" ] && [ -z "$tag_regex" ] -then - list_command+="| git rev-list --stdin --date-order --first-parent --no-walk=unsorted --reverse" -fi - -get_tags_matching_filter() { - local list_command=$1 - local tags=$2 - for tag in $tags; do - # We turn the tag ref (e.g. v1.0.0) into the object name - # (e.g. 1a410efbd13591db07496601ebc7a059dd55cfe9) and use grep to check it is in the output - # of list_command - if it isn't, it doesn't pass one of the other filters and shouldn't be - # outputted. - local commit=$(git rev-list -n 1 $tag) - local this_list_command="$list_command | grep -cFx \"$commit\"" - local list_output="$(set -f; eval "$this_list_command"; set +f)" - if [ "$list_output" -ge 1 ]; then - jq -cn '{ref: $tag, commit: $commit}' --arg tag $tag --arg commit $commit - fi - done -} - -get_tags_match_ancestors_filter() { - local list_command=$1 - local tags=$2 - - # Sort commits so that we look at the oldest commits first - local this_list_command="$list_command | git rev-list --stdin --date-order --first-parent --no-walk=unsorted --reverse" - local list_output="$(set -f; eval "$this_list_command"; set +f)" - - # Store all the tag names in an associative array for quick lookups. - # Also gather the commits each tag is attached to so we can quickly match - # candidate commits in a best-case scenario. - declare -A eligible_tag_names=() - declare -A eligible_tag_commits=() - local tag tag_commit - for tag in $tags; do - eligible_tag_names["$tag"]=1 - tag_commit=$(git rev-list -n 1 "$tag" 2>/dev/null || true) - if [ -n "$tag_commit" ]; then - eligible_tag_commits["$tag_commit"]=1 - fi - done - - # Check each commit and only output commits that are ancestors of tags. - for commit in $list_output; do - local is_ancestor=false - - # Fast path: check if the commit itself is the target of an eligible tag - if [ -n "${eligible_tag_commits[$commit]+x}" ]; then - is_ancestor=true - else - # Find all the tags whose history contains this commit - then check if - # any are eligible tags - local containing_tag - while IFS= read -r containing_tag; do - [ -z "$containing_tag" ] && continue - if [ -n "${eligible_tag_names[$containing_tag]+x}" ]; then - is_ancestor=true - break - fi - done < <(git tag --contains "$commit") - fi - - if [ "$is_ancestor" = true ]; then - jq -cn '{ref: $commit}' --arg commit $commit - fi - done -} - -if [ -n "$tag_filter" ] || [ -n "$tag_regex" ]; then - # Create a suffix to "git tag" that will apply the tag filter - if [ -n "$tag_filter" ]; then - tag_filter_cmd="--list \"$tag_filter\"" - elif [ -n "$tag_regex" ]; then - tag_filter_cmd="| grep -Ex \"$tag_regex\"" - fi - - # Build a list of tag refs (e.g. v1.0.0) that match the filter - if [ -n "$ref" ] && [ -n "$branch" ]; then - tags=$(set -f; eval "git tag --sort=creatordate --contains $ref --merged $branch $tag_filter_cmd"; set +f) - elif [ -n "$ref" ]; then - tags=$(set -f; eval "git tag --sort=creatordate $tag_filter_cmd | lines_including_and_after $ref"; set +f) - else - branch_flag= - if [ -n "$branch" ]; then - branch_flag="--merged $branch" - fi - tags=$(set -f; eval "git tag --sort=creatordate $branch_flag $tag_filter_cmd"; set +f) - fi - - # Only proceed if we actually found any tags - if [ -n "$tags" ]; then - if [ "$tag_behaviour" == "match_tagged" ]; then - get_tags_matching_filter "$list_command" "$tags" | tail "-$version_depth" | jq -s "map(.)" >&3 - else - get_tags_match_ancestors_filter "$list_command" "$tags" | tail "-$version_depth" | jq -s "map(.)" >&3 - fi - else - jq -n "[]" >&3 - fi -else - { - set -f - eval "$list_command" - set +f - } | jq -R '.' | jq -s "map({ref: .})" >&3 -fi +version_type=$(jq -r '.source.version_type // "commits"' <<< "$payload") + +case "$version_type" in + commits) + source "${assets}/check_commits.sh" + ;; + tags) + source "${assets}/check_tags.sh" + ;; + branches) + source "${assets}/check_branches.sh" + ;; + *) + echo "unknown version_type: $version_type" + exit 1 + ;; +esac diff --git a/assets/check_branches.sh b/assets/check_branches.sh new file mode 100755 index 00000000..5b5c3f8c --- /dev/null +++ b/assets/check_branches.sh @@ -0,0 +1,53 @@ +branch_filters=$(jq '.source.branch_filters // []' <<< "$payload") +branch_regex=$(jq -r '.source.branch_regex // ""' <<< "$payload") +prev_branches=$(jq -r '.version.branches // ""' <<< "$payload") + +if [[ $(jq 'length' <<< "$branch_filters") -ge 1 && -n "$branch_regex" ]]; then + echo "only one of branch_filters or branch_regex can be specified" + exit 1 +fi + +all_branches=$(git ls-remote --heads "$uri" | awk '{sub("refs/heads/", "", $2); print $2}') + +filtered_branches="" +filtered=false +if [[ $(jq 'length' <<< "$branch_filters") -ge 1 ]]; then + filtered=true + while IFS= read -r branch; do + while IFS= read -r filter; do + if [[ "$branch" == $filter ]]; then + filtered_branches+="${branch}"$'\n' + break + fi + done <<< "$(jq -r '.[]' <<< "$branch_filters")" + done <<< "$all_branches" +fi + +if [[ -n "$branch_regex" ]]; then + filtered=true + filtered_branches=$(echo "$all_branches" | grep -E "$branch_regex" -) +fi + +if [[ "$filtered" == "false" ]]; then + filtered_branches=$all_branches +fi + +sorted_branches=$(echo "$filtered_branches" | sort | paste -sd ',' -) +sorted_branches=${sorted_branches#,} +sorted_branches=${sorted_branches%,} + +if [[ -z "$sorted_branches" ]]; then + echo "No matching branches found. Setting empty version." + sorted_branches="NONE" +fi + +if [[ "$sorted_branches" == "$prev_branches" ]]; then + echo "No change from previous version" + echo "[]" >&3 + exit 0 +fi + +jq -n \ + --arg branches "$sorted_branches" \ + --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%S)" \ + '[{branches: $branches, timestamp: $timestamp}]' >&3 diff --git a/assets/check_commits.sh b/assets/check_commits.sh new file mode 100755 index 00000000..9ecbd681 --- /dev/null +++ b/assets/check_commits.sh @@ -0,0 +1,292 @@ +branch=$(jq -r '.source.branch // ""' <<< "$payload") +branch=${branch# } +paths="$(jq -r '(.source.paths // ["."])[]' <<< "$payload")" # those "'s are important +ignore_paths="$(jq -r '":!" + (.source.ignore_paths // [])[]' <<< "$payload")" # these ones too +tag_filter=$(jq -r '.source.tag_filter // ""' <<< "$payload") +tag_filter=${tag_filter# } +tag_regex=$(jq -r '.source.tag_regex // ""' <<< "$payload") +tag_regex=${tag_regex# } +tag_behaviour=$(jq -r '.source.tag_behaviour // "match_tagged"' <<< "$payload") +ref=$(jq -r '.version.ref // ""' <<< "$payload") +skip_ci_disabled=$(jq -r '.source.disable_ci_skip // false' <<< "$payload") +filter_include=$(jq '.source.commit_filter.include // []' <<< "$payload") +filter_include_all_match=$(jq -r '.source.commit_filter.include_all_match // false' <<< "$payload") +filter_exclude=$(jq '.source.commit_filter.exclude // []' <<< "$payload") +filter_exclude_all_match=$(jq -r '.source.commit_filter.exclude_all_match // false' <<< "$payload") +version_depth=$(jq -r '.source.version_depth // 1' <<< "$payload") +reverse=false + +# Optimization when last commit only is checked and skip ci is disabled +# Get the commit id with git ls-remote instead of downloading the whole repo +if [ "$skip_ci_disabled" = "true" ] && \ + [ "$version_depth" = "1" ] && \ + [ "$paths" = "." ] && \ + [ -z "$ignore_paths" ] && \ + [ -z "$tag_filter" ] && \ + [ -z "$tag_regex" ] && \ + jq -e 'length == 0' <<<"$filter_include" &>/dev/null && \ + jq -e 'length == 0' <<<"$filter_exclude" &>/dev/null +then + branchflag="HEAD" + if [ -n "$branch" ]; then + branchflag="$branch" + fi + commit=$(git ls-remote $uri $branchflag | awk 'NR<=1{print $1}') + if [ -z "$commit" ]; then + echo "No commit returned. Invalid branch?" + exit 1 + fi + if [ -z "$ref" ] || [ "$ref" = "$commit" ]; then + echo $commit | jq -R '.' | jq -s "map({ref: .})" >&3 + exit 0 + fi +fi + +tagflag="" +if [ -n "$tag_filter" ] && [ -n "$tag_regex" ] ; then + echo "Cannot provide both tag_filter and tag_regex" + exit 1 +elif [ -n "$tag_filter" ] || [ -n "$tag_regex" ] ; then + tagflag="--tags" +else + tagflag="--no-tags" +fi + +if [ "$tag_behaviour" != "match_tagged" ] && [ "$tag_behaviour" != "match_tag_ancestors" ]; then + echo "Invalid tag_behaviour. Must be one of 'match_tagged' or 'match_tag_ancestors'." + exit 1 +fi + +for filter in "$filter_include" "$filter_exclude" +do + if jq -e 'type != "array"' <<<"$filter" &>/dev/null + then + echo 'invalid commit filter (expected array of strings)' + echo "$filter" + exit 1 + fi +done + +if [ "$version_depth" -le 0 ]; then + echo "Invalid version_depth. Must be > 0." + exit 1 +fi + +# We're just checking for commits; we don't ever need to fetch LFS files here! +export GIT_LFS_SKIP_SMUDGE=1 + +if [ -d $destination ]; then + cd $destination + if [ "$tagflag" = "--tags" ]; then + git fetch origin 'refs/heads/*:refs/heads/*' --tags -f + else + git fetch origin $tagflag $branch -f + git reset --soft FETCH_HEAD + fi +else + branchflag="" + if [ -n "$branch" ]; then + branchflag="--branch $branch" + fi + + # Determine optimal filter based on whether path filtering is used + if [ "$paths" = "." ] && [ -z "$ignore_paths" ]; then + # No path filtering - use most aggressive filter for maximum performance + filter_flag="--filter=tree:0" + else + # Path filtering needed - keep trees to analyze paths, skip blobs + filter_flag="--filter=blob:none" + fi + + if [ "$tagflag" = "--tags" ]; then + # For tag filtering, don't use --single-branch - tags may be on any branch + git clone --bare $filter_flag --progress $uri $destination --tags + else + git clone --bare $filter_flag --single-branch --progress $uri $branchflag $destination $tagflag + fi + cd $destination + # bare clones don't configure the refspec + if [ -n "$branch" ]; then + git remote set-branches --add origin $branch + fi +fi + +if [ -n "$ref" ] && git cat-file -e "$ref"; then + reverse=true + log_range="${ref}~1..HEAD" + + # if ${ref} does not have parents, ${ref}~1 raises the error: "unknown revision or path not in the working tree" + # the initial commit in a branch will never have parents, but rarely, subsequent commits can also be parentless + orphan_commits=$(git rev-list --max-parents=0 HEAD) + for orphan_commit in ${orphan_commits}; do + if [ "${ref}" = "${orphan_commit}" ]; then + log_range="HEAD" + break + fi + done +else + log_range="" + ref="" +fi + +if [ "$paths" = "." ] && [ -z "$ignore_paths" ]; then + paths_search="" +else + paths_search=`echo "-- $paths $ignore_paths" | tr "\n\r" " "` +fi + +list_command="git rev-list --all --first-parent $log_range $paths_search" +if jq -e 'length > 0' <<<"$filter_include" &>/dev/null +then + list_command+=" | git rev-list --stdin --date-order --first-parent --no-walk=unsorted " + include_items=$(echo $filter_include | jq -r -c '.[]') + for wli in "$include_items" + do + list_command+=" --grep=\"$wli\"" + done + if [ "$filter_include_all_match" == "true" ]; then + list_command+=" --all-match" + fi +fi + +if jq -e 'length > 0' <<<"$filter_exclude" &>/dev/null +then + list_command+=" | git rev-list --stdin --date-order --invert-grep --first-parent --no-walk=unsorted " + exclude_items=$(echo $filter_exclude | jq -r -c '.[]') + for bli in "$exclude_items" + do + list_command+=" --grep=\"$bli\"" + done + if [ "$filter_exclude_all_match" == "true" ]; then + list_command+=" --all-match" + fi +fi + + +if [ "$skip_ci_disabled" != "true" ]; then + list_command+=" | git rev-list --stdin --date-order --grep=\"\\[ci\\sskip\\]\" --grep=\"\\[skip\\sci\\]\" --invert-grep --first-parent --no-walk=unsorted" +fi + +replace_escape_chars() { + sed -e 's/[]\/$*.^[]/\\&/g' <<< $1 +} + +lines_including_and_after() { + local escaped_string=$(replace_escape_chars $1) + sed -ne "/$escaped_string/,$ p" +} + +#if no range is selected just grab the last commit that fits the filter +if [ -z "$log_range" ] && [ -z "$tag_filter" ] && [ -z "$tag_regex" ] +then + list_command+="| git rev-list --stdin --date-order --no-walk=unsorted -$version_depth --reverse" +fi + +if [ "$reverse" == "true" ] && [ -z "$tag_filter" ] && [ -z "$tag_regex" ] +then + list_command+="| git rev-list --stdin --date-order --first-parent --no-walk=unsorted --reverse" +fi + +get_tags_matching_filter() { + local list_command=$1 + local tags=$2 + for tag in $tags; do + # We turn the tag ref (e.g. v1.0.0) into the object name + # (e.g. 1a410efbd13591db07496601ebc7a059dd55cfe9) and use grep to check it is in the output + # of list_command - if it isn't, it doesn't pass one of the other filters and shouldn't be + # outputted. + local commit=$(git rev-list -n 1 $tag) + local this_list_command="$list_command | grep -cFx \"$commit\"" + local list_output="$(set -f; eval "$this_list_command"; set +f)" + if [ "$list_output" -ge 1 ]; then + jq -cn '{ref: $tag, commit: $commit}' --arg tag $tag --arg commit $commit + fi + done +} + +get_tags_match_ancestors_filter() { + local list_command=$1 + local tags=$2 + + # Sort commits so that we look at the oldest commits first + local this_list_command="$list_command | git rev-list --stdin --date-order --first-parent --no-walk=unsorted --reverse" + local list_output="$(set -f; eval "$this_list_command"; set +f)" + + # Store all the tag names in an associative array for quick lookups. + # Also gather the commits each tag is attached to so we can quickly match + # candidate commits in a best-case scenario. + declare -A eligible_tag_names=() + declare -A eligible_tag_commits=() + local tag tag_commit + for tag in $tags; do + eligible_tag_names["$tag"]=1 + tag_commit=$(git rev-list -n 1 "$tag" 2>/dev/null || true) + if [ -n "$tag_commit" ]; then + eligible_tag_commits["$tag_commit"]=1 + fi + done + + # Check each commit and only output commits that are ancestors of tags. + for commit in $list_output; do + local is_ancestor=false + + # Fast path: check if the commit itself is the target of an eligible tag + if [ -n "${eligible_tag_commits[$commit]+x}" ]; then + is_ancestor=true + else + # Find all the tags whose history contains this commit - then check if + # any are eligible tags + local containing_tag + while IFS= read -r containing_tag; do + [ -z "$containing_tag" ] && continue + if [ -n "${eligible_tag_names[$containing_tag]+x}" ]; then + is_ancestor=true + break + fi + done < <(git tag --contains "$commit") + fi + + if [ "$is_ancestor" = true ]; then + jq -cn '{ref: $commit}' --arg commit $commit + fi + done +} + +if [ -n "$tag_filter" ] || [ -n "$tag_regex" ]; then + # Create a suffix to "git tag" that will apply the tag filter + if [ -n "$tag_filter" ]; then + tag_filter_cmd="--list \"$tag_filter\"" + elif [ -n "$tag_regex" ]; then + tag_filter_cmd="| grep -Ex \"$tag_regex\"" + fi + + # Build a list of tag refs (e.g. v1.0.0) that match the filter + if [ -n "$ref" ] && [ -n "$branch" ]; then + tags=$(set -f; eval "git tag --sort=creatordate --contains $ref --merged $branch $tag_filter_cmd"; set +f) + elif [ -n "$ref" ]; then + tags=$(set -f; eval "git tag --sort=creatordate $tag_filter_cmd | lines_including_and_after $ref"; set +f) + else + branch_flag= + if [ -n "$branch" ]; then + branch_flag="--merged $branch" + fi + tags=$(set -f; eval "git tag --sort=creatordate $branch_flag $tag_filter_cmd"; set +f) + fi + + # Only proceed if we actually found any tags + if [ -n "$tags" ]; then + if [ "$tag_behaviour" == "match_tagged" ]; then + get_tags_matching_filter "$list_command" "$tags" | tail "-$version_depth" | jq -s "map(.)" >&3 + else + get_tags_match_ancestors_filter "$list_command" "$tags" | tail "-$version_depth" | jq -s "map(.)" >&3 + fi + else + jq -n "[]" >&3 + fi +else + { + set -f + eval "$list_command" + set +f + } | jq -R '.' | jq -s "map({ref: .})" >&3 +fi diff --git a/assets/check_tags.sh b/assets/check_tags.sh new file mode 100755 index 00000000..1952ec68 --- /dev/null +++ b/assets/check_tags.sh @@ -0,0 +1,81 @@ +tag_filters=$(jq '(.source.tag_filters // []) + (if .source.tag_filter then [.source.tag_filter] else [] end)' <<< "$payload") +tag_regex=$(jq -r '.source.tag_regex // ""' <<< "$payload") +tag_sort=$(jq -r '.source.tag_sort // "creatordate"' <<< "$payload") +prev_tag=$(jq -r '.version.tag // ""' <<< "$payload") + +if [[ $(jq 'length' <<< "$tag_filters") -ge 1 && -n "$tag_regex" ]]; then + echo "only one of tag_filters or tag_regex can be specified" + exit 1 +fi + +if [[ ! -d "$destination" ]]; then + git init --bare --quiet "$destination" +fi + +cd "$destination" + +# git fetch exits 1 when the refspec matches no refs, so short-circuit +# when the remote has no tags. +if [[ -z "$(git ls-remote --tags "$uri")" ]]; then + echo '[]' >&3 + exit 0 +fi + +git fetch --depth=1 \ + --filter=tree:0 \ + --no-tags \ + "$uri" \ + '+refs/tags/*:refs/tags/*' + +# get all tags, sorting by creation-date +all_tags=$(git for-each-ref \ + --sort=creatordate \ + --format='%(refname:short)%09%(objectname)%09%(*objectname)' \ + refs/tags/) + +filtered_tags="" +filtered=false +if [[ $(jq 'length' <<< "$tag_filters") -ge 1 ]]; then + filtered=true + while IFS= read -r tag; do + while IFS= read -r filter; do + # $tag is the tag name and refs joined by tabs. We only want to + # match against the tag name, so strip everything from the first + # tab onward. + if [[ "${tag%%$'\t'*}" == $filter ]]; then + filtered_tags+="${tag}"$'\n' + break + fi + done <<< "$(jq -r '.[]' <<< "$tag_filters")" + done <<< "$all_tags" +fi + +if [[ -n "$tag_regex" ]]; then + filtered=true + while IFS= read -r tag; do + if echo "${tag%%$'\t'*}" | grep -E "$tag_regex" >/dev/null; then + filtered_tags+="${tag}"$'\n' + fi + done <<< "$all_tags" +fi + +if [[ "$filtered" == "false" ]]; then + filtered_tags=$all_tags +fi + +sorted_tags="" +sorted=false +if [[ "$tag_sort" == "semver" ]]; then + sorted=true + sorted_tags=$(echo "$filtered_tags" | sort -V) +fi + +if [[ "$sorted" == "false" ]]; then + sorted_tags=$filtered_tags +fi + +jtags=$(printf '%s' "$sorted_tags" | jq -Rn \ + --arg prevtag "$prev_tag" \ + '[inputs | (./"\t") | {tag: .[0], ref: (.[2] // .[1])}] | .[(map(.tag) | index($prevtag)):]') + +echo "$jtags" >&3 diff --git a/assets/common.sh b/assets/common.sh index a0dc76e3..eb88dfa1 100644 --- a/assets/common.sh +++ b/assets/common.sh @@ -120,28 +120,31 @@ configure_git_ssl_verification() { } add_git_metadata_basic() { - local commit=$(git rev-parse HEAD | jq -R .) - local author=$(git log -1 --format=format:%an | jq -s -R .) - local author_date=$(git log -1 --format=format:%ai | jq -R .) - - jq ". + [ - {name: \"commit\", value: ${commit}}, - {name: \"author\", value: ${author}}, - {name: \"author_date\", value: ${author_date}, type: \"time\"} - ]" + local commit=$(git rev-parse HEAD) + local author=$(git log -1 --format=format:%an) + local author_date=$(git log -1 --format=format:%ai) + + jq --arg commit "$commit" \ + --arg author "$author" \ + --arg author_date "$author_date" \ + '. + [ + {name: "commit", value: $commit}, + {name: "author", value: $author}, + {name: "author_date", value: $author_date, type: "time"} + ]' } add_git_metadata_committer() { - local author=$(git log -1 --format=format:%an | jq -s -R .) - local author_date=$(git log -1 --format=format:%ai | jq -R .) - local committer=$(git log -1 --format=format:%cn | jq -s -R .) - local committer_date=$(git log -1 --format=format:%ci | jq -R .) + local author=$(git log -1 --format=format:%an) + local author_date=$(git log -1 --format=format:%ai) + local committer=$(git log -1 --format=format:%cn) + local committer_date=$(git log -1 --format=format:%ci) if [ "$author" = "$committer" ] && [ "$author_date" = "$committer_date" ]; then - jq ". + [ - {name: \"committer\", value: ${committer}}, - {name: \"committer_date\", value: ${committer_date}, type: \"time\"} - ]" + jq --arg committer "$committer" --arg committer_date "$committer_date" '. + [ + {name: "committer", value: $committer}, + {name: "committer_date", value: $committer_date, type: "time"} + ]' else cat fi @@ -153,9 +156,9 @@ add_git_metadata_branch() { jq -R ". | select(. != \"\")" | jq -r -s "map(.) | join (\",\")") if [ -n "${branch}" ]; then - jq ". + [ - {name: \"branch\", value: \"${branch}\"} - ]" + jq --arg branch "$branch" '. + [ + {name: "branch", value: $branch} + ]' else cat fi @@ -167,20 +170,32 @@ add_git_metadata_tags() { jq -r -s "map(.) | join(\",\")") if [ -n "${tags}" ]; then - jq ". + [ - {name: \"tags\", value: \"${tags}\"} - ]" + jq --arg tags "$tags" '. + [ + {name: "tags", value: $tags} + ]' + else + cat + fi +} + +add_git_metadata_tag() { + local tag=$(git tag --points-at HEAD) + + if [ -n "${tag}" ]; then + jq --arg tag "$tag" '. + [ + {name: "tag", value: $tag} + ]' else cat fi } add_git_metadata_message() { - local message=$(git log -1 --format=format:%B | head -c 10240 | jq -s -R .) + local message=$(git log -1 --format=format:%B | head -c 10240) - jq ". + [ - {name: \"message\", value: ${message}, type: \"message\"} - ]" + jq --arg message "$message" '. + [ + {name: "message", value: $message, type: "message"} + ]' } add_git_metadata_url() { @@ -213,9 +228,9 @@ add_git_metadata_url() { esac if [ -n "$url" ]; then - jq ". + [ - {name: \"url\", value: \"${url}\"} - ]" + jq --arg url "$url" '. + [ + {name: "url", value: $url} + ]' else jq ". + []" fi @@ -232,6 +247,15 @@ git_metadata() { add_git_metadata_url } +git_tag_metadata() { + jq -n "[]" | \ + add_git_metadata_basic | \ + add_git_metadata_committer | \ + add_git_metadata_tag | \ + add_git_metadata_message | \ + add_git_metadata_url +} + configure_submodule_credentials() { local username local password diff --git a/assets/in b/assets/in index 7c50dda4..0f002813 100755 --- a/assets/in +++ b/assets/in @@ -1,12 +1,12 @@ #!/bin/bash -# vim: set ft=sh set -e exec 3>&1 # make stdout available as fd 3 for the result exec 1>&2 # redirect all output to stderr for logging -source $(dirname $0)/common.sh +assets="$(dirname $0)" +source "${assets}/common.sh" destination=$1 @@ -15,17 +15,9 @@ if [ -z "$destination" ]; then exit 1 fi -# for jq -PATH=/usr/local/bin:$PATH - -bin_dir="${0%/*}" -if [ "${bin_dir#/}" == "$bin_dir" ]; then - bin_dir="$PWD/$bin_dir" -fi - payload="$(cat <&0)" -unknown_keys=$(jq --slurpfile schema "$(dirname $0)/in_schema.json" '(.params // [] | keys_unsorted) - ($schema[0] | keys_unsorted)' <<< "$payload") +unknown_keys=$(jq --slurpfile schema "${assets}/in_schema.json" '(.params // [] | keys_unsorted) - ($schema[0] | keys_unsorted)' <<< "$payload") if jq --exit-status 'length > 0' <<< "$unknown_keys" &>/dev/null; then echo "Found unknown keys in get params:" @@ -43,280 +35,36 @@ if [[ "$src_debug" == "true" || "$params_debug" == "true" ]]; then export GIT_CURL_VERBOSE=1 fi +git_config_payload=$(jq -r '.source.git_config // []' <<< "$payload") +configure_git_global "${git_config_payload}" load_pubkey "$payload" load_git_crypt_key "$payload" configure_https_tunnel "$payload" configure_git_ssl_verification "$payload" configure_credentials "$payload" +# These vars are used by multiple version_type's uri=$(jq -r '.source.uri // ""' <<< "$payload") -branch=$(jq -r '.source.branch // ""' <<< "$payload") -sparse_paths="$(jq -r '(.source.sparse_paths // ["."])[]' <<< "$payload")" # those "'s are important -git_config_payload=$(jq -r '.source.git_config // []' <<< "$payload") -ref=$(jq -r '.version.ref // "HEAD"' <<< "$payload") -override_branch=$(jq -r '.version.branch // ""' <<< "$payload") -depth=$(jq -r '(.params.depth // 0)' <<< "$payload") -fetch=$(jq -r '(.params.fetch // [])[]' <<< "$payload") -submodules=$(jq -r '(.params.submodules // "all")' <<< "$payload") -submodule_recursive=$(jq -r '(.params.submodule_recursive // true)' <<< "$payload") -submodule_remote=$(jq -r '(.params.submodule_remote // false)' <<< "$payload") -commit_verification_key_ids=$(jq -r '(.source.commit_verification_key_ids // [])[]' <<< "$payload") -commit_verification_keys=$(jq -r '(.source.commit_verification_keys // [])[]' <<< "$payload") -tag_filter=$(jq -r '.source.tag_filter // ""' <<< "$payload") -tag_regex=$(jq -r '.source.tag_regex // ""' <<< "$payload") -fetch_tags=$(jq -r '.params.fetch_tags' <<< "$payload") -gpg_keyserver=$(jq -r '.source.gpg_keyserver // "hkp://keyserver.ubuntu.com/"' <<< "$payload") -disable_git_lfs=$(jq -r '(.params.disable_git_lfs // false)' <<< "$payload") -clean_tags=$(jq -r '(.params.clean_tags // false)' <<< "$payload") -short_ref_format=$(jq -r '(.params.short_ref_format // "%s")' <<< "$payload") -timestamp_format=$(jq -r '(.params.timestamp_format // "iso8601")' <<< "$payload") -describe_ref_options=$(jq -r '(.params.describe_ref_options // "--always --dirty --broken")' <<< "$payload") -search_remote_refs_flag=$(jq -r '(.source.search_remote_refs // false)' <<< "$payload") -all_branches=$(jq -r '(.params.all_branches // false)' <<< "$payload") - -# If params not defined, get it from source -if [ -z "$fetch_tags" ] || [ "$fetch_tags" == "null" ] ; then - fetch_tags=$(jq -r '.source.fetch_tags' <<< "$payload") -fi - -configure_git_global "${git_config_payload}" - -if [ -z "$uri" ]; then - echo "invalid payload (missing uri):" >&2 - cat $payload >&2 - exit 1 -fi - -branchflag="" -if [ -n "$branch" ]; then - branchflag="--branch $branch" -fi - -if [ -n "$override_branch" ]; then - echo "Override $branch with $override_branch" - branchflag="--branch $override_branch" -fi - -depthflag="" -if test "$depth" -gt 0 2> /dev/null; then - depthflag="--depth $depth" -fi - -tagflag="" -if [ "$fetch_tags" == "false" ] ; then - tagflag="--no-tags" -elif [ -n "$tag_filter" ] || [ -n "$tag_regex" ] || [ "$fetch_tags" == "true" ] ; then - tagflag="--tags" -fi - -nocheckoutflag="" -if [ "$sparse_paths" != "." ] && [ "$sparse_paths" != "" ]; then - nocheckoutflag=" --no-checkout" -fi - -if [ "$disable_git_lfs" == "true" ]; then - # skip the fetching of LFS objects for all following git commands - export GIT_LFS_SKIP_SMUDGE=1 -fi - -singlebranchflag="--single-branch" -if [ "${all_branches,,}" == "true" ]; then - singlebranchflag="" -fi - -git clone --progress $singlebranchflag $depthflag $uri $branchflag $destination $tagflag $nocheckoutflag - -cd $destination - -configure_git_local "${git_config_payload}" - -if [ "$sparse_paths" != "." ] && [ "$sparse_paths" != "" ]; then - git config core.sparseCheckout true - echo "$sparse_paths" >> ./.git/info/sparse-checkout -fi - -git fetch origin refs/notes/*:refs/notes/* $tagflag - -if [ "$depth" -gt 0 ]; then - "$bin_dir"/deepen_shallow_clone_until_ref_is_found_then_check_out "$depth" "$ref" "$tagflag" -else - if [ "$search_remote_refs_flag" == "true" ] && ! [ -z "$branchflag" ] && ! git rev-list -1 $ref 2> /dev/null > /dev/null; then - change_ref=$(git ls-remote origin | grep $ref | cut -f2) - if ! [ -z "$change_ref" ]; then - echo "$ref not found locally, but search_remote_refs is enabled. Attempting to fetch $change_ref first." - git fetch origin $change_ref - else - echo "WARNING: couldn't find a ref for $ref listed on the remote" - fi - fi - git checkout -f -q "$ref" -fi - -invalid_key() { - echo "Invalid GPG key in: ${commit_verification_keys}" - exit 2 -} - -commit_not_signed() { - commit_id=$(git rev-parse ${ref}) - echo "The commit ${commit_id} is not signed" - exit 1 -} - -if [ ! -z "${commit_verification_keys}" ] || [ ! -z "${commit_verification_key_ids}" ] ; then - if [ ! -z "${commit_verification_keys}" ]; then - echo "${commit_verification_keys}" | gpg --batch --import || invalid_key "${commit_verification_keys}" - fi - if [ ! -z "${commit_verification_key_ids}" ]; then - echo "${commit_verification_key_ids}" | \ - xargs --no-run-if-empty -n1 gpg --batch --keyserver $gpg_keyserver --recv-keys - fi - git verify-commit $(git rev-list -n 1 $ref) || commit_not_signed -fi - -git log -1 --oneline -git clean --force --force -d -git submodule sync - -if [ -f $GIT_CRYPT_KEY_PATH ]; then - echo "unlocking git repo" - git-crypt unlock $GIT_CRYPT_KEY_PATH -fi - - -submodule_parameters="" -if [ "$submodule_remote" != "false" ]; then - submodule_parameters+=" --remote " -fi -if [ "$submodule_recursive" != "false" ]; then - submodule_parameters+=" --recursive " -fi - -if [ "$submodules" != "none" ]; then - value_regexp="." - if [ "$submodules" != "all" ]; then - value_regexp="$(echo $submodules | jq -r 'map(. + "$") | join("|")')" - fi - - { - git config --file .gitmodules --name-only --get-regexp '\.path$' "$value_regexp" | - sed -e 's/^submodule\.\(.\+\)\.path$/\1/' - } | while read submodule_name; do - submodule_path="$(git config --file .gitmodules --get "submodule.${submodule_name}.path")" - submodule_url="$(git config --file .gitmodules --get "submodule.${submodule_name}.url")" - - if [ "$depth" -gt 0 ]; then - git config "submodule.${submodule_name}.update" "!$bin_dir/deepen_shallow_clone_until_ref_is_found_then_check_out $depth" - fi - - if ! [ -e "$submodule_path" ]; then - echo $'\e[31m'"warning: skipping missing submodule: $submodule_path"$'\e[0m' - continue - fi - - # check for ssh submodule_credentials - submodule_cred=$(jq --arg submodule_url "${submodule_url}" '.source.submodule_credentials // [] | [.[] | select(.url==$submodule_url)] | first // empty' <<< ${payload}) - - if [[ -z ${submodule_cred} ]]; then - - # update normally - git submodule update --init --no-fetch $depthflag $submodule_parameters "$submodule_path" - - else - - # create or re-initialize ssh-agent - init_ssh_agent - - private_key=$(jq -r '.private_key' <<< ${submodule_cred}) - passphrase=$(jq -r '.private_key_passphrase // empty' <<< ${submodule_cred}) - - private_key_path=$(mktemp -t git-resource-submodule-private-key.XXXXXX) - echo "${private_key}" > ${private_key_path} - chmod 0600 ${private_key_path} - - # add submodule private_key identity - SSH_ASKPASS_REQUIRE=force SSH_ASKPASS=$(dirname $0)/askpass.sh GIT_SSH_PRIVATE_KEY_PASS="$passphrase" DISPLAY= ssh-add $private_key_path > /dev/null - - git submodule update --init --no-fetch $depthflag $submodule_parameters "$submodule_path" - - # restore main ssh-agent (if needed) - load_pubkey "${payload}" - - fi - - if [ "$depth" -gt 0 ]; then - git config --unset "submodule.${submodule_name}.update" - fi - done -fi - -for branch in $fetch; do - git fetch origin $branch - if ! git show-ref --verify --quiet refs/heads/$branch; then - git branch $branch FETCH_HEAD - fi -done - -if [ "$ref" == "HEAD" ]; then - return_ref=$(git rev-parse HEAD) -else - return_ref=$ref -fi - -# Store committer email in .git/committer. Can be used to send email to last committer on failed build -# Using https://github.com/mdomke/concourse-email-resource for example -git --no-pager log -1 --pretty=format:"%ae" > .git/committer - -git --no-pager log -1 --pretty=format:"%an" > .git/committer_name - -# Store git-resource returned version ref .git/ref. Useful to know concourse -# pulled ref in following tasks and resources. -echo "${return_ref}" > .git/ref - -metadata=$(git_metadata) -echo "${metadata}" | jq '.' > .git/metadata.json - -# Store short ref with templating. Useful to build Docker images with -# a custom tag -echo "${return_ref}" | cut -c1-7 | awk "{ printf \"${short_ref_format}\", \$1 }" > .git/short_ref - -# Write individual metadata fields to separate files - -# .git/commit - full SHA hash -echo "${metadata}" | jq -r '.[] | select(.name == "commit") | .value' > .git/commit -# .git/author - commit author name -echo "${metadata}" | jq -r '.[] | select(.name == "author") | .value' > .git/author -# .git/author_date - timestamp when the author originally created the commit -echo "${metadata}" | jq -r '.[] | select(.name == "author_date") | .value' > .git/author_date - -# .git/tags - branch name(s) containing this commit (comma-separated if multiple) -echo "${metadata}" | jq -r '.[] | select(.name == "branch") | .value // ""' > .git/branch -# .git/tags - comma-separated list of tags -echo "${metadata}" | jq -r '.[] | select(.name == "tags") | .value // ""' > .git/tags -# .git/url - web URL to view commit (if applicable) -echo "${metadata}" | jq -r '.[] | select(.name == "url") | .value // ""' > .git/url -# .git/committer_date - timestamp when the commit was created in the repository -echo "${metadata}" | jq -r '.[] | select(.name == "committer_date") | .value // ""' > .git/committer_date - -# Store commit message in .git/commit_message. Can be used to inform about -# the content of a successful build. -# Using https://github.com/cloudfoundry-community/slack-notification-resource -# for example -git log -1 --format=format:%B > .git/commit_message - -# Store commit date in .git/commit_timestamp. Can be used for tagging builds -git log -1 --format=%cd --date=${timestamp_format} > .git/commit_timestamp - -# Store describe_ref when available. Useful to build Docker images with -# a custom tag, or package to publish -echo "$(git describe ${describe_ref_options})" > .git/describe_ref - - -if [ "$clean_tags" == "true" ]; then - git tag | xargs git tag -d +uri=${uri# } +if [[ -z "$uri" ]]; then + echo "source.uri is required and must not be empty" + exit 1 fi -jq -n "{ - version: {ref: $(echo $return_ref | jq -R .)}, - metadata: $metadata -}" >&3 +version_type=$(jq -r '.source.version_type // "commits"' <<< "$payload") + +case "$version_type" in + commits) + source "${assets}/in_commits.sh" + ;; + tags) + source "${assets}/in_tags.sh" + ;; + branches) + source "${assets}/in_branches.sh" + ;; + *) + echo "unknown version_type: $version_type" + exit 1 + ;; +esac diff --git a/assets/in_branches.sh b/assets/in_branches.sh new file mode 100755 index 00000000..890a387a --- /dev/null +++ b/assets/in_branches.sh @@ -0,0 +1,20 @@ +branches=$(jq -r '.version.branches // ""' <<< "$payload") + +if [[ "$branches" == "NONE" ]]; then + echo "[]" > "${destination}/branches.json" + jq -n \ + --argjson version "$(jq -r '.version' <<< "$payload")" \ + --arg branches_value "NONE" \ + '{version: $version, metadata: [{name: "branches", value: $branches_value}]}' >&3 + exit 0 +fi + +echo "$branches" | jq -Rc 'split(",")' > "${destination}/branches.json" + +# Format branches as a multi-line string for nicely printing in metadata +branches_metadata=$(echo "$branches" | jq -Rrc 'split(",") | join("\n")') + +jq -n \ + --argjson version "$(jq -r '.version' <<< "$payload")" \ + --arg branches_value "$branches_metadata" \ + '{version: $version, metadata: [{name: "branches", value: $branches_value}]}' >&3 diff --git a/assets/in_commits.sh b/assets/in_commits.sh new file mode 100755 index 00000000..1f3d28f1 --- /dev/null +++ b/assets/in_commits.sh @@ -0,0 +1,264 @@ +branch=$(jq -r '.source.branch // ""' <<< "$payload") +sparse_paths="$(jq -r '(.source.sparse_paths // ["."])[]' <<< "$payload")" # those "'s are important +ref=$(jq -r '.version.ref // "HEAD"' <<< "$payload") +override_branch=$(jq -r '.version.branch // ""' <<< "$payload") +depth=$(jq -r '(.params.depth // 0)' <<< "$payload") +fetch=$(jq -r '(.params.fetch // [])[]' <<< "$payload") +submodules=$(jq -r '(.params.submodules // "all")' <<< "$payload") +submodule_recursive=$(jq -r '(.params.submodule_recursive // true)' <<< "$payload") +submodule_remote=$(jq -r '(.params.submodule_remote // false)' <<< "$payload") +commit_verification_key_ids=$(jq -r '(.source.commit_verification_key_ids // [])[]' <<< "$payload") +commit_verification_keys=$(jq -r '(.source.commit_verification_keys // [])[]' <<< "$payload") +tag_filter=$(jq -r '.source.tag_filter // ""' <<< "$payload") +tag_regex=$(jq -r '.source.tag_regex // ""' <<< "$payload") +fetch_tags=$(jq -r '.params.fetch_tags' <<< "$payload") +gpg_keyserver=$(jq -r '.source.gpg_keyserver // "hkp://keyserver.ubuntu.com/"' <<< "$payload") +disable_git_lfs=$(jq -r '(.params.disable_git_lfs // false)' <<< "$payload") +clean_tags=$(jq -r '(.params.clean_tags // false)' <<< "$payload") +short_ref_format=$(jq -r '(.params.short_ref_format // "%s")' <<< "$payload") +timestamp_format=$(jq -r '(.params.timestamp_format // "iso8601")' <<< "$payload") +describe_ref_options=$(jq -r '(.params.describe_ref_options // "--always --dirty --broken")' <<< "$payload") +search_remote_refs_flag=$(jq -r '(.source.search_remote_refs // false)' <<< "$payload") +all_branches=$(jq -r '(.params.all_branches // false)' <<< "$payload") + +# If params not defined, get it from source +if [ -z "$fetch_tags" ] || [ "$fetch_tags" == "null" ] ; then + fetch_tags=$(jq -r '.source.fetch_tags' <<< "$payload") +fi + +branchflag="" +if [ -n "$branch" ]; then + branchflag="--branch $branch" +fi + +if [ -n "$override_branch" ]; then + echo "Override $branch with $override_branch" + branchflag="--branch $override_branch" +fi + +depthflag="" +if test "$depth" -gt 0 2> /dev/null; then + depthflag="--depth $depth" +fi + +tagflag="" +if [ "$fetch_tags" == "false" ] ; then + tagflag="--no-tags" +elif [ -n "$tag_filter" ] || [ -n "$tag_regex" ] || [ "$fetch_tags" == "true" ] ; then + tagflag="--tags" +fi + +nocheckoutflag="" +if [ "$sparse_paths" != "." ] && [ "$sparse_paths" != "" ]; then + nocheckoutflag=" --no-checkout" +fi + +if [ "$disable_git_lfs" == "true" ]; then + # skip the fetching of LFS objects for all following git commands + export GIT_LFS_SKIP_SMUDGE=1 +fi + +singlebranchflag="--single-branch" +if [ "${all_branches,,}" == "true" ]; then + singlebranchflag="" +fi + +git clone --progress $singlebranchflag $depthflag $uri $branchflag $destination $tagflag $nocheckoutflag + +cd $destination + +configure_git_local "${git_config_payload}" + +if [ "$sparse_paths" != "." ] && [ "$sparse_paths" != "" ]; then + git config core.sparseCheckout true + echo "$sparse_paths" >> ./.git/info/sparse-checkout +fi + +git fetch origin refs/notes/*:refs/notes/* $tagflag + +if [ "$depth" -gt 0 ]; then + "${assets}/deepen_shallow_clone_until_ref_is_found_then_check_out" "$depth" "$ref" "$tagflag" +else + if [ "$search_remote_refs_flag" == "true" ] && ! [ -z "$branchflag" ] && ! git rev-list -1 $ref 2> /dev/null > /dev/null; then + change_ref=$(git ls-remote origin | grep $ref | cut -f2) + if ! [ -z "$change_ref" ]; then + echo "$ref not found locally, but search_remote_refs is enabled. Attempting to fetch $change_ref first." + git fetch origin $change_ref + else + echo "WARNING: couldn't find a ref for $ref listed on the remote" + fi + fi + git checkout -f -q "$ref" +fi + +invalid_key() { + echo "Invalid GPG key in: ${commit_verification_keys}" + exit 2 +} + +commit_not_signed() { + commit_id=$(git rev-parse ${ref}) + echo "The commit ${commit_id} is not signed" + exit 1 +} + +if [ ! -z "${commit_verification_keys}" ] || [ ! -z "${commit_verification_key_ids}" ] ; then + if [ ! -z "${commit_verification_keys}" ]; then + echo "${commit_verification_keys}" | gpg --batch --import || invalid_key "${commit_verification_keys}" + fi + if [ ! -z "${commit_verification_key_ids}" ]; then + echo "${commit_verification_key_ids}" | \ + xargs --no-run-if-empty -n1 gpg --batch --keyserver $gpg_keyserver --recv-keys + fi + git verify-commit $(git rev-list -n 1 $ref) || commit_not_signed +fi + +git log -1 --oneline +git clean --force --force -d +git submodule sync + +if [ -f $GIT_CRYPT_KEY_PATH ]; then + echo "unlocking git repo" + git-crypt unlock $GIT_CRYPT_KEY_PATH +fi + + +submodule_parameters="" +if [ "$submodule_remote" != "false" ]; then + submodule_parameters+=" --remote " +fi +if [ "$submodule_recursive" != "false" ]; then + submodule_parameters+=" --recursive " +fi + +if [ "$submodules" != "none" ]; then + value_regexp="." + if [ "$submodules" != "all" ]; then + value_regexp="$(echo $submodules | jq -r 'map(. + "$") | join("|")')" + fi + + { + git config --file .gitmodules --name-only --get-regexp '\.path$' "$value_regexp" | + sed -e 's/^submodule\.\(.\+\)\.path$/\1/' + } | while read submodule_name; do + submodule_path="$(git config --file .gitmodules --get "submodule.${submodule_name}.path")" + submodule_url="$(git config --file .gitmodules --get "submodule.${submodule_name}.url")" + + if [ "$depth" -gt 0 ]; then + git config "submodule.${submodule_name}.update" "!${assets}/deepen_shallow_clone_until_ref_is_found_then_check_out $depth" + fi + + if ! [ -e "$submodule_path" ]; then + echo $'\e[31m'"warning: skipping missing submodule: $submodule_path"$'\e[0m' + continue + fi + + # check for ssh submodule_credentials + submodule_cred=$(jq --arg submodule_url "${submodule_url}" '.source.submodule_credentials // [] | [.[] | select(.url==$submodule_url)] | first // empty' <<< ${payload}) + + if [[ -z ${submodule_cred} ]]; then + + # update normally + git submodule update --init --no-fetch $depthflag $submodule_parameters "$submodule_path" + + else + + # create or re-initialize ssh-agent + init_ssh_agent + + private_key=$(jq -r '.private_key' <<< ${submodule_cred}) + passphrase=$(jq -r '.private_key_passphrase // empty' <<< ${submodule_cred}) + + private_key_path=$(mktemp -t git-resource-submodule-private-key.XXXXXX) + echo "${private_key}" > ${private_key_path} + chmod 0600 ${private_key_path} + + # add submodule private_key identity + SSH_ASKPASS_REQUIRE=force SSH_ASKPASS=$(dirname $0)/askpass.sh GIT_SSH_PRIVATE_KEY_PASS="$passphrase" DISPLAY= ssh-add $private_key_path > /dev/null + + git submodule update --init --no-fetch $depthflag $submodule_parameters "$submodule_path" + + # restore main ssh-agent (if needed) + load_pubkey "${payload}" + + fi + + if [ "$depth" -gt 0 ]; then + git config --unset "submodule.${submodule_name}.update" + fi + done +fi + +for branch in $fetch; do + git fetch origin $branch + if ! git show-ref --verify --quiet refs/heads/$branch; then + git branch $branch FETCH_HEAD + fi +done + +if [ "$ref" == "HEAD" ]; then + return_ref=$(git rev-parse HEAD) +else + return_ref=$ref +fi + +# Store committer email in .git/committer. Can be used to send email to last committer on failed build +# Using https://github.com/mdomke/concourse-email-resource for example +git --no-pager log -1 --pretty=format:"%ae" > .git/committer + +git --no-pager log -1 --pretty=format:"%an" > .git/committer_name + +# Store git-resource returned version ref .git/ref. Useful to know concourse +# pulled ref in following tasks and resources. +echo "${return_ref}" > .git/ref + +metadata=$(git_metadata) +echo "${metadata}" | jq '.' > .git/metadata.json + +# Store short ref with templating. Useful to build Docker images with +# a custom tag +echo "${return_ref}" | cut -c1-7 | awk "{ printf \"${short_ref_format}\", \$1 }" > .git/short_ref + +# Write individual metadata fields to separate files + +# .git/commit - full SHA hash +echo "${metadata}" | jq -r '.[] | select(.name == "commit") | .value' > .git/commit +# .git/author - commit author name +echo "${metadata}" | jq -r '.[] | select(.name == "author") | .value' > .git/author +# .git/author_date - timestamp when the author originally created the commit +echo "${metadata}" | jq -r '.[] | select(.name == "author_date") | .value' > .git/author_date + +# .git/tags - branch name(s) containing this commit (comma-separated if multiple) +echo "${metadata}" | jq -r '.[] | select(.name == "branch") | .value // ""' > .git/branch +# .git/tags - comma-separated list of tags +echo "${metadata}" | jq -r '.[] | select(.name == "tags") | .value // ""' > .git/tags +# .git/url - web URL to view commit (if applicable) +echo "${metadata}" | jq -r '.[] | select(.name == "url") | .value // ""' > .git/url +# .git/committer_date - timestamp when the commit was created in the repository +echo "${metadata}" | jq -r '.[] | select(.name == "committer_date") | .value // ""' > .git/committer_date + +# Store commit message in .git/commit_message. Can be used to inform about +# the content of a successful build. +# Using https://github.com/cloudfoundry-community/slack-notification-resource +# for example +git log -1 --format=format:%B > .git/commit_message + +# Store commit date in .git/commit_timestamp. Can be used for tagging builds +git log -1 --format=%cd --date=${timestamp_format} > .git/commit_timestamp + +# Store describe_ref when available. Useful to build Docker images with +# a custom tag, or package to publish +echo "$(git describe ${describe_ref_options})" > .git/describe_ref + + +if [ "$clean_tags" == "true" ]; then + git tag | xargs git tag -d +fi + +jq -n \ + --arg ref "$return_ref" \ + --argjson metadata "$metadata" \ + '{ + version: {ref: $ref}, + metadata: $metadata +}' >&3 diff --git a/assets/in_tags.sh b/assets/in_tags.sh new file mode 100755 index 00000000..b1fb2a8e --- /dev/null +++ b/assets/in_tags.sh @@ -0,0 +1,168 @@ +tag=$(jq -r '.version.tag // ""' <<< "$payload") +ref=$(jq -r '.version.ref // ""' <<< "$payload") +submodules=$(jq -r '(.params.submodules // "all")' <<< "$payload") +submodule_recursive=$(jq -r '(.params.submodule_recursive // true)' <<< "$payload") +submodule_remote=$(jq -r '(.params.submodule_remote // false)' <<< "$payload") +disable_git_lfs=$(jq -r '(.params.disable_git_lfs // false)' <<< "$payload") +commit_verification_key_ids=$(jq -r '(.source.commit_verification_key_ids // [])[]' <<< "$payload") +commit_verification_keys=$(jq -r '(.source.commit_verification_keys // [])[]' <<< "$payload") +gpg_keyserver=$(jq -r '.source.gpg_keyserver // "hkp://keyserver.ubuntu.com/"' <<< "$payload") +short_ref_format=$(jq -r '(.params.short_ref_format // "%s")' <<< "$payload") +timestamp_format=$(jq -r '(.params.timestamp_format // "iso8601")' <<< "$payload") + +if [ "$disable_git_lfs" == "true" ]; then + # skip the fetching of LFS objects for all following git commands + export GIT_LFS_SKIP_SMUDGE=1 +fi + +git config --global advice.detachedHead false +git clone --progress --depth 1 --branch "$tag" "$uri" "$destination" + +cd $destination + +configure_git_local "${git_config_payload}" + +invalid_key() { + echo "Invalid GPG key in: ${commit_verification_keys}" + exit 2 +} + +commit_not_signed() { + commit_id=$(git rev-parse ${ref}) + echo "The commit ${commit_id} is not signed" + exit 1 +} + +if [ ! -z "${commit_verification_keys}" ] || [ ! -z "${commit_verification_key_ids}" ] ; then + if [ ! -z "${commit_verification_keys}" ]; then + echo "${commit_verification_keys}" | gpg --batch --import || invalid_key "${commit_verification_keys}" + fi + if [ ! -z "${commit_verification_key_ids}" ]; then + echo "${commit_verification_key_ids}" | \ + xargs --no-run-if-empty -n1 gpg --batch --keyserver $gpg_keyserver --recv-keys + fi + git verify-commit $(git rev-list -n 1 $ref) || commit_not_signed +fi + +git log -1 --oneline +git clean --force --force -d +git submodule sync + +if [ -f $GIT_CRYPT_KEY_PATH ]; then + echo "unlocking git repo" + git-crypt unlock $GIT_CRYPT_KEY_PATH +fi + +submodule_parameters="" +if [ "$submodule_remote" != "false" ]; then + submodule_parameters+=" --remote " +fi +if [ "$submodule_recursive" != "false" ]; then + submodule_parameters+=" --recursive " +fi + +if [ "$submodules" != "none" ]; then + value_regexp="." + if [ "$submodules" != "all" ]; then + value_regexp="$(echo $submodules | jq -r 'map(. + "$") | join("|")')" + fi + + { + git config --file .gitmodules --name-only --get-regexp '\.path$' "$value_regexp" | + sed -e 's/^submodule\.\(.\+\)\.path$/\1/' + } | while read submodule_name; do + submodule_path="$(git config --file .gitmodules --get "submodule.${submodule_name}.path")" + submodule_url="$(git config --file .gitmodules --get "submodule.${submodule_name}.url")" + + if ! [ -e "$submodule_path" ]; then + echo $'\e[31m'"warning: skipping missing submodule: $submodule_path"$'\e[0m' + continue + fi + + # check for ssh submodule_credentials + submodule_cred=$(jq --arg submodule_url "${submodule_url}" '.source.submodule_credentials // [] | [.[] | select(.url==$submodule_url)] | first // empty' <<< "${payload}") + + if [[ -z ${submodule_cred} ]]; then + + # update normally + git submodule update --init --no-fetch $submodule_parameters "$submodule_path" + + else + + # create or re-initialize ssh-agent + init_ssh_agent + + private_key=$(jq -r '.private_key' <<< ${submodule_cred}) + passphrase=$(jq -r '.private_key_passphrase // empty' <<< ${submodule_cred}) + + private_key_path=$(mktemp -t git-resource-submodule-private-key.XXXXXX) + echo "${private_key}" > ${private_key_path} + chmod 0600 ${private_key_path} + + # add submodule private_key identity + SSH_ASKPASS_REQUIRE=force SSH_ASKPASS=$(dirname $0)/askpass.sh GIT_SSH_PRIVATE_KEY_PASS="$passphrase" DISPLAY= ssh-add $private_key_path > /dev/null + + git submodule update --init --no-fetch $submodule_parameters "$submodule_path" + + # restore main ssh-agent (if needed) + load_pubkey "${payload}" + + fi + + done +fi + +if [ "$ref" == "HEAD" ]; then + return_ref=$(git rev-parse HEAD) +else + return_ref=$ref +fi + +# Store committer email in .git/committer. Can be used to send email to last committer on failed build +# Using https://github.com/mdomke/concourse-email-resource for example +git --no-pager log -1 --pretty=format:"%ae" > .git/committer + +git --no-pager log -1 --pretty=format:"%an" > .git/committer_name + +# Store git-resource returned version ref .git/ref. Useful to know concourse +# pulled ref in following tasks and resources. +echo "${return_ref}" > .git/ref + +metadata=$(git_tag_metadata) +echo "${metadata}" | jq '.' > .git/metadata.json + +# Store short ref with templating. Useful to build Docker images with +# a custom tag +echo "${return_ref}" | cut -c1-7 | awk "{ printf \"${short_ref_format}\", \$1 }" > .git/short_ref + +# Write individual metadata fields to separate files + +# .git/tag - the tag that's been checked out +echo "${metadata}" | jq -r '.[] | select(.name == "tag") | .value' > .git/tag +# .git/author - commit author name +echo "${metadata}" | jq -r '.[] | select(.name == "author") | .value' > .git/author +# .git/author_date - timestamp when the author originally created the commit +echo "${metadata}" | jq -r '.[] | select(.name == "author_date") | .value' > .git/author_date +# .git/url - web URL to view commit (if applicable) +echo "${metadata}" | jq -r '.[] | select(.name == "url") | .value // ""' > .git/url +# .git/committer_date - timestamp when the commit was created in the repository +echo "${metadata}" | jq -r '.[] | select(.name == "committer_date") | .value // ""' > .git/committer_date + +# Store commit message in .git/commit_message. Can be used to inform about +# the content of a successful build. +# Using https://github.com/cloudfoundry-community/slack-notification-resource +# for example +git log -1 --format=format:%B > .git/commit_message + +# Store commit date in .git/commit_timestamp. Can be used for tagging builds +git log -1 --format=%cd --date=${timestamp_format} > .git/commit_timestamp + + +jq -n \ + --arg ref "$return_ref" \ + --arg tag "$tag" \ + --argjson metadata "$metadata" \ + '{ + version: {ref: $ref, tag: $tag}, + metadata: $metadata +}' >&3 diff --git a/assets/out b/assets/out index 3c30d6ad..c294215b 100755 --- a/assets/out +++ b/assets/out @@ -1,12 +1,12 @@ #!/bin/bash -# vim: set ft=sh set -e exec 3>&1 # make stdout available as fd 3 for the result exec 1>&2 # redirect all output to stderr for logging -source $(dirname $0)/common.sh +assets="$(dirname $0)" +source "${assets}/common.sh" source=$1 @@ -15,12 +15,9 @@ if [ -z "$source" ]; then exit 1 fi -# for jq -PATH=/usr/local/bin:$PATH - payload="$(cat <&0)" -unknown_keys=$(jq --slurpfile schema "$(dirname $0)/out_schema.json" '(.params // [] | keys_unsorted) - ($schema[0] | keys_unsorted)' <<< "$payload") +unknown_keys=$(jq --slurpfile schema "${assets}/out_schema.json" '(.params // [] | keys_unsorted) - ($schema[0] | keys_unsorted)' <<< "$payload") if jq --exit-status 'length > 0' <<< "$unknown_keys" &>/dev/null; then echo "Found unknown keys in put params:" @@ -38,235 +35,37 @@ if [[ "$src_debug" == "true" || "$params_debug" == "true" ]]; then export GIT_CURL_VERBOSE=1 fi +git_config_payload=$(jq -r '.source.git_config // []' <<< "$payload") +configure_git_global "${git_config_payload}" load_pubkey "$payload" configure_https_tunnel "$payload" configure_git_ssl_verification "$payload" configure_credentials "$payload" +# These vars are used by multiple version_type's uri=$(jq -r '.source.uri // ""' <<< "$payload") -branch=$(jq -r '.source.branch // ""' <<< "$payload") -git_config_payload=$(jq -r '.source.git_config // []' <<< "$payload") -repository=$(jq -r '.params.repository // ""' <<< "$payload") -tag=$(jq -r '.params.tag // ""' <<< "$payload") -tag_prefix=$(jq -r '.params.tag_prefix // ""' <<< "$payload") -rebase=$(jq -r '.params.rebase // false' <<< "$payload") -rebase_strategy=$(jq -r '.params.rebase_strategy // ""' <<< "$payload") -rebase_strategy_option=$(jq -r '.params.rebase_strategy_option // ""' <<< "$payload") -merge=$(jq -r '.params.merge // false' <<< "$payload") -returning=$(jq -r '.params.returning // "merged"' <<< "$payload") -force=$(jq -r '.params.force // false' <<< "$payload") -only_tag=$(jq -r '.params.only_tag // false' <<< "$payload") -annotation_file=$(jq -r '.params.annotate // ""' <<< "$payload") -notes_file=$(jq -r '.params.notes // ""' <<< "$payload") -override_branch=$(jq -r '.params.branch // ""' <<< "$payload") -push_options=$(jq -r '.params.push_options // []' <<< "$payload") -# useful for pushing to special ref types like refs/for in gerrit. -refs_prefix=$(jq -r '.params.refs_prefix // "refs/heads"' <<< "$payload") - -configure_git_global "${git_config_payload}" - -if [ -z "$uri" ]; then - echo "invalid payload (missing uri)" - exit 1 -fi - -if [ -z "$branch" ] && [ "$only_tag" != "true" ] && [ -z "$override_branch" ]; then - echo "invalid payload (missing branch)" - exit 1 -fi - -if [ -z "$repository" ]; then - echo "invalid payload (missing repository)" - exit 1 -fi - -if [ "$merge" = "true" ] && [ "$rebase" = "true" ]; then - echo "invalid push strategy (either merge or rebase can be set, but not both)" - exit 1 -fi - -cd $source - -if [ -n "$tag" ] && [ ! -f "$tag" ]; then - echo "tag file '$tag' does not exist" - exit 1 -fi - -if [ -n "$annotation_file" ] && [ ! -f $annotation_file ]; then - echo "annotation file '$annotation_file' does not exist" - exit 1 -fi - -forceflag="" -if [ $force = "true" ]; then - forceflag="--force" -fi - -push_options_flags="" -if [ "$push_options" != "[]" ]; then - push_options_flags=$(echo "$push_options" | jq -r '.[] | "--push-option " + .') -fi - -if [ -n "$override_branch" ]; then - echo "Override $branch with $override_branch" - branch=$override_branch -fi - -tag_name="" -if [ -n "$tag" ]; then - tag_name="$(cat $tag)" -fi - -annotate="" -if [ -n "$annotation_file" ]; then - annotate=" -a -F $source/$annotation_file" -fi - -cd $repository - -tag() { - if [ -n "$tag_name" ]; then - git tag -f "${tag_prefix}${tag_name}" $annotate - fi -} - -push_src_and_tags() { - git push --tags push-target HEAD:$refs_prefix/$branch $forceflag $push_options_flags -} - -push_tags() { - git push --tags push-target $forceflag $push_options_flags -} - -add_and_push_notes() { - if [ -n "$notes_file" ]; then - git notes add -F "../${notes_file}" - git push push-target refs/notes/* $push_options_flags - fi -} - -push_with_result_check() { - # oh god this is really the only way to do this - result_file=$(mktemp $TMPDIR/git-result.XXXXXX) - - echo 0 > $result_file - - { - tag 2>&1 && push_src_and_tags 2>&1 && add_and_push_notes 2>&1 || { - echo $? > $result_file - } - } | tee $TMPDIR/push-failure - - # despite what you may think, the embedded cat does not include the - # trailing linebreak - # - # $() appears to trim it - # - # someone rewrite this please - # - # pull requests welcome - if [ "$(cat $result_file)" = "0" ]; then - echo "pushed" - eval "$1=0" - return - fi - - # failed for reason other than non-fast-forward / fetch-first - if ! grep -q '\[rejected\]\|\[remote rejected\].*cannot lock ref' $TMPDIR/push-failure; then - echo "failed with non-rebase error" - eval "$1=1" - return - fi - - eval "$1=2" -} - -git remote add push-target $uri -commit_to_push=$(git rev-parse HEAD) - -if [ "$only_tag" = "true" ]; then - tag - push_tags -elif [ "$merge" = "true" ]; then - while true; do - echo "merging..." - - git reset --hard $commit_to_push - - git fetch push-target "refs/notes/*:refs/notes/*" - git pull --no-edit push-target $branch - - result="0" - push_with_result_check result - if [ "$result" = "0" ]; then - break - elif [ "$result" = "1" ]; then - exit 1 - fi - - echo "merging and trying again..." - done -elif [ "$rebase" = "true" ]; then - rebase_cmd="git pull --rebase=merges" - - # Add strategy if specified - if [ -n "$rebase_strategy" ]; then - rebase_cmd="$rebase_cmd --strategy=$rebase_strategy" - fi - - # Add strategy options if specified, can be string or array - if [ -n "$rebase_strategy_option" ] && [ "$rebase_strategy_option" != "null" ]; then - # Check if it's an array or string - if echo "$rebase_strategy_option" | jq -e 'type == "array"' > /dev/null 2>&1; then - # It's already an array from jq - strategy_options=$(echo "$rebase_strategy_option" | jq -r '.[] | "-X" + .') - for opt in $strategy_options; do - rebase_cmd="$rebase_cmd $opt" - done - else - # It's a string - could be space-separated options - for opt in $rebase_strategy_option; do - rebase_cmd="$rebase_cmd -X$opt" - done - fi - fi - - while true; do - echo "rebasing using rebase command: $rebase_cmd push-target $branch..." - - git fetch push-target "refs/notes/*:refs/notes/*" - $rebase_cmd push-target $branch - - result="0" - push_with_result_check result - if [ "$result" = "0" ]; then - break - elif [ "$result" = "1" ]; then - exit 1 - fi - - echo "rebasing and trying again..." - done -else - tag - push_src_and_tags - add_and_push_notes -fi - -if [ "$merge" = "true" ] && [ "$returning" = "unmerged" ]; then - version_ref="$(echo "$commit_to_push" | jq -R .)" -else - version_ref="$(git rev-parse HEAD | jq -R .)" +uri=${uri# } +if [[ -z "$uri" ]]; then + echo "source.uri is required and must not be empty" + exit 1 fi -if [ -n "$override_branch" ]; then - jq -n "{ - version: {branch: $(echo $override_branch | jq -R .), ref: $version_ref}, - metadata: $(git_metadata) - }" >&3 -else - jq -n "{ - version: {ref: $version_ref}, - metadata: $(git_metadata) - }" >&3 -fi +version_type=$(jq -r '.source.version_type // "commits"' <<< "$payload") + +case "$version_type" in + commits) + source "${assets}/out_commits.sh" + ;; + tags) + echo "version_type 'tags' does not support the put step. Use version_type 'commits' if you want to push tags." + exit 1 + ;; + branches) + echo "version_type 'branches' does not support the put step" + exit 1 + ;; + *) + echo "unknown version_type: $version_type" + exit 1 + ;; +esac diff --git a/assets/out_commits.sh b/assets/out_commits.sh new file mode 100755 index 00000000..9b36ef52 --- /dev/null +++ b/assets/out_commits.sh @@ -0,0 +1,218 @@ +branch=$(jq -r '.source.branch // ""' <<< "$payload") +repository=$(jq -r '.params.repository // ""' <<< "$payload") +tag=$(jq -r '.params.tag // ""' <<< "$payload") +tag_prefix=$(jq -r '.params.tag_prefix // ""' <<< "$payload") +rebase=$(jq -r '.params.rebase // false' <<< "$payload") +rebase_strategy=$(jq -r '.params.rebase_strategy // ""' <<< "$payload") +rebase_strategy_option=$(jq -r '.params.rebase_strategy_option // ""' <<< "$payload") +merge=$(jq -r '.params.merge // false' <<< "$payload") +returning=$(jq -r '.params.returning // "merged"' <<< "$payload") +force=$(jq -r '.params.force // false' <<< "$payload") +only_tag=$(jq -r '.params.only_tag // false' <<< "$payload") +annotation_file=$(jq -r '.params.annotate // ""' <<< "$payload") +notes_file=$(jq -r '.params.notes // ""' <<< "$payload") +override_branch=$(jq -r '.params.branch // ""' <<< "$payload") +push_options=$(jq -r '.params.push_options // []' <<< "$payload") +# useful for pushing to special ref types like refs/for in gerrit. +refs_prefix=$(jq -r '.params.refs_prefix // "refs/heads"' <<< "$payload") + +if [ -z "$branch" ] && [ "$only_tag" != "true" ] && [ -z "$override_branch" ]; then + echo "invalid payload. Must specify one of: source.branch, params.branch, or set params.only_tag=true" + exit 1 +fi + +if [ -z "$repository" ]; then + echo "invalid payload (missing params.repository)" + exit 1 +fi + +if [ "$merge" = "true" ] && [ "$rebase" = "true" ]; then + echo "invalid push strategy (either merge or rebase can be set, but not both)" + exit 1 +fi + +cd $source + +if [ -n "$tag" ] && [ ! -f "$tag" ]; then + echo "tag file '$tag' does not exist" + exit 1 +fi + +if [ -n "$annotation_file" ] && [ ! -f $annotation_file ]; then + echo "annotation file '$annotation_file' does not exist" + exit 1 +fi + +forceflag="" +if [ $force = "true" ]; then + forceflag="--force" +fi + +push_options_flags="" +if [ "$push_options" != "[]" ]; then + push_options_flags=$(echo "$push_options" | jq -r '.[] | "--push-option " + .') +fi + +if [ -n "$override_branch" ]; then + echo "Override $branch with $override_branch" + branch=$override_branch +fi + +tag_name="" +if [ -n "$tag" ]; then + tag_name="$(cat $tag)" +fi + +annotate="" +if [ -n "$annotation_file" ]; then + annotate=" -a -F $source/$annotation_file" +fi + +cd $repository + +tag() { + if [ -n "$tag_name" ]; then + git tag -f "${tag_prefix}${tag_name}" $annotate + fi +} + +push_src_and_tags() { + git push --tags push-target HEAD:$refs_prefix/$branch $forceflag $push_options_flags +} + +push_tags() { + git push --tags push-target $forceflag $push_options_flags +} + +add_and_push_notes() { + if [ -n "$notes_file" ]; then + git notes add -F "../${notes_file}" + git push push-target refs/notes/* $push_options_flags + fi +} + +push_with_result_check() { + # oh god this is really the only way to do this + result_file=$(mktemp $TMPDIR/git-result.XXXXXX) + + echo 0 > $result_file + + { + tag 2>&1 && push_src_and_tags 2>&1 && add_and_push_notes 2>&1 || { + echo $? > $result_file + } + } | tee $TMPDIR/push-failure + + # despite what you may think, the embedded cat does not include the + # trailing linebreak + # + # $() appears to trim it + # + # someone rewrite this please + # + # pull requests welcome + if [ "$(cat $result_file)" = "0" ]; then + echo "pushed" + eval "$1=0" + return + fi + + # failed for reason other than non-fast-forward / fetch-first + if ! grep -q '\[rejected\]\|\[remote rejected\].*cannot lock ref' $TMPDIR/push-failure; then + echo "failed with non-rebase error" + eval "$1=1" + return + fi + + eval "$1=2" +} + +git remote add push-target $uri +commit_to_push=$(git rev-parse HEAD) + +if [ "$only_tag" = "true" ]; then + tag + push_tags +elif [ "$merge" = "true" ]; then + while true; do + echo "merging..." + + git reset --hard $commit_to_push + + git fetch push-target "refs/notes/*:refs/notes/*" + git pull --no-edit push-target $branch + + result="0" + push_with_result_check result + if [ "$result" = "0" ]; then + break + elif [ "$result" = "1" ]; then + exit 1 + fi + + echo "merging and trying again..." + done +elif [ "$rebase" = "true" ]; then + rebase_cmd="git pull --rebase=merges" + + # Add strategy if specified + if [ -n "$rebase_strategy" ]; then + rebase_cmd="$rebase_cmd --strategy=$rebase_strategy" + fi + + # Add strategy options if specified, can be string or array + if [ -n "$rebase_strategy_option" ] && [ "$rebase_strategy_option" != "null" ]; then + # Check if it's an array or string + if echo "$rebase_strategy_option" | jq -e 'type == "array"' > /dev/null 2>&1; then + # It's already an array from jq + strategy_options=$(echo "$rebase_strategy_option" | jq -r '.[] | "-X" + .') + for opt in $strategy_options; do + rebase_cmd="$rebase_cmd $opt" + done + else + # It's a string - could be space-separated options + for opt in $rebase_strategy_option; do + rebase_cmd="$rebase_cmd -X$opt" + done + fi + fi + + while true; do + echo "rebasing using rebase command: $rebase_cmd push-target $branch..." + + git fetch push-target "refs/notes/*:refs/notes/*" + $rebase_cmd push-target $branch + + result="0" + push_with_result_check result + if [ "$result" = "0" ]; then + break + elif [ "$result" = "1" ]; then + exit 1 + fi + + echo "rebasing and trying again..." + done +else + tag + push_src_and_tags + add_and_push_notes +fi + +if [ "$merge" = "true" ] && [ "$returning" = "unmerged" ]; then + version_ref="$(echo "$commit_to_push" | jq -R .)" +else + version_ref="$(git rev-parse HEAD | jq -R .)" +fi + +if [ -n "$override_branch" ]; then + jq -n "{ + version: {branch: $(echo $override_branch | jq -R .), ref: $version_ref}, + metadata: $(git_metadata) + }" >&3 +else + jq -n "{ + version: {ref: $version_ref}, + metadata: $(git_metadata) + }" >&3 +fi diff --git a/assets/source_schema.json b/assets/source_schema.json index 9885c10e..b9fa8775 100644 --- a/assets/source_schema.json +++ b/assets/source_schema.json @@ -1,4 +1,5 @@ { + "version_type": "", "uri": "", "branch": "", "private_key": "", @@ -26,5 +27,9 @@ "commit_filter": "", "version_depth": "", "search_remote_refs": "", - "debug": "" + "debug": "", + "branch_filters": "", + "branch_regex": "", + "tag_filters": "", + "tag_sort": "" } diff --git a/test/all.sh b/test/all.sh index 43a34736..1ce25d6d 100755 --- a/test/all.sh +++ b/test/all.sh @@ -4,6 +4,10 @@ set -e $(dirname $0)/image.sh $(dirname $0)/check.sh +$(dirname $0)/check_branches.sh +$(dirname $0)/check_tags.sh +$(dirname $0)/get_branches.sh +$(dirname $0)/get_tags.sh $(dirname $0)/common.sh $(dirname $0)/get.sh $(dirname $0)/put.sh diff --git a/test/check_branches.sh b/test/check_branches.sh new file mode 100755 index 00000000..ee0e21e2 --- /dev/null +++ b/test/check_branches.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +set -e + +source $(dirname $0)/helpers.sh + +it_returns_all_branches() { + # init_repo already creates master and bogus; add a third branch + local repo=$(init_repo) + make_commit_to_branch $repo develop >/dev/null + + check_uri_with_branches $repo | jq -e ' + .[0].branches == "bogus,develop,master" + ' +} + +it_sorts_branches() { + local repo=$(init_repo) + make_commit_to_branch $repo feat/qwer >/dev/null + make_commit_to_branch $repo feat/hjkl >/dev/null + make_commit_to_branch $repo feat/wasd >/dev/null + make_commit_to_branch $repo feat/abcd >/dev/null + + # bogus and master are created by init_repo and sort around the feat/* branches + check_uri_with_branches $repo | jq -e ' + .[0].branches == "bogus,feat/abcd,feat/hjkl,feat/qwer,feat/wasd,master" + ' +} + +it_errors_if_branch_filters_and_branch_regex_are_set() { + local repo=$(init_repo) + local failed_output=$TMPDIR/filters-and-regex-output + + if check_uri_with_branch_filters_and_regex $repo "feat/.*" "feat/*" 2>"$failed_output"; then + echo "checking should have failed" + return 1 + fi + + grep "only one of branch_filters or branch_regex can be specified" "$failed_output" +} + +it_uses_all_branch_filters() { + local repo=$(init_repo) + make_commit_to_branch $repo issue/hjkl >/dev/null + make_commit_to_branch $repo feat/oiuy >/dev/null + make_commit_to_branch $repo bug/876 >/dev/null + make_commit_to_branch $repo refactor/ui >/dev/null + + check_uri_with_branch_filters $repo "bug/*" "issue/*" | jq -e ' + .[0].branches == "bug/876,issue/hjkl" + ' +} + +it_uses_branch_regex() { + local repo=$(init_repo) + make_commit_to_branch $repo issue/hjkl >/dev/null + make_commit_to_branch $repo feat/oiuy >/dev/null + make_commit_to_branch $repo bug/876 >/dev/null + make_commit_to_branch $repo refactor/ui >/dev/null + + check_uri_with_branch_regex $repo '(refactor\/|feat\/).*' | jq -e ' + .[0].branches == "feat/oiuy,refactor/ui" + ' +} + +it_returns_empty_array_when_no_new_branches() { + # init_repo already creates two branches: master and bogus + local repo=$(init_repo) + + local branches=$(check_uri_with_branches $repo | jq -r '.[0].branches') + + check_uri_with_branches_from $repo "$branches" | jq -e '. == []' +} + +it_returns_none_when_no_branches_found() { + local repo=$(init_repo) + + # only master and bogus branch exist, therefore no matching branches should be found + check_uri_with_branch_filters $repo "issue/*" | jq -e ' + .[0].branches == "NONE" + ' +} + +run it_returns_all_branches +run it_sorts_branches +run it_errors_if_branch_filters_and_branch_regex_are_set +run it_uses_all_branch_filters +run it_uses_branch_regex +run it_returns_empty_array_when_no_new_branches +run it_returns_none_when_no_branches_found diff --git a/test/check_tags.sh b/test/check_tags.sh new file mode 100755 index 00000000..bd6e1947 --- /dev/null +++ b/test/check_tags.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +set -e + +source $(dirname $0)/helpers.sh + +it_gets_all_tags() { + local repo=$(init_repo) + make_annotated_tag $repo foo "tag foo" true >/dev/null + make_annotated_tag $repo bar "tag bar" true >/dev/null + make_annotated_tag $repo wasd "tag wasd" true >/dev/null + make_annotated_tag $repo other "tag other" true >/dev/null + + check_uri_with_tags $repo | jq -e 'map(.tag) == ["foo","bar","wasd","other"]' +} + +it_uses_tag_filter() { + local repo=$(init_repo) + make_annotated_tag $repo foo-1 "tag foo-1" true >/dev/null + make_annotated_tag $repo foo-2 "tag foo-2" true >/dev/null + make_annotated_tag $repo wasd "tag wasd" true >/dev/null + make_annotated_tag $repo 3-foo "tag 3-foo" true >/dev/null + + check_uri_with_tags_filter $repo "foo-*" | jq -e 'map(.tag) == ["foo-1","foo-2"]' +} + +it_uses_tag_filters() { + local repo=$(init_repo) + make_annotated_tag $repo foo-1 "tag foo-1" true >/dev/null + make_annotated_tag $repo foo-2 "tag foo-2" true >/dev/null + make_annotated_tag $repo wasd "tag wasd" true >/dev/null + make_annotated_tag $repo 3-foo "tag 3-foo" true >/dev/null + + check_uri_with_tags_filters $repo "foo-*" | jq -e 'map(.tag) == ["foo-1","foo-2"]' +} + +it_combines_tag_filter_and_tag_filters() { + local repo=$(init_repo) + make_annotated_tag $repo foo-1 "tag foo-1" true >/dev/null + make_annotated_tag $repo foo-2 "tag foo-2" true >/dev/null + make_annotated_tag $repo wasd "tag wasd" true >/dev/null + make_annotated_tag $repo 3-foo "tag 3-foo" true >/dev/null + + check_uri_with_tags_filter_and_filters $repo "*-foo" "foo-*" \ + | jq -e 'map(.tag) == ["foo-1","foo-2","3-foo"]' +} + +it_uses_tag_regex() { + local repo=$(init_repo) + make_annotated_tag $repo foo-1 "tag foo-1" true >/dev/null + make_annotated_tag $repo foo-2 "tag foo-2" true >/dev/null + make_annotated_tag $repo wasd "tag wasd" true >/dev/null + make_annotated_tag $repo 3-foo "tag 3-foo" true >/dev/null + + check_uri_with_tags_regex $repo "foo-.*" | jq -e 'map(.tag) == ["foo-1","foo-2"]' +} + +it_sorts_by_semver() { + local repo=$(init_repo) + make_annotated_tag $repo v1.1.0 "tag v1.1.0" >/dev/null + make_annotated_tag $repo v1.2.0 "tag v1.2.0" >/dev/null + make_annotated_tag $repo v1.1.1 "tag v1.1.1" >/dev/null + make_annotated_tag $repo v1.1.2 "tag v1.1.2" >/dev/null + make_annotated_tag $repo v1.2.1 "tag v1.2.1" >/dev/null + + check_uri_with_tags_sort $repo "semver" \ + | jq -e 'map(.tag) == ["v1.1.0","v1.1.1","v1.1.2","v1.2.0","v1.2.1"]' +} + +it_returns_new_tags() { + local repo=$(init_repo) + make_annotated_tag $repo v1.1.0 "tag v1.1.0" >/dev/null + make_annotated_tag $repo v1.2.0 "tag v1.2.0" >/dev/null + make_annotated_tag $repo v1.1.1 "tag v1.1.1" >/dev/null + make_annotated_tag $repo v1.1.2 "tag v1.1.2" >/dev/null + make_annotated_tag $repo v1.2.1 "tag v1.2.1" >/dev/null + + check_uri_with_tags_sort_from $repo "semver" "v1.2.0" \ + | jq -e 'map(.tag) == ["v1.2.0","v1.2.1"]' +} + +it_finds_no_tags() { + local repo=$(init_repo) + + check_uri_with_tags $repo | jq -e '. == []' +} + +it_returns_no_tags_due_to_filtering() { + local repo=$(init_repo) + make_annotated_tag $repo foo-1 "tag foo-1" true >/dev/null + + check_uri_with_tags_filter $repo "nomatch*" | jq -e '. == []' +} + +run it_gets_all_tags +run it_uses_tag_filter +run it_uses_tag_filters +run it_combines_tag_filter_and_tag_filters +run it_uses_tag_regex +run it_sorts_by_semver +run it_returns_new_tags +run it_finds_no_tags +run it_returns_no_tags_due_to_filtering diff --git a/test/get.sh b/test/get.sh index 9875622a..331a0b1c 100755 --- a/test/get.sh +++ b/test/get.sh @@ -1075,12 +1075,11 @@ it_returns_list_of_all_tags_in_metadata() { local ref1=$(make_commit_to_branch $repo branch-a) local ref2=$(make_annotated_tag $repo "v1.1-pre" "tag 1") local ref3=$(make_annotated_tag $repo "v1.1-final" "tag 2") - local ref4=$(make_commit_to_branch $repo branch-b) - local ref5=$(make_annotated_tag $repo "v1.1-branch-b" "tag 3") + local ref4=$(make_annotated_tag $repo "v1.1-branch-b" "tag 3") local dest=$TMPDIR/destination get_uri_at_branch_with_fetch_tags $repo branch-a $dest | jq -e " - .version == {ref: $(echo $ref4 | jq -R .)} + .version == {ref: $(echo $ref1 | jq -R .)} and (.metadata | .[] | select(.name == \"tags\") | .value == \"v1.1-branch-b,v1.1-final,v1.1-pre\") " diff --git a/test/get_branches.sh b/test/get_branches.sh new file mode 100755 index 00000000..b982d2d3 --- /dev/null +++ b/test/get_branches.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -e + +source $(dirname $0)/helpers.sh + +it_saves_the_given_branches() { + local dest=$TMPDIR/destination + mkdir -p $dest + + get_branches "some-uri" "feat/foo,feat/bar,issue/qwerty" $dest | jq -e ' + .version.branches == "feat/foo,feat/bar,issue/qwerty" + ' + + jq -e '. == ["feat/foo","feat/bar","issue/qwerty"]' < $dest/branches.json +} + +it_saves_a_single_branch() { + local dest=$TMPDIR/destination + mkdir -p $dest + + get_branches "some-uri" "feat/foo" $dest | jq -e ' + .version.branches == "feat/foo" + ' + + jq -e '. == ["feat/foo"]' < $dest/branches.json +} + +it_saves_empty_array_when_version_is_none() { + local dest=$TMPDIR/destination + mkdir -p $dest + + get_branches "some-uri" "NONE" $dest + + jq -e '. == []' < $dest/branches.json +} + +it_saves_metadata_as_multiline_string() { + local dest=$TMPDIR/destination + mkdir -p $dest + + get_branches "some-uri" "feat/foo,feat/bar,issue/qwerty" $dest | jq -e ' + (.metadata[] | select(.name == "branches") | .value) == "feat/foo\nfeat/bar\nissue/qwerty" + ' +} + +run it_saves_the_given_branches +run it_saves_a_single_branch +run it_saves_empty_array_when_version_is_none +run it_saves_metadata_as_multiline_string diff --git a/test/get_tags.sh b/test/get_tags.sh new file mode 100755 index 00000000..5ee3f010 --- /dev/null +++ b/test/get_tags.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e + +source $(dirname $0)/helpers.sh + +it_clones_the_repo_at_the_given_tag() { + local repo=$(init_repo) + make_commit $repo >/dev/null + make_annotated_tag $repo v1 "tag v1" >/dev/null + local tag_ref=$(git -C $repo rev-parse v1^{commit}) + + local dest=$TMPDIR/destination + + get_tags $repo v1 $dest | jq -e '.version.tag == "v1"' + + ! git -C $dest symbolic-ref -q HEAD + + test "$(git -C $dest rev-parse HEAD)" = "$tag_ref" +} + +it_saves_the_tag_metadata() { + local repo=$(init_repo) + make_commit $repo >/dev/null + make_annotated_tag $repo v1 "tag v1" >/dev/null + + local dest=$TMPDIR/destination + + get_tags $repo v1 $dest + + jq -e 'any(.name == "tag" and .value == "v1")' < $dest/.git/metadata.json +} + +run it_clones_the_repo_at_the_given_tag +run it_saves_the_tag_metadata diff --git a/test/helpers.sh b/test/helpers.sh index f834822e..a60e30f1 100644 --- a/test/helpers.sh +++ b/test/helpers.sh @@ -40,18 +40,22 @@ init_repo() { git config maintenance.auto false # start with an initial commit - git \ + local commit_date=$(_test_seq_date $(_test_next_seq .)) + GIT_COMMITTER_DATE="$commit_date" git \ -c user.name='test' \ -c user.email='test@example.com' \ - commit -q --allow-empty -m "init" + commit -q --allow-empty -m "init" \ + --date "$commit_date" # create some bogus branch git checkout -q -b bogus - git \ + commit_date=$(_test_seq_date $(_test_next_seq .)) + GIT_COMMITTER_DATE="$commit_date" git \ -c user.name='test' \ -c user.email='test@example.com' \ - commit -q --allow-empty -m "commit on other branch" + commit -q --allow-empty -m "commit on other branch" \ + --date "$commit_date" # back to master git checkout -q master @@ -127,6 +131,23 @@ fetch_head_ref() { git -C $repo rev-parse HEAD } +# Per-repo monotonic counter used to give commits and waited tags distinct, +# deterministic timestamps. git's committer-date and tag creator-date have +# 1-second resolution; tests previously got distinct timestamps with `sleep 1`, +# which made the suite slow. The counter is persisted to a file so it survives +# the command-substitution subshells callers use. +_test_next_seq() { + local repo=$1 + local seq_file="$repo/.git/.test_seq" + local seq=$(( $(cat "$seq_file" 2>/dev/null || echo 0) + 1 )) + echo "$seq" > "$seq_file" + echo "$seq" +} + +_test_seq_date() { + date -u -d "@$(( 946684800 + $1 ))" "+%Y-%m-%d %H:%M:%S +0000" +} + make_commit_to_file_on_branch() { local repo=$1 local file=$2 @@ -134,7 +155,7 @@ make_commit_to_file_on_branch() { local msg=${4-} # ensure branch exists - if ! git -C $repo rev-parse --verify $branch >/dev/null; then + if ! git -C $repo rev-parse --verify $branch &>/dev/null; then git -C $repo branch $branch master fi @@ -154,10 +175,12 @@ make_commit_to_file_on_branch() { commit -q -m "commit $(wc -l $repo/$file) $msg" \ --date "$(date -R -d '1 year')" else - git -C $repo \ + local commit_date=$(_test_seq_date $(_test_next_seq $repo)) + GIT_COMMITTER_DATE="$commit_date" git -C $repo \ -c user.name='test' \ -c user.email='test@example.com' \ - commit -q -m "commit $(wc -l $repo/$file) $msg" + commit -q -m "commit $(wc -l $repo/$file) $msg" \ + --date "$commit_date" fi # output resulting sha @@ -172,7 +195,7 @@ make_commit_to_file_on_branch_with_path() { local msg=${5-} # ensure branch exists - if ! git -C $repo rev-parse --verify $branch >/dev/null; then + if ! git -C $repo rev-parse --verify $branch &>/dev/null; then git -C $repo branch $branch master fi @@ -183,10 +206,12 @@ make_commit_to_file_on_branch_with_path() { mkdir -p $repo/$path echo x >> $repo/$path/$file git -C $repo add $path/$file - git -C $repo \ + local commit_date=$(_test_seq_date $(_test_next_seq $repo)) + GIT_COMMITTER_DATE="$commit_date" git -C $repo \ -c user.name='test' \ -c user.email='test@example.com' \ - commit -q -m "commit $(wc -l $repo/$path/$file) $msg" + commit -q -m "commit $(wc -l $repo/$path/$file) $msg" \ + --date "$commit_date" # output resulting sha git -C $repo rev-parse HEAD @@ -233,7 +258,7 @@ merge_branch() { } delete_public_key() { - if gpg -k ${fingerprint} > /dev/null; then + if gpg -k ${fingerprint} &> /dev/null; then gpg --batch --yes --delete-keys ${fingerprint} fi } @@ -254,10 +279,12 @@ make_empty_commit() { local repo=$1 local msg=${2-} - git -C $repo \ + local commit_date=$(_test_seq_date $(_test_next_seq $repo)) + GIT_COMMITTER_DATE="$commit_date" git -C $repo \ -c user.name='test' \ -c user.email='test@example.com' \ - commit -q --allow-empty -m "commit $msg" + commit -q --allow-empty -m "commit $msg" \ + --date "$commit_date" # output resulting sha git -C $repo rev-parse HEAD @@ -269,14 +296,17 @@ make_annotated_tag() { local msg=$3 local wait=${4:-false} - git -C $repo tag -f -a "$tag" -m "$msg" - - git -C $repo describe --tags --abbrev=0 - if [ "$wait" == true ]; then - # Ensure creation date difference between tags - git does not sort with sub-second accuracy. - sleep 1 + # Give each successive waited tag a distinct, increasing creation date so tags + # sort deterministically. The shared per-repo counter (see _test_next_seq) is + # also bumped by commits, so commit and tag ordering interleaves correctly. + GIT_COMMITTER_DATE="$(_test_seq_date $(_test_next_seq $repo))" \ + git -C $repo tag -f -a "$tag" -m "$msg" + else + git -C $repo tag -f -a "$tag" -m "$msg" fi + + git -C $repo describe --tags --abbrev=0 } check_uri() { @@ -722,6 +752,154 @@ check_uri_with_filters() { }" | ${resource_dir}/check | tee /dev/stderr } +check_uri_with_branches() { + jq -n "{ + source: { + uri: $(echo $1 | jq -R .), + version_type: \"branches\" + } + }" | ${resource_dir}/check | tee /dev/stderr +} + +check_uri_with_branches_from() { + local uri=$1 + local prev_branches=$2 + jq -n "{ + source: { + uri: $(echo $uri | jq -R .), + version_type: \"branches\" + }, + version: { + branches: $(echo "$prev_branches" | jq -R .) + } + }" | ${resource_dir}/check | tee /dev/stderr +} + +check_uri_with_branch_filters() { + local uri=$1 + shift + jq -n "{ + source: { + uri: $(echo $uri | jq -R .), + version_type: \"branches\", + branch_filters: $(echo "$@" | jq -R '. | split(" ")') + } + }" | ${resource_dir}/check | tee /dev/stderr +} + +check_uri_with_branch_regex() { + local uri=$1 + local branch_regex=$2 + jq -n "{ + source: { + uri: $(echo $uri | jq -R .), + version_type: \"branches\", + branch_regex: $(echo "$branch_regex" | jq -R .) + } + }" | ${resource_dir}/check | tee /dev/stderr +} + +check_uri_with_branch_filters_and_regex() { + local uri=$1 + local branch_regex=$2 + shift 2 + jq -n "{ + source: { + uri: $(echo $uri | jq -R .), + version_type: \"branches\", + branch_filters: $(echo "$@" | jq -R '. | split(" ")'), + branch_regex: $(echo "$branch_regex" | jq -R .) + } + }" | ${resource_dir}/check | tee /dev/stderr +} + +check_uri_with_tags() { + jq -n "{ + source: { + uri: $(echo $1 | jq -R .), + version_type: \"tags\" + } + }" | ${resource_dir}/check | tee /dev/stderr +} + +check_uri_with_tags_filter() { + local uri=$1 + local tag_filter=$2 + jq -n "{ + source: { + uri: $(echo $uri | jq -R .), + version_type: \"tags\", + tag_filter: $(echo "$tag_filter" | jq -R .) + } + }" | ${resource_dir}/check | tee /dev/stderr +} + +check_uri_with_tags_filters() { + local uri=$1 + shift + jq -n "{ + source: { + uri: $(echo $uri | jq -R .), + version_type: \"tags\", + tag_filters: $(echo "$@" | jq -R '. | split(" ")') + } + }" | ${resource_dir}/check | tee /dev/stderr +} + +check_uri_with_tags_filter_and_filters() { + local uri=$1 + local tag_filter=$2 + shift 2 + jq -n "{ + source: { + uri: $(echo $uri | jq -R .), + version_type: \"tags\", + tag_filter: $(echo "$tag_filter" | jq -R .), + tag_filters: $(echo "$@" | jq -R '. | split(" ")') + } + }" | ${resource_dir}/check | tee /dev/stderr +} + +check_uri_with_tags_regex() { + local uri=$1 + local tag_regex=$2 + jq -n "{ + source: { + uri: $(echo $uri | jq -R .), + version_type: \"tags\", + tag_regex: $(echo "$tag_regex" | jq -R .) + } + }" | ${resource_dir}/check | tee /dev/stderr +} + +check_uri_with_tags_sort() { + local uri=$1 + local tag_sort=$2 + jq -n "{ + source: { + uri: $(echo $uri | jq -R .), + version_type: \"tags\", + tag_sort: $(echo "$tag_sort" | jq -R .) + } + }" | ${resource_dir}/check | tee /dev/stderr +} + +check_uri_with_tags_sort_from() { + local uri=$1 + local tag_sort=$2 + local prev_tag=$3 + jq -n "{ + source: { + uri: $(echo $uri | jq -R .), + version_type: \"tags\", + tag_sort: $(echo "$tag_sort" | jq -R .) + }, + version: { + tag: $(echo "$prev_tag" | jq -R .) + } + }" | ${resource_dir}/check | tee /dev/stderr +} + get_uri() { jq -n "{ source: { @@ -1114,6 +1292,38 @@ get_uri_with_fetch_branches() { }" | ${resource_dir}/in "$dest" | tee /dev/stderr } +get_branches() { + local uri=$1 + local branches=$2 + local dest=$3 + + jq -n "{ + source: { + uri: $(echo $uri | jq -R .), + version_type: \"branches\" + }, + version: { + branches: $(echo $branches | jq -R .) + } + }" | ${resource_dir}/in "$dest" | tee /dev/stderr +} + +get_tags() { + local uri=$1 + local tag=$2 + local dest=$3 + + jq -n "{ + source: { + uri: $(echo $uri | jq -R .), + version_type: \"tags\" + }, + version: { + tag: $(echo $tag | jq -R .) + } + }" | ${resource_dir}/in "$dest" | tee /dev/stderr +} + get_uri_with_all_branches() { jq -n "{ source: {