diff --git a/pillar/registry.sls b/pillar/registry.sls index 9736888e..b8e1d9f5 100644 --- a/pillar/registry.sls +++ b/pillar/registry.sls @@ -55,6 +55,7 @@ sync: /home/collect/scrapyd/dbs: /home/collect/scrapyd/eggs: /home/collect/scrapyd/jobs: + exclude: "*/requests.queue/*" /home/collect/scrapyd/logs: apache: diff --git a/salt/aws/files/sync-to-s3.sh b/salt/aws/files/sync-to-s3.sh index 96a35d5f..4c6464d5 100644 --- a/salt/aws/files/sync-to-s3.sh +++ b/salt/aws/files/sync-to-s3.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Backup multiple directories and upload to AWS S3 +# Sync a single directory into S3. set -euo pipefail @@ -24,16 +24,21 @@ if [ ! -x "$AWS_CLI" ]; then exit 3 fi -if [ -z "$SYNC_DIRECTORIES" ]; then - echo "Error: SYNC_DIRECTORIES isn't set or is empty" +if [ -z "$S3_SYNC_BUCKET" ]; then + echo "Error: S3_SYNC_BUCKET isn't set or is empty" exit 4 fi -for DIRECTORY in "${SYNC_DIRECTORIES[@]}"; do - SAFENAME="${DIRECTORY/#\//}" - SAFENAME="${SAFENAME/%\//}" +if [ ! -e "$1" ]; then + echo "Error: DIRECTORY isn't set or doesn't exist." + exit 5 +fi + +DIRECTORY=$1 +SAFENAME=${DIRECTORY/#\//} +SAFENAME="${SAFENAME/%\//}" +shift - set +e - $AWS_CLI s3 sync "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$SAFENAME/" --only-show-errors --delete 2>&1 | grep -v "You did not provide the number of bytes specified by the Content-Length HTTP header" - set -e -done +set +e +$AWS_CLI s3 sync --only-show-errors --delete "${@}" "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$SAFENAME/" +set -e diff --git a/salt/aws/sync.sls b/salt/aws/sync.sls index be5a4330..2821ccc1 100644 --- a/salt/aws/sync.sls +++ b/salt/aws/sync.sls @@ -16,16 +16,10 @@ include: file.managed: - contents: | MAILTO=root - 15 03,15 * * * root /home/sysadmin-tools/bin/sync-to-s3.sh +{%- for directory, entry in pillar.sync.directories|items %} +{%- set minute = (loop.index0 * 5) % 60 %} + {{minute}} 03,15 * * * root /home/sysadmin-tools/bin/sync-to-s3.sh {{ directory }} + {%- for option, value in entry | default({}, true) | items %} --{{ option }} "{{ value }}"{% endfor %} +{%- endfor %} - require: - file: /home/sysadmin-tools/bin/sync-to-s3.sh - -set SYNC_DIRECTORIES setting: - file.keyvalue: - - name: /home/sysadmin-tools/aws-settings.local - - key: SYNC_DIRECTORIES - - value: '( "{{ pillar.sync.directories|join('" "') }}" )' - - append_if_not_found: True - - require: - - file: /home/sysadmin-tools/bin - - sls: aws