From e9bd21d8c63e20d2264f20ef53f39bb829d06e4c Mon Sep 17 00:00:00 2001
From: RobHooper <robh@linux.com>
Date: Tue, 12 May 2026 15:10:10 +0100
Subject: [PATCH 1/6] refactor: sync per directory for added configuration

---
 pillar/registry.sls          |  1 +
 salt/aws/files/sync-to-s3.sh | 44 ++++++++++++++++++++++++++++--------
 salt/aws/sync.sls            | 16 ++++---------
 3 files changed, 41 insertions(+), 20 deletions(-)

diff --git a/pillar/registry.sls b/pillar/registry.sls
index 9736888e..b8e1d9f5 100644
--- a/pillar/registry.sls
+++ b/pillar/registry.sls
@@ -55,6 +55,7 @@ sync:
     /home/collect/scrapyd/dbs:
     /home/collect/scrapyd/eggs:
     /home/collect/scrapyd/jobs:
+      exclude: "*/requests.queue/*"
     /home/collect/scrapyd/logs:
 
 apache:
diff --git a/salt/aws/files/sync-to-s3.sh b/salt/aws/files/sync-to-s3.sh
index 96a35d5f..d3a4efc4 100644
--- a/salt/aws/files/sync-to-s3.sh
+++ b/salt/aws/files/sync-to-s3.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Backup multiple directories and upload to AWS S3
+# Sync a single directory into S3.
 
 set -euo pipefail
 
@@ -24,16 +24,42 @@ if [ ! -x "$AWS_CLI" ]; then
     exit 3
 fi
 
-if [ -z "$SYNC_DIRECTORIES" ]; then
-    echo "Error: SYNC_DIRECTORIES isn't set or is empty"
+if [ -z "$S3_SYNC_BUCKET" ]; then
+    echo "Error: S3_SYNC_BUCKET isn't set or is empty"
     exit 4
 fi
 
-for DIRECTORY in "${SYNC_DIRECTORIES[@]}"; do
-    SAFENAME="${DIRECTORY/#\//}"
-    SAFENAME="${SAFENAME/%\//}"
+AWS_ARGS=(--only-show-errors --delete)
 
-    set +e
-    $AWS_CLI s3 sync "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$SAFENAME/" --only-show-errors --delete 2>&1 | grep -v "You did not provide the number of bytes specified by the Content-Length HTTP header"
-    set -e
+# Parse script arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+    --exclude)
+        if [ -z "$2" ]; then
+            echo "Missing path."
+            exit 1
+        fi
+        AWS_ARGS+=(--exclude "\"$2\"")
+        shift 2
+        ;;
+    -*)
+        echo "Unknown option"
+        exit 1
+        ;;
+    *)
+        DIRECTORY=$1
+        SAFENAME=${DIRECTORY/#\//}
+        SAFENAME="${SAFENAME/%\//}"
+        shift
+        ;;
+    esac
 done
+
+if [ -e "$DIRECTORY" ]; then
+    echo "Error: DIRECTORY isn't set or doesn't exist."
+    exit 5
+fi
+
+set +e
+$AWS_CLI s3 sync "${AWS_ARGS[@]}" "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$SAFENAME/"
+set -e
diff --git a/salt/aws/sync.sls b/salt/aws/sync.sls
index be5a4330..f0b9bbb0 100644
--- a/salt/aws/sync.sls
+++ b/salt/aws/sync.sls
@@ -16,16 +16,10 @@ include:
   file.managed:
     - contents: |
         MAILTO=root
-        15 03,15 * * * root /home/sysadmin-tools/bin/sync-to-s3.sh
+{%- for directory, entry in pillar.sync.directories|items %}
+{%- set minute = (loop.index0 * 5) % 60 %}
+        {{minute}} 03,15 * * * root /home/sysadmin-tools/bin/sync-to-s3.sh {{ directory }}
+        {%- if entry and 'exclude' in entry %} --exclude "{{ entry.exclude }}"{% endif %}
+{%- endfor %}
     - require:
       - file: /home/sysadmin-tools/bin/sync-to-s3.sh
-
-set SYNC_DIRECTORIES setting:
-  file.keyvalue:
-    - name: /home/sysadmin-tools/aws-settings.local
-    - key: SYNC_DIRECTORIES
-    - value: '( "{{ pillar.sync.directories|join('" "') }}" )'
-    - append_if_not_found: True
-    - require:
-      - file: /home/sysadmin-tools/bin
-      - sls: aws

From 4a1da63ef4aa123066ac219b55fc092623db69e6 Mon Sep 17 00:00:00 2001
From: RobHooper <robh@linux.com>
Date: Wed, 13 May 2026 08:19:51 +0100
Subject: [PATCH 2/6] feat: dynamic sync options

---
 salt/aws/sync.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/aws/sync.sls b/salt/aws/sync.sls
index f0b9bbb0..8869a57c 100644
--- a/salt/aws/sync.sls
+++ b/salt/aws/sync.sls
@@ -19,7 +19,7 @@ include:
 {%- for directory, entry in pillar.sync.directories|items %}
 {%- set minute = (loop.index0 * 5) % 60 %}
         {{minute}} 03,15 * * * root /home/sysadmin-tools/bin/sync-to-s3.sh {{ directory }}
-        {%- if entry and 'exclude' in entry %} --exclude "{{ entry.exclude }}"{% endif %}
+        {%- for option, value in (entry or {}) | items %} --{{ option }} "{{ value }}"{% endfor %}
 {%- endfor %}
     - require:
       - file: /home/sysadmin-tools/bin/sync-to-s3.sh

From d983d08dec9b864f6293e2ef49f0793148bca9a7 Mon Sep 17 00:00:00 2001
From: RobHooper <robh@linux.com>
Date: Wed, 13 May 2026 15:12:06 +0100
Subject: [PATCH 3/6] feat: pass args directly to aws cli

---
 salt/aws/files/sync-to-s3.sh | 35 ++++++++---------------------------
 1 file changed, 8 insertions(+), 27 deletions(-)

diff --git a/salt/aws/files/sync-to-s3.sh b/salt/aws/files/sync-to-s3.sh
index d3a4efc4..4c188829 100644
--- a/salt/aws/files/sync-to-s3.sh
+++ b/salt/aws/files/sync-to-s3.sh
@@ -29,37 +29,18 @@ if [ -z "$S3_SYNC_BUCKET" ]; then
     exit 4
 fi
 
-AWS_ARGS=(--only-show-errors --delete)
-
-# Parse script arguments
-while [[ $# -gt 0 ]]; do
-    case $1 in
-    --exclude)
-        if [ -z "$2" ]; then
-            echo "Missing path."
-            exit 1
-        fi
-        AWS_ARGS+=(--exclude "\"$2\"")
-        shift 2
-        ;;
-    -*)
-        echo "Unknown option"
-        exit 1
-        ;;
-    *)
-        DIRECTORY=$1
-        SAFENAME=${DIRECTORY/#\//}
-        SAFENAME="${SAFENAME/%\//}"
-        shift
-        ;;
-    esac
-done
-
-if [ -e "$DIRECTORY" ]; then
+if [ ! -e "$1" ]; then
     echo "Error: DIRECTORY isn't set or doesn't exist."
     exit 5
 fi
 
+DIRECTORY=$1
+SAFENAME=${DIRECTORY/#\//}
+SAFENAME="${SAFENAME/%\//}"
+shift
+
+AWS_ARGS=(--only-show-errors --delete "${@}")
+
 set +e
 $AWS_CLI s3 sync "${AWS_ARGS[@]}" "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$SAFENAME/"
 set -e

From 56e25028fd0d1077d92d6448490be8e61744ecee Mon Sep 17 00:00:00 2001
From: "Bob H." <57331626+RobHooper@users.noreply.github.com>
Date: Thu, 14 May 2026 11:50:55 +0100
Subject: [PATCH 4/6] Update salt/aws/files/sync-to-s3.sh

Co-authored-by: James McKinney <26463+jpmckinney@users.noreply.github.com>
---
 salt/aws/files/sync-to-s3.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/aws/files/sync-to-s3.sh b/salt/aws/files/sync-to-s3.sh
index 4c188829..a8a7103c 100644
--- a/salt/aws/files/sync-to-s3.sh
+++ b/salt/aws/files/sync-to-s3.sh
@@ -42,5 +42,5 @@ shift
 AWS_ARGS=(--only-show-errors --delete "${@}")
 
 set +e
-$AWS_CLI s3 sync "${AWS_ARGS[@]}" "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$SAFENAME/"
+$AWS_CLI s3 sync --only-show-errors --delete "${@}" "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$SAFENAME/"
 set -e

From 9effd1a24c2b4aed055a3a7a397cab939778194f Mon Sep 17 00:00:00 2001
From: RobHooper <robh@linux.com>
Date: Thu, 14 May 2026 11:52:16 +0100
Subject: [PATCH 5/6] fix: remove no longer needed var

---
 salt/aws/files/sync-to-s3.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/salt/aws/files/sync-to-s3.sh b/salt/aws/files/sync-to-s3.sh
index a8a7103c..4c6464d5 100644
--- a/salt/aws/files/sync-to-s3.sh
+++ b/salt/aws/files/sync-to-s3.sh
@@ -39,8 +39,6 @@ SAFENAME=${DIRECTORY/#\//}
 SAFENAME="${SAFENAME/%\//}"
 shift
 
-AWS_ARGS=(--only-show-errors --delete "${@}")
-
 set +e
 $AWS_CLI s3 sync --only-show-errors --delete "${@}" "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$SAFENAME/"
 set -e

From b512ae3fdc74ea82c50ac3dff852756e0be0f401 Mon Sep 17 00:00:00 2001
From: RobHooper <robh@linux.com>
Date: Thu, 14 May 2026 11:59:42 +0100
Subject: [PATCH 6/6] style: use default

---
 salt/aws/sync.sls | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/salt/aws/sync.sls b/salt/aws/sync.sls
index 8869a57c..2821ccc1 100644
--- a/salt/aws/sync.sls
+++ b/salt/aws/sync.sls
@@ -19,7 +19,7 @@ include:
 {%- for directory, entry in pillar.sync.directories|items %}
 {%- set minute = (loop.index0 * 5) % 60 %}
         {{minute}} 03,15 * * * root /home/sysadmin-tools/bin/sync-to-s3.sh {{ directory }}
-        {%- for option, value in (entry or {}) | items %} --{{ option }} "{{ value }}"{% endfor %}
+        {%- for option, value in entry | default({}, true) | items %} --{{ option }} "{{ value }}"{% endfor %}
 {%- endfor %}
     - require:
       - file: /home/sysadmin-tools/bin/sync-to-s3.sh