Fixed cf-remote config migration on Python < 3.8 by replacing shutil.copytree with a custom helper

.github/workflows/tests.yml

@@ -53,6 +53,8 @@ jobs:
         run: bash tests/docker/0*.sh
       - name: Run unsafe tests
         run: bash tests/unsafe/0*.sh
+      - name: Run shell tests
+        run: bash tests/shell/all.sh
 
   test-legacy:
     runs-on: ubuntu-24.04
@@ -93,3 +95,5 @@ jobs:
         run: bash tests/docker/0*.sh
       - name: Run unsafe tests
         run: bash tests/unsafe/0*.sh
+      - name: Run shell tests
+        run: bash tests/shell/all.sh

cf_remote/utils.py

@@ -341,34 +341,40 @@ def has_unescaped_character(string, char):
     return False
 
 
+def copytree_merge(src, dst, ignore=None):
+    """Backwards-compatible replacement for shutil.copytree(src, dst, dirs_exist_ok=True)"""
+    os.makedirs(dst, exist_ok=True)
+    names = os.listdir(src)
+    ignored_names = ignore(src, names) if ignore else set()
+    for name in names:
+        if name in ignored_names:
+            continue
+        src_path = os.path.join(src, name)
+        dst_path = os.path.join(dst, name)
+        if os.path.isdir(src_path):
+            copytree_merge(src_path, dst_path, ignore=ignore)
+        else:
+            shutil.copy2(src_path, dst_path)
+
+
 def migrate_config_paths():
     old_dir = os.path.expanduser("~/.cfengine/cf-remote/")
     conf_dir = os.path.expanduser("~/.config/cfengine/cf-remote/")
     cache_dir = os.path.expanduser("~/.cache/cfengine/cf-remote/")
     if not os.path.exists(os.path.dirname(old_dir)):
         return  # nothing to migrate
-    if os.path.exists(conf_dir) and os.path.exists(cache_dir):
-        pass  # Migration has already occured
-    else:
-        shutil.copytree(
-            old_dir,
-            conf_dir,
-            dirs_exist_ok=True,
-            ignore=shutil.ignore_patterns("json", "packages"),
-        )
-        print("MIGRATION: config files have been moved to '%s'" % conf_dir)
-
-        shutil.copytree(
-            os.path.join(old_dir, "json"),
-            os.path.join(cache_dir, "json"),
-            dirs_exist_ok=True,
-        )
-        shutil.copytree(
-            os.path.join(old_dir, "packages"),
-            os.path.join(cache_dir, "packages"),
-            dirs_exist_ok=True,
-        )
-        print("MIGRATION: cache files have been moved to '%s'" % cache_dir)
+    copytree_merge(
+        old_dir,
+        conf_dir,
+        ignore=shutil.ignore_patterns("json", "packages"),
+    )
+    print("MIGRATION: config files have been moved to '%s'" % conf_dir)
+
+    for sub in ("json", "packages"):
+        src = os.path.join(old_dir, sub)
+        if os.path.isdir(src):
+            copytree_merge(src, os.path.join(cache_dir, sub))
+    print("MIGRATION: cache files have been moved to '%s'" % cache_dir)
 
     shutil.rmtree(old_dir)
     print("REMOVED: %s" % old_dir)

tests/shell/001_migrate_dirs.sh

@@ -0,0 +1,18 @@
+set -e
+set -x
+
+mkdir -p ~/.cfengine/cf-remote/json/
+mkdir -p ~/.cfengine/cf-remote/packages/
+touch ~/.cfengine/cf-remote/json/test_file.json
+touch ~/.cfengine/cf-remote/packages/test_pkg.tar.gz
+touch ~/.cfengine/cf-remote/test_config
+
+cf-remote -V
+
+! test -e ~/.cfengine/cf-remote
+
+test -f ~/.config/cfengine/cf-remote/test_config
+! test -e ~/.config/cfengine/cf-remote/json
+
+test -f ~/.cache/cfengine/cf-remote/json/test_file.json
+test -f ~/.cache/cfengine/cf-remote/packages/test_pkg.tar.gz

tests/shell/all.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+echo "Warning: These shell based tests use the cf-remote you have installed"
+echo "         If you haven't already, run: pip install ."
+set -e
+
+_passed=0
+_failed=0
+_skipped=0
+_total=0
+_failures=""
+_suite_start=$(date +%s)
+
+run_test() {
+    local test_script="$1"
+    local test_name
+    test_name=$(basename "$test_script" .sh)
+    _total=$((_total + 1))
+
+    local start end elapsed
+    start=$(date +%s)
+
+    local output
+    local exit_code=0
+    output=$(bash "$test_script" 2>&1) || exit_code=$?
+
+    end=$(date +%s)
+    elapsed=$((end - start))
+
+    if [ $exit_code -eq 0 ]; then
+        if echo "$output" | grep -q "^--- SKIP:"; then
+            _skipped=$((_skipped + 1))
+            echo "--- SKIP: $test_name (${elapsed}s)"
+        else
+            _passed=$((_passed + 1))
+            echo "--- PASS: $test_name (${elapsed}s)"
+        fi
+    else
+        _failed=$((_failed + 1))
+        _failures="${_failures}  ${test_name}\n"
+        echo "--- FAIL: $test_name (${elapsed}s)"
+        echo "$output"
+        echo "---"
+    fi
+}
+
+run_test tests/shell/001_migrate_dirs.sh
+
+# Summary
+_suite_end=$(date +%s)
+_suite_elapsed=$((_suite_end - _suite_start))
+echo ""
+echo "=============================="
+echo "Test Results: $_passed passed, $_failed failed, $_skipped skipped (total: $_total, ${_suite_elapsed}s)"
+if [ $_failed -gt 0 ]; then
+    echo ""
+    echo "Failed tests:"
+    echo -e "$_failures"
+    exit 1
+fi
+echo "=============================="
+echo "All cf-remote shell tests completed successfully!"