Skip to content

Commit 233b416

Browse files
skulidropekskulidropek
authored
feat: add shared mirror cache for repeated repository clones (#51)
* feat(clone): add shared mirror cache for repeated repo clones * fix(lib): reduce state gitignore sync complexity for lint * test(e2e): verify clone mirror cache reuse * fix(e2e): detect mirror path from generated cache directory * fix(e2e): assert mirror path under projects root cache
1 parent e84b64e commit 233b416

8 files changed

Lines changed: 395 additions & 10 deletions

File tree

‎.github/workflows/check.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,19 @@ jobs:
9898
- name: OpenCode autoconnect
9999
run: bash scripts/e2e/opencode-autoconnect.sh
100100

101+
e2e-clone-cache:
102+
name: E2E (Clone cache)
103+
runs-on: ubuntu-latest
104+
timeout-minutes: 25
105+
steps:
106+
- uses: actions/checkout@v6
107+
- name: Install dependencies
108+
uses: ./.github/actions/setup
109+
- name: Docker info
110+
run: docker version && docker compose version
111+
- name: Clone cache reuse
112+
run: bash scripts/e2e/clone-cache.sh
113+
101114
e2e-login-context:
102115
name: E2E (Login context)
103116
runs-on: ubuntu-latest

‎packages/docker-git/tests/core/templates.test.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,11 @@ describe("planFiles", () => {
7171
"GIT_CREDENTIAL_HELPER_PATH=\"/usr/local/bin/docker-git-credential-helper\""
7272
)
7373
expect(entrypointSpec.contents).toContain("token=\"$GITHUB_TOKEN\"")
74+
expect(entrypointSpec.contents).toContain("CACHE_ROOT=\"/home/dev/.docker-git/.cache/git-mirrors\"")
75+
expect(entrypointSpec.contents).toContain("CLONE_CACHE_ARGS=\"--reference-if-able '$CACHE_REPO_DIR' --dissociate\"")
76+
expect(entrypointSpec.contents).toContain("[clone-cache] using mirror: $CACHE_REPO_DIR")
77+
expect(entrypointSpec.contents).toContain("git clone --progress $CLONE_CACHE_ARGS")
78+
expect(entrypointSpec.contents).toContain("[clone-cache] mirror created: $CACHE_REPO_DIR")
7479
}
7580
}))
7681

‎packages/lib/src/core/templates-entrypoint/tasks.ts‎

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,42 @@ else
4646
AUTH_REPO_URL="$REPO_URL"
4747
if [[ -n "$GIT_AUTH_TOKEN" && "$REPO_URL" == https://* ]]; then
4848
AUTH_REPO_URL="$(printf "%s" "$REPO_URL" | sed "s#^https://#https://\${GIT_AUTH_USER}:\${GIT_AUTH_TOKEN}@#")"
49+
fi
50+
51+
CLONE_CACHE_ARGS=""
52+
CACHE_REPO_DIR=""
53+
CACHE_ROOT="/home/${config.sshUser}/.docker-git/.cache/git-mirrors"
54+
if command -v sha256sum >/dev/null 2>&1; then
55+
REPO_CACHE_KEY="$(printf "%s" "$REPO_URL" | sha256sum | awk '{print $1}')"
56+
elif command -v shasum >/dev/null 2>&1; then
57+
REPO_CACHE_KEY="$(printf "%s" "$REPO_URL" | shasum -a 256 | awk '{print $1}')"
58+
else
59+
REPO_CACHE_KEY="$(printf "%s" "$REPO_URL" | tr '/:@' '_' | tr -cd '[:alnum:]_.-')"
60+
fi
61+
62+
if [[ -n "$REPO_CACHE_KEY" ]]; then
63+
CACHE_REPO_DIR="$CACHE_ROOT/$REPO_CACHE_KEY.git"
64+
mkdir -p "$CACHE_ROOT"
65+
chown 1000:1000 "$CACHE_ROOT" || true
66+
if [[ -d "$CACHE_REPO_DIR" ]]; then
67+
if su - ${config.sshUser} -c "git --git-dir '$CACHE_REPO_DIR' rev-parse --is-bare-repository >/dev/null 2>&1"; then
68+
if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git --git-dir '$CACHE_REPO_DIR' fetch --progress --prune '$REPO_URL' '+refs/*:refs/*'"; then
69+
echo "[clone-cache] mirror refresh failed for $REPO_URL"
70+
fi
71+
CLONE_CACHE_ARGS="--reference-if-able '$CACHE_REPO_DIR' --dissociate"
72+
echo "[clone-cache] using mirror: $CACHE_REPO_DIR"
73+
else
74+
echo "[clone-cache] invalid mirror removed: $CACHE_REPO_DIR"
75+
rm -rf "$CACHE_REPO_DIR"
76+
fi
77+
fi
4978
fi`
5079

5180
const renderCloneBodyRef = (config: TemplateConfig): string =>
5281
` if [[ -n "$REPO_REF" ]]; then
5382
if [[ "$REPO_REF" == refs/pull/* ]]; then
5483
REF_BRANCH="pr-$(printf "%s" "$REPO_REF" | tr '/:' '--')"
55-
if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress '$AUTH_REPO_URL' '$TARGET_DIR'"; then
84+
if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress $CLONE_CACHE_ARGS '$AUTH_REPO_URL' '$TARGET_DIR'"; then
5685
echo "[clone] git clone failed for $REPO_URL"
5786
CLONE_OK=0
5887
else
@@ -62,12 +91,12 @@ const renderCloneBodyRef = (config: TemplateConfig): string =>
6291
fi
6392
fi
6493
else
65-
if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress --branch '$REPO_REF' '$AUTH_REPO_URL' '$TARGET_DIR'"; then
94+
if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress $CLONE_CACHE_ARGS --branch '$REPO_REF' '$AUTH_REPO_URL' '$TARGET_DIR'"; then
6695
DEFAULT_REF="$(git ls-remote --symref "$AUTH_REPO_URL" HEAD 2>/dev/null | awk '/^ref:/ {print $2}' | head -n 1 || true)"
6796
DEFAULT_BRANCH="$(printf "%s" "$DEFAULT_REF" | sed 's#^refs/heads/##')"
6897
if [[ -n "$DEFAULT_BRANCH" ]]; then
6998
echo "[clone] branch '$REPO_REF' missing; retrying with '$DEFAULT_BRANCH'"
70-
if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress --branch '$DEFAULT_BRANCH' '$AUTH_REPO_URL' '$TARGET_DIR'"; then
99+
if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress $CLONE_CACHE_ARGS --branch '$DEFAULT_BRANCH' '$AUTH_REPO_URL' '$TARGET_DIR'"; then
71100
echo "[clone] git clone failed for $REPO_URL"
72101
CLONE_OK=0
73102
elif [[ "$REPO_REF" == issue-* ]]; then
@@ -83,12 +112,27 @@ const renderCloneBodyRef = (config: TemplateConfig): string =>
83112
fi
84113
fi
85114
else
86-
if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress '$AUTH_REPO_URL' '$TARGET_DIR'"; then
115+
if ! su - ${config.sshUser} -c "GIT_TERMINAL_PROMPT=0 git clone --progress $CLONE_CACHE_ARGS '$AUTH_REPO_URL' '$TARGET_DIR'"; then
87116
echo "[clone] git clone failed for $REPO_URL"
88117
CLONE_OK=0
89118
fi
90119
fi`
91120

121+
const renderCloneCacheFinalize = (config: TemplateConfig): string =>
122+
`if [[ "$CLONE_OK" -eq 1 && -d "$TARGET_DIR/.git" && -n "$CACHE_REPO_DIR" && ! -d "$CACHE_REPO_DIR" ]]; then
123+
CACHE_TMP_DIR="$CACHE_REPO_DIR.tmp-$$"
124+
if su - ${config.sshUser} -c "rm -rf '$CACHE_TMP_DIR' && GIT_TERMINAL_PROMPT=0 git clone --mirror --progress '$TARGET_DIR/.git' '$CACHE_TMP_DIR'"; then
125+
if mv "$CACHE_TMP_DIR" "$CACHE_REPO_DIR" 2>/dev/null; then
126+
echo "[clone-cache] mirror created: $CACHE_REPO_DIR"
127+
else
128+
rm -rf "$CACHE_TMP_DIR"
129+
fi
130+
else
131+
echo "[clone-cache] mirror bootstrap failed for $REPO_URL"
132+
rm -rf "$CACHE_TMP_DIR"
133+
fi
134+
fi`
135+
92136
const renderIssueWorkspaceAgentsResolve = (): string =>
93137
`ISSUE_ID="$(printf "%s" "$REPO_REF" | sed -E 's#^issue-##')"
94138
ISSUE_URL=""
@@ -180,6 +224,8 @@ const renderCloneBody = (config: TemplateConfig): string =>
180224
"",
181225
renderCloneRemotes(config),
182226
"",
227+
renderCloneCacheFinalize(config),
228+
"",
183229
renderIssueWorkspaceAgents()
184230
].join("\n")
185231

‎packages/lib/src/usecases/docker-git-config-search.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ type DockerGitConfigSearchState = {
1010

1111
const isDockerGitConfig = (entry: string): boolean => entry.endsWith("docker-git.json")
1212

13-
const shouldSkipDir = (entry: string): boolean => entry === ".git" || entry === ".orch"
13+
const shouldSkipDir = (entry: string): boolean => entry === ".git" || entry === ".orch" || entry === ".docker-git"
1414

1515
const processDockerGitEntry = (
1616
fs: FileSystem.FileSystem,

‎packages/lib/src/usecases/state-repo/gitignore.ts‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,18 @@ const volatileCodexIgnorePatterns: ReadonlyArray<string> = [
1717
"**/.orch/auth/codex/models_cache.json"
1818
]
1919

20+
const repositoryCacheIgnorePatterns: ReadonlyArray<string> = [
21+
".cache/git-mirrors/"
22+
]
23+
2024
const defaultStateGitignore = [
2125
stateGitignoreMarker,
2226
"# NOTE: this repo intentionally tracks EVERYTHING under the state dir, including .orch/env and .orch/auth.",
2327
"# Keep the remote private; treat it as sensitive infrastructure state.",
2428
"",
29+
"# Shared git mirrors cache (do not commit)",
30+
...repositoryCacheIgnorePatterns,
31+
"",
2532
"# Volatile Codex artifacts (do not commit)",
2633
...volatileCodexIgnorePatterns,
2734
""
@@ -32,6 +39,34 @@ const normalizeGitignoreText = (text: string): string =>
3239
.replaceAll("\r\n", "\n")
3340
.trim()
3441

42+
type MissingManagedPatterns = {
43+
readonly repositoryCache: ReadonlyArray<string>
44+
readonly volatileCodex: ReadonlyArray<string>
45+
}
46+
47+
const collectMissingManagedPatterns = (prevLines: ReadonlySet<string>): MissingManagedPatterns => ({
48+
repositoryCache: repositoryCacheIgnorePatterns.filter((p) => !prevLines.has(p)),
49+
volatileCodex: volatileCodexIgnorePatterns.filter((p) => !prevLines.has(p))
50+
})
51+
52+
const hasMissingManagedPatterns = (missing: MissingManagedPatterns): boolean =>
53+
missing.repositoryCache.length > 0 || missing.volatileCodex.length > 0
54+
55+
const appendManagedBlocks = (
56+
prev: string,
57+
missing: MissingManagedPatterns
58+
): string => {
59+
const blocks = [
60+
missing.repositoryCache.length > 0
61+
? `# Shared git mirrors cache (do not commit)\n${missing.repositoryCache.join("\n")}`
62+
: "",
63+
missing.volatileCodex.length > 0
64+
? `# Volatile Codex artifacts (do not commit)\n${missing.volatileCodex.join("\n")}`
65+
: ""
66+
].filter((block) => block.length > 0)
67+
return `${[prev.trimEnd(), ...blocks].join("\n\n")}\n`
68+
}
69+
3570
export const ensureStateGitignore = (
3671
fs: FileSystem.FileSystem,
3772
path: Path.Path,
@@ -65,11 +100,10 @@ export const ensureStateGitignore = (
65100
return
66101
}
67102

68-
// Ensure volatile Codex artifacts are ignored; append if missing.
69-
const missingVolatile = volatileCodexIgnorePatterns.filter((p) => !prevLines.has(p))
70-
if (missingVolatile.length === 0) {
103+
// Ensure managed ignore patterns exist; append any missing entries.
104+
const missing = collectMissingManagedPatterns(prevLines)
105+
if (!hasMissingManagedPatterns(missing)) {
71106
return
72107
}
73-
const next = `${prev.trimEnd()}\n\n# Volatile Codex artifacts (do not commit)\n${missingVolatile.join("\n")}\n`
74-
yield* _(fs.writeFileString(gitignorePath, next))
108+
yield* _(fs.writeFileString(gitignorePath, appendManagedBlocks(prev, missing)))
75109
})

‎packages/lib/tests/usecases/docker-git-config-search.test.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import * as FileSystem from "@effect/platform/FileSystem"
2+
import * as Path from "@effect/platform/Path"
3+
import { NodeContext } from "@effect/platform-node"
4+
import { describe, expect, it } from "@effect/vitest"
5+
import { Effect } from "effect"
6+
7+
import { findDockerGitConfigPaths } from "../../src/usecases/docker-git-config-search.js"
8+
9+
const withTempDir = <A, E, R>(
10+
use: (tempDir: string) => Effect.Effect<A, E, R>
11+
): Effect.Effect<A, E, R | FileSystem.FileSystem> =>
12+
Effect.scoped(
13+
Effect.gen(function*(_) {
14+
const fs = yield* _(FileSystem.FileSystem)
15+
const tempDir = yield* _(
16+
fs.makeTempDirectoryScoped({
17+
prefix: "docker-git-config-search-"
18+
})
19+
)
20+
return yield* _(use(tempDir))
21+
})
22+
)
23+
24+
const writeFileWithParents = (
25+
fs: FileSystem.FileSystem,
26+
path: Path.Path,
27+
filePath: string
28+
) =>
29+
Effect.gen(function*(_) {
30+
const parent = path.dirname(filePath)
31+
yield* _(fs.makeDirectory(parent, { recursive: true }))
32+
yield* _(fs.writeFileString(filePath, "{}\n"))
33+
})
34+
35+
describe("findDockerGitConfigPaths", () => {
36+
it.effect("skips metadata and shared docker-git cache directories", () =>
37+
withTempDir((root) =>
38+
Effect.gen(function*(_) {
39+
const fs = yield* _(FileSystem.FileSystem)
40+
const path = yield* _(Path.Path)
41+
const includedMain = path.join(root, "org/repo-a/docker-git.json")
42+
const includedNested = path.join(root, "org/repo-b/nested/docker-git.json")
43+
const ignoredGit = path.join(root, "org/repo-a/.git/docker-git.json")
44+
const ignoredOrch = path.join(root, "org/repo-a/.orch/docker-git.json")
45+
const ignoredDockerGit = path.join(root, ".docker-git/.cache/git-mirrors/docker-git.json")
46+
47+
yield* _(writeFileWithParents(fs, path, includedMain))
48+
yield* _(writeFileWithParents(fs, path, includedNested))
49+
yield* _(writeFileWithParents(fs, path, ignoredGit))
50+
yield* _(writeFileWithParents(fs, path, ignoredOrch))
51+
yield* _(writeFileWithParents(fs, path, ignoredDockerGit))
52+
53+
const found = yield* _(findDockerGitConfigPaths(fs, path, root))
54+
expect([...found].sort()).toEqual([includedMain, includedNested].sort())
55+
})
56+
).pipe(Effect.provide(NodeContext.layer)))
57+
})

‎packages/lib/tests/usecases/state-repo-gitignore.test.ts‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import * as FileSystem from "@effect/platform/FileSystem"
2+
import * as Path from "@effect/platform/Path"
3+
import { NodeContext } from "@effect/platform-node"
4+
import { describe, expect, it } from "@effect/vitest"
5+
import { Effect } from "effect"
6+
7+
import { ensureStateGitignore } from "../../src/usecases/state-repo/gitignore.js"
8+
9+
const withTempDir = <A, E, R>(
10+
use: (tempDir: string) => Effect.Effect<A, E, R>
11+
): Effect.Effect<A, E, R | FileSystem.FileSystem> =>
12+
Effect.scoped(
13+
Effect.gen(function*(_) {
14+
const fs = yield* _(FileSystem.FileSystem)
15+
const tempDir = yield* _(
16+
fs.makeTempDirectoryScoped({
17+
prefix: "docker-git-state-gitignore-"
18+
})
19+
)
20+
return yield* _(use(tempDir))
21+
})
22+
)
23+
24+
describe("ensureStateGitignore", () => {
25+
it.effect("creates managed .gitignore with repository mirror cache ignored", () =>
26+
withTempDir((root) =>
27+
Effect.gen(function*(_) {
28+
const fs = yield* _(FileSystem.FileSystem)
29+
const path = yield* _(Path.Path)
30+
31+
yield* _(ensureStateGitignore(fs, path, root))
32+
33+
const gitignore = yield* _(fs.readFileString(path.join(root, ".gitignore")))
34+
expect(gitignore).toContain("# docker-git state repository")
35+
expect(gitignore).toContain(".cache/git-mirrors/")
36+
expect(gitignore).toContain("**/.orch/auth/codex/models_cache.json")
37+
})
38+
).pipe(Effect.provide(NodeContext.layer)))
39+
40+
it.effect("appends missing cache ignore pattern for managed files", () =>
41+
withTempDir((root) =>
42+
Effect.gen(function*(_) {
43+
const fs = yield* _(FileSystem.FileSystem)
44+
const path = yield* _(Path.Path)
45+
const gitignorePath = path.join(root, ".gitignore")
46+
const existing = [
47+
"# docker-git state repository",
48+
"# NOTE: this repo intentionally tracks EVERYTHING under the state dir, including .orch/env and .orch/auth.",
49+
"# Keep the remote private; treat it as sensitive infrastructure state.",
50+
"",
51+
"custom-ignore/",
52+
""
53+
].join("\n")
54+
55+
yield* _(fs.writeFileString(gitignorePath, existing))
56+
yield* _(ensureStateGitignore(fs, path, root))
57+
58+
const gitignore = yield* _(fs.readFileString(gitignorePath))
59+
expect(gitignore).toContain("custom-ignore/")
60+
expect(gitignore).toContain("# Shared git mirrors cache (do not commit)")
61+
expect(gitignore).toContain(".cache/git-mirrors/")
62+
})
63+
).pipe(Effect.provide(NodeContext.layer)))
64+
})

0 commit comments

Comments
 (0)