Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions packages/core/src/remote/runners.ts
Original file line number Diff line number Diff line change
Expand Up @@ -528,10 +528,10 @@ async function connected(id: string, state: LinkState) {
const r = row(id);
if (!r) return;
if (toPrimary(r)) {
await deviceHooks?.deviceConnected(id).catch((err) => log.warn("could not catch up with the primary", err instanceof Error ? err.message : err));
await deviceHooks?.deviceConnected(id).catch((err) => logFailure("could not catch up with the primary", err));
return;
}
if (r.kind === "device") await deviceHooks?.deviceConnected(id).catch((err) => log.warn(`could not take the changes of ${r.name}`, err instanceof Error ? err.message : err));
if (r.kind === "device") await deviceHooks?.deviceConnected(id).catch((err) => logFailure(`could not take the changes of ${r.name}`, err));
if (dropped()) return;
await releaseChats(id);
const catchUpNow = async () => {
Expand All @@ -545,7 +545,7 @@ async function connected(id: string, state: LinkState) {
try {
await syncRunner(id);
} catch (err) {
if (!dropped()) log.warn(`could not copy the setup to runner ${id}`, err instanceof Error ? err.message : err);
if (!dropped()) logFailure(`could not copy the setup to runner ${id}`, err);
}
};
// A computer that just joined the group knows this setup's agents only after the copy: its chats come after it.
Expand Down Expand Up @@ -696,7 +696,7 @@ function scheduleSync(force: boolean) {
emit(id);
continue;
}
void syncRunner(id, { force: forced }).catch((err) => log.warn(`could not copy the setup to runner ${id}`, err instanceof Error ? err.message : err));
void syncRunner(id, { force: forced }).catch((err) => logFailure(`could not copy the setup to runner ${id}`, err));
}
}, SYNC_DEBOUNCE_MS);
syncTimer.unref?.();
Expand Down Expand Up @@ -732,6 +732,19 @@ function message(err: unknown): string {
return err instanceof Error ? err.message.replace(/^Error:\s*/, "") : String(err);
}

/** The other computer went away or stopped answering: the next connect tries again on its own. */
export function linkGone(err: unknown): boolean {
if (!(err instanceof Error)) return false;
if (err instanceof HttpError && (err.code === "runner_offline" || err.code === "runner_timeout")) return true;
return linkGone(err.cause);
}

function logFailure(text: string, err: unknown): void {
const detail = err instanceof Error ? err.message : err;
if (linkGone(err)) log.info(text, detail);
else log.warn(text, detail);
}

/**
* Bring a runner up to date: the setup when it differs (or `force`), then — for a chat that starts — the agent's memory
* and the browser sessions of the profile it uses. Returns what was done; a failed setup copy throws.
Expand Down Expand Up @@ -771,7 +784,7 @@ export function syncRunner(id: string, opts: { agentId?: string; profileId?: str
}
} catch (err) {
update("runners", id, { sync_error: `Couldn't copy your setup: ${message(err)}`, updated_at: now() });
throw new HttpError(502, `Couldn't copy your setup to ${r.name}: ${message(err)}`, "runner_sync_failed");
throw Object.assign(new HttpError(502, `Couldn't copy your setup to ${r.name}: ${message(err)}`, "runner_sync_failed"), { cause: err });
} finally {
syncing.delete(id);
emit(id);
Expand All @@ -783,16 +796,17 @@ export function syncRunner(id: string, opts: { agentId?: string; profileId?: str
memory = await exchangeMemory(id, l, r.name, opts.agentId);
} catch (err) {
warnings.push(`Couldn't merge the agent's memory with ${r.name}: ${message(err)}`);
logFailure(warnings.at(-1)!, err);
}
}
if (opts.profileId && r.sync_browser === 1) {
try {
cookies = await copyCookies(r, l, opts.profileId);
} catch (err) {
warnings.push(`Couldn't copy the browser sessions to ${r.name}: ${message(err)}`);
logFailure(warnings.at(-1)!, err);
}
}
for (const w of warnings) log.warn(w);
return { config, memory, cookies, warnings };
});
}
Expand Down
2 changes: 1 addition & 1 deletion packages/core/src/services/workspaceSources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ let offlineRetryMs = [3_000, 8_000];
export function __setOfflineRetryForTests(ms: number[]) {
offlineRetryMs = ms;
}
const UNRESOLVED_HOST = /could not resolve host|name or service not known|nodename nor servname/i;
const UNRESOLVED_HOST = /could not resolve host|name or service not known|nodename nor servname|resolving timed out/i;
/** How long a run waits for a first clone; it goes on without it after that (the clone continues). */
const RUN_CLONE_WAIT_MS = 90_000;
const SAFE_SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/;
Expand Down
16 changes: 14 additions & 2 deletions packages/core/test/remote-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@ import { getAccessToken } from "../src/server/auth";
import { resetSettingsCache, updateSettings } from "../src/services/settings";
import { createCredential } from "../src/vault/credentials";
import * as vault from "../src/vault/vault";
import { connectRunner, getRunner, link, listRunners, pairWithCode, removeRunner, startAutofix, startRunners, stopRunners } from "../src/remote/runners";
import { connectRunner, getRunner, link, linkGone, listRunners, pairWithCode, removeRunner, startAutofix, startRunners, stopRunners } from "../src/remote/runners";
import { Database } from "bun:sqlite";
import { callTool, listToolsFor } from "../src/mcp/tools";
import { getAgent } from "../src/agents/service";
import { transcriptPath } from "../src/services/conversations";
import { childEnv, sleep } from "../src/util";
import { HttpError, childEnv, sleep } from "../src/util";
import { __setClaudeBinaryForTests, cancelRun, listActiveRuns, waitForRun } from "../src/runner/runner";

const INDEX = join(import.meta.dir, "..", "src", "index.ts");
Expand Down Expand Up @@ -418,3 +418,15 @@ describe("a runner, end to end", () => {
runnerDb.close();
}, 30_000);
});

describe("link failures", () => {
test("a runner that went away or stalled is no problem for the log, other failures are", () => {
expect(linkGone(new HttpError(409, "Mac is offline", "runner_offline"))).toBe(true);
expect(linkGone(new HttpError(504, "Mac didn't answer in time.", "runner_timeout"))).toBe(true);
const wrapped = Object.assign(new HttpError(502, "Couldn't copy your setup", "runner_sync_failed"), { cause: new HttpError(504, "", "runner_timeout") });
expect(linkGone(wrapped)).toBe(true);
expect(linkGone(new HttpError(502, "Couldn't copy your setup", "runner_sync_failed"))).toBe(false);
expect(linkGone(new Error("disk full"))).toBe(false);
expect(linkGone("offline")).toBe(false);
});
});
3 changes: 3 additions & 0 deletions packages/core/test/workspace-sources.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,9 @@ describe("workspace repositories", () => {
"sign in to git",
);
expect(gitFailure({ stderr: "", timedOut: true }, "git@github.com:a/b.git", null)).toBe("Timed out talking to github.com.");
expect(
gitFailure({ stderr: "fatal: unable to access 'https://github.com/a/b.git/': Resolving timed out after 987676 milliseconds", timedOut: false }, "https://github.com/a/b.git", null),
).toBe("Can't reach github.com — check the URL and your internet connection.");
});
});

Expand Down
Loading