diff --git a/.claude/rules/sim-sandbox.md b/.claude/rules/sim-sandbox.md index e696f04f843..ff41c6cec2f 100644 --- a/.claude/rules/sim-sandbox.md +++ b/.claude/rules/sim-sandbox.md @@ -2,6 +2,8 @@ description: Isolated-vm sandbox worker security policy. Hard rules for anything that lives in the worker child process that runs user code. paths: - "apps/sim/lib/execution/isolated-vm-worker.cjs" + - "apps/sim/lib/execution/isolated-vm-host.cjs" + - "apps/sim/lib/execution/isolated-vm-remote.ts" - "apps/sim/lib/execution/isolated-vm.ts" - "apps/sim/lib/execution/sandbox/**" - "apps/sim/sandboxes/**" @@ -75,7 +77,31 @@ payload or `ivm.Reference` wrapper in the worker: - [ ] Does the worker read a new env var? Add it to the `buildWorkerEnv()` allowlist in `isolated-vm.ts` **and** to the allowlist regression test in `isolated-vm.test.ts` — the worker does not inherit the app environment, - so an un-allowlisted var is simply absent in the child. + so an un-allowlisted var is simply absent in the child. If the app owns + its value, put it in `appWorkerEnv()` and add it with its accepted shape + to `CLIENT_WORKER_ENV` in `isolated-vm-host.cjs`, so a remote worker + gets the same value. +- [ ] Does the host send the worker a new message type? Add it to + `ALLOWED_INBOUND_TYPES` in `isolated-vm-host.cjs`, or remote workers + never receive it. + +## Where the worker runs + +The pool in `isolated-vm.ts` opens each worker either as a local child process +or, when `IVM_REMOTE_WORKER_URL` is set (with `IVM_REMOTE_WORKER_TOKEN`; a URL +without a valid token fails executions rather than falling back) and the +`isolated-vm-local-workers` kill switch is off, as a WebSocket connection to +`isolated-vm-host.cjs`, which runs the +same worker in a dedicated service that holds no platform credentials. Every +rule in this file applies in both modes. + +- `fetch()` and brokers always run in the app. The host only relays the + worker's messages, so it needs no network access of its own and never gets + a broker implementation. +- A remote target never falls back to a local child process. An unreachable + host fails executions with a system error; keep it that way. +- Changes to the worker message protocol must stay compatible with the + previous release: the host and its clients roll out independently. ## What the worker *may* hold diff --git a/.cursor/rules/sim-sandbox.mdc b/.cursor/rules/sim-sandbox.mdc index 351c4e74292..f9ce1e078b6 100644 --- a/.cursor/rules/sim-sandbox.mdc +++ b/.cursor/rules/sim-sandbox.mdc @@ -1,6 +1,6 @@ --- description: "Isolated-vm sandbox worker security policy. Hard rules for anything that lives in the worker child process that runs user code." -globs: ["apps/sim/lib/execution/isolated-vm-worker.cjs","apps/sim/lib/execution/isolated-vm.ts","apps/sim/lib/execution/sandbox/**","apps/sim/sandboxes/**"] +globs: ["apps/sim/lib/execution/isolated-vm-worker.cjs","apps/sim/lib/execution/isolated-vm-host.cjs","apps/sim/lib/execution/isolated-vm-remote.ts","apps/sim/lib/execution/isolated-vm.ts","apps/sim/lib/execution/sandbox/**","apps/sim/sandboxes/**"] --- @@ -73,7 +73,31 @@ payload or `ivm.Reference` wrapper in the worker: - [ ] Does the worker read a new env var? Add it to the `buildWorkerEnv()` allowlist in `isolated-vm.ts` **and** to the allowlist regression test in `isolated-vm.test.ts` — the worker does not inherit the app environment, - so an un-allowlisted var is simply absent in the child. + so an un-allowlisted var is simply absent in the child. If the app owns + its value, put it in `appWorkerEnv()` and add it with its accepted shape + to `CLIENT_WORKER_ENV` in `isolated-vm-host.cjs`, so a remote worker + gets the same value. +- [ ] Does the host send the worker a new message type? Add it to + `ALLOWED_INBOUND_TYPES` in `isolated-vm-host.cjs`, or remote workers + never receive it. + +## Where the worker runs + +The pool in `isolated-vm.ts` opens each worker either as a local child process +or, when `IVM_REMOTE_WORKER_URL` is set (with `IVM_REMOTE_WORKER_TOKEN`; a URL +without a valid token fails executions rather than falling back) and the +`isolated-vm-local-workers` kill switch is off, as a WebSocket connection to +`isolated-vm-host.cjs`, which runs the +same worker in a dedicated service that holds no platform credentials. Every +rule in this file applies in both modes. + +- `fetch()` and brokers always run in the app. The host only relays the + worker's messages, so it needs no network access of its own and never gets + a broker implementation. +- A remote target never falls back to a local child process. An unreachable + host fails executions with a system error; keep it that way. +- Changes to the worker message protocol must stay compatible with the + previous release: the host and its clients roll out independently. ## What the worker *may* hold diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index bb19cfda9bd..8ae0c89dae4 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -35,7 +35,7 @@ runs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.4.2 + bun-version: 1.4.4 - name: Setup Node if: inputs.node-version != '' diff --git a/.github/scripts/wait-for-ecs-cutover.sh b/.github/scripts/wait-for-ecs-cutover.sh index 4e9b10f5d66..8d3ed477370 100755 --- a/.github/scripts/wait-for-ecs-cutover.sh +++ b/.github/scripts/wait-for-ecs-cutover.sh @@ -92,12 +92,15 @@ while [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = 'None' ]; do DEPLOYMENT_ID=$(printf '%s\n' "$deploy_actions" | DEPLOY_STATE="$deploy_state" EXECUTION_ID="$EXECUTION_ID" python3 -c ' import json, os, re, sys state = json.loads(os.environ["DEPLOY_STATE"]) -actions = json.load(sys.stdin) +# The Deploy stage can also run sidecar actions (e.g. a CodeBuild image deploy); only the +# CodeDeploy blue/green action carries the ECS cutover. +actions = [a for a in json.load(sys.stdin) + if a.get("input", {}).get("actionTypeId", {}).get("provider") == "CodeDeployToECS"] if not state or state.get("latestExecution", {}).get("pipelineExecutionId") != os.environ["EXECUTION_ID"] or not actions: print("") sys.exit(0) if len({a["actionName"] for a in actions}) != 1: - raise SystemExit("ERROR: expected one Deploy action in the app pipeline") + raise SystemExit("ERROR: expected one CodeDeploy ECS action in the app pipeline Deploy stage") latest = max(actions, key=lambda a: a["startTime"]) matches = [a["latestExecution"] for a in state.get("actionStates", []) if a["actionName"] == latest["actionName"] diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ef97d255b12..7abb0de6951 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -45,7 +45,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.4.2 + bun-version: 1.4.4 - name: Install only the pinned task planner continue-on-error: true run: bun install --global "turbo@$(jq -r .devDependencies.turbo package.json)" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62879db98f2..18ccd11c62f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -569,6 +569,9 @@ jobs: needs.promote.result == 'success' && needs.trigger-upload.result == 'success' && needs.promote.outputs.promoted == 'true' + outputs: + # The new app serves traffic; indexnow keys off this, not the later Trigger.dev promotion. + cutover: ${{ steps.cutover.outcome }} # Mostly waiting on the ECS cutover after a dependency install: the smallest runner is enough. runs-on: *runner-2vcpu # Leave setup/promotion headroom above the 70-minute cutover poll. @@ -596,6 +599,7 @@ jobs: # An unchanged tag may belong to a failed or still-running earlier deploy. # Verify its latest cutover rather than treating tag equality as success. - name: Wait for ECS traffic cutover + id: cutover env: OVERALL_TIMEOUT: 4200 APP_IMAGE_CHANGED: ${{ needs.promote.outputs.app_image_changed }} @@ -636,6 +640,59 @@ jobs: fi bunx trigger.dev@4.5.16 promote "$VERSION" "${TARGET_ARGS[@]}" + # Production only, once the new app serves traffic: submit the library and comparison URLs this + # push changed to IndexNow. Best effort: nothing depends on it, and a failure never fails the run. + indexnow: + name: indexnow + needs: [promote, trigger-promote] + if: >- + !cancelled() && + github.event_name == 'push' && + github.ref == 'refs/heads/main' && + needs.promote.outputs.promoted == 'true' && + needs.trigger-promote.outputs.cutover == 'success' + runs-on: *runner-2vcpu + timeout-minutes: 5 + continue-on-error: true + permissions: + contents: read + actions: read + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.4 + + # Diff from the head of the last earlier run whose submission succeeded, not the previous + # push: a push that never deployed, or whose submission failed, would otherwise drop its + # changed URLs for good. Falls back to the previous push when no such run is in view. The + # diff needs only the two trees, so that one commit is fetched alone; an all-zero `before` + # (the branch's first push) has no base, which the script skips. + - name: Submit changed URLs to IndexNow + env: + GH_TOKEN: ${{ github.token }} + PUSH_BEFORE: ${{ github.event.before }} + run: | + BASE_SHA="" + while read -r run_id head_sha; do + conclusion="$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?per_page=100" \ + --jq '.jobs[] | select(.name == "indexnow") | .conclusion')" + if [[ "$conclusion" == "success" ]]; then + BASE_SHA="$head_sha" + break + fi + done < <(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/ci.yml/runs?branch=main&event=push&per_page=30" \ + --jq ".workflow_runs[] | select(.id < $GITHUB_RUN_ID) | \"\\(.id) \\(.head_sha)\"") + BASE_SHA="${BASE_SHA:-$PUSH_BEFORE}" + echo "Submitting URLs changed in $BASE_SHA..$GITHUB_SHA" + if [[ ! "$BASE_SHA" =~ ^0+$ ]]; then + git fetch --no-tags --depth=1 origin "$BASE_SHA" + fi + bun scripts/ping-indexnow.ts "$BASE_SHA" "$GITHUB_SHA" + # Build ARM64 images for GHCR (main branch only, runs in parallel with # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64 # are applied by ghcr after the gate, so a failing run diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml index a91e7010363..ab14614fc5b 100644 --- a/.github/workflows/desktop-e2e.yml +++ b/.github/workflows/desktop-e2e.yml @@ -79,7 +79,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.4.2 + bun-version: 1.4.4 # Electron, electron-builder and Playwright download their binaries on every cold install. All # three caches are keyed by version inside, so an older entry restored by prefix is harmless. @@ -151,7 +151,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.4.2 + bun-version: 1.4.4 # Electron, electron-builder and Playwright download their binaries on every cold install. All # three caches are keyed by version inside, so an older entry restored by prefix is harmless. @@ -213,7 +213,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.4.2 + bun-version: 1.4.4 - name: Setup Node uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6 diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 225cb703074..f8d7da153c0 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -143,7 +143,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.4.2 + bun-version: 1.4.4 - name: Setup Node uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6 diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 15823a571bc..7bfddf84b66 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -62,7 +62,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.4.2 + bun-version: 1.4.4 # Docker Compose and Kubernetes must run the same background jobs on the # same schedules; this fails the build if the two drift apart. The script @@ -286,7 +286,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.4.2 + bun-version: 1.4.4 # Derives appVersion rather than checking it, so the published chart cannot # be pinned to an older Sim than the release it ships with. The committed @@ -481,7 +481,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.4.2 + bun-version: 1.4.4 # Derives appVersion rather than checking it, so the published chart cannot # be pinned to an older Sim than the release it ships with. The committed diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 87b3699ca7a..303f47a78f4 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -9,7 +9,7 @@ "type": "module", "main": "dist/main.cjs", "engines": { - "bun": ">=1.4.2", + "bun": ">=1.4.4", "node": ">=20.0.0" }, "scripts": { diff --git a/apps/docs/COPY_AUDIT.md b/apps/docs/COPY_AUDIT.md index 34e2ad4049e..2691adeb91a 100644 --- a/apps/docs/COPY_AUDIT.md +++ b/apps/docs/COPY_AUDIT.md @@ -517,7 +517,7 @@ Every row below represents a fully reviewed inventory entry. “Rewritten” inc | [platform/self-hosting/redis.mdx](content/docs/platform/self-hosting/redis.mdx) | rewritten | Separated managed-service guidance from Compose/Helm deployment tabs. Qualified managed service clustering compatibility and sizing/failover assumptions; updated Azure retirement advice. Scoped TLS server-name override to app IPv4 and recommended certificate-matching DNS for app/realtime. Kept fallback/persistence/secret/egress details; verification no longer prints credential-bearing URLs. Used shared authored tabs with padded mixed content, wrapped prose, visible horizontal scrolling, and mobile touch targets. | Verified improvement. Replaces universal managed-service/HA sizing assurances with measured estimates and compatible single-endpoint cluster-policy requirements, including Azure retirement guidance. Correctly scopes TLS server-name override to app IPv4 literals and recommends certificate-matching DNS for both app and realtime. Verification checks URL presence without disclosing credentials and separates a bundled redis-cli ping from testing external connectivity/cross-pod behavior. Preserved: All seven fallback matrix entries; pairing-only Redis requirement vs OAuth; cross-pod collaboration/live-event prerequisite; in-flight streaming/cancellation loss vs committed-data preservation remain. Both Redis/TLS URL forms, Compose external override, Helm auto-managed suppression, Secret envFrom precedence, nonpersistent 512 MiB bundle, external egress rule, private endpoint/auth/TLS recommendations, SNI example and cross-replica functional check remain. Managed Redis tab content is promoted to a normal section, preserving the three cloud choices. Renamed verifying heading has no current inbound repository reference. | | [platform/self-hosting/reference-architectures.mdx](content/docs/platform/self-hosting/reference-architectures.mdx) | rewritten | Put cloud provisioning and secrets before sizing; retained IaC, identity, network and placeholder-secret prerequisites. Qualified chart minimum vs supported cloud Kubernetes releases and selected StorageClass/provisioner requirements. Labelled legacy AKS ingress-nginx and Azure Cache recommendations without choosing unsupported replacements. Narrowed ALB certificate warning and allowed supported GKE certificate coexistence/migration. Updated chart example to reconciled published 1.11.8/v0.9.17 and explained appVersion vs explicit pins; added Related docs. Used shared authored tabs with padded mixed content, wrapped prose, visible horizontal scrolling, and mobile touch targets. | Verified improvement. Leads with the operator/chart provisioning boundary; removes unsupported Kubernetes support, EKS StorageClass, throughput, and blanket certificate claims. Updates Azure ingress and managed Redis recommendations and clarifies ALB certificate requirements and GKE mixed-certificate migration support. Corrects version pinning: a pinned chart selects its appVersion when image tags are empty, while explicit tags/digests override. Distinguishes published chart metadata from checkout metadata. Preserved: Complete evaluation/production component matrix, local durability, remote Function/Pi image requirements, default database volume, managed-datastore egress warning, per-cloud infrastructure/identity/timeouts, and optional GPU prerequisites remain. All three secret-delivery strategies, complete existing Secret keys and shadowing, ESO mapping failure, requests/limits, full Terraform helm_release example including six sensitive values, placeholder-secret warning, and onward install/security/verification links remain. | | [platform/self-hosting/sandboxes.mdx](content/docs/platform/self-hosting/sandboxes.mdx) | rewritten | Rewrote as execution decision, build/verify, deploy, UI readiness, promotion/rollback and separate-image procedure. Preserved E2B-first Daytona parity requirement and all immutable identifiers, scopes, commands and manifest variables. Added checkout/dependency prerequisites, quoted shell placeholders and removed duplicated Daytona build command. Kept 422/503 conditions, billing-disabled entitlement and browser/server distinction; retained shell/Function/document/Pi separation. Used shared authored tabs with padded mixed content, wrapped prose, visible horizontal scrolling, and mobile touch targets. | Verified improvement. Reorders build, verify, configure, enable controls, and promote/rollback so the accepted E2B manifest exists before Daytona consumes it. Quotes shell placeholders and declares checkout/dependency prerequisites, keeps explicit selected-provider verification, and separates public UI assertion from server readiness. Summarizes local/remote execution paths in a table and adds Pi image distinction without overstating local worker isolation. Preserved: Local JavaScript exceptions, selected sandbox 503 and file-path 422 distinctions, Python fail-closed behavior, entitlement/master-switch/browser flags and visible Settings behavior remain. Both full build commands, API-key scopes, immutable image formats, positive monotonic E2B generations, accepted E2B parity manifest capture and Daytona baseline verification, --generation/--base-template, and higher-generation rollback requirement remain. Public Chat run_function/run_code shell image versus workflow function_execute Function image and separate shell/Function/document/Pi contracts remain. Renamed heading anchors have no remaining inbound repository reference. | -| [platform/self-hosting/scaling.mdx](content/docs/platform/self-hosting/scaling.mdx) | rewritten | Put prerequisites and measurements before scaling settings; removed unsupported observed-memory guarantee. Corrected Node child workers and deployment-wide scheduled-run cap. Added main/subpool ceilings and actual 30-primary app budget; excluded HTTP-only cron pods from independent database counts. Qualified pooling compatibility and linked direct migration DSN; preserved replica overrides/lag and PDB/HPA behavior. Separated billing-disabled plan limits from fixed safety/admission controls and labelled sizing/topology illustrative. | Verified improvement. Puts Redis and shared-storage prerequisites before scale examples and replaces unverified workload benchmarks with measurements/defaults and clearly labelled estimates. Corrects connection budgeting to sum all active main/subpools rather than charging cron pods for direct database pools; preserves per-role read-replica limits and warns about session locks/direct migrator connections. Corrects scheduled concurrency to a shared database cap; qualifies billing-disabled plan limits versus fixed security controls and removes a false 100-user throughput guarantee. Preserved: All component scaling behavior, full replicas/resources and HPA examples, metrics-server requirement, omitted replicas under HPA, PDB/shared-HPA caveat, scale-down interruption warning, and realtime disable example remain. External database two-flag requirement and app.env.DATABASE_URL override caveat; all read-replica keys/fallback/lag; all sizing values; concurrency settings; limit opt-ins/timeouts; full topology/secrets/password/ESO/PDB example remain. | +| [platform/self-hosting/scaling.mdx](content/docs/platform/self-hosting/scaling.mdx) | rewritten | Put prerequisites and measurements before scaling settings; removed unsupported observed-memory guarantee. Corrected Node child workers and deployment-wide scheduled-run cap. Added main/subpool ceilings and actual 37-primary app budget; excluded HTTP-only cron pods from independent database counts. Qualified pooling compatibility and linked direct migration DSN; preserved replica overrides/lag and PDB/HPA behavior. Separated billing-disabled plan limits from fixed safety/admission controls and labelled sizing/topology illustrative. | Verified improvement. Puts Redis and shared-storage prerequisites before scale examples and replaces unverified workload benchmarks with measurements/defaults and clearly labelled estimates. Corrects connection budgeting to sum all active main/subpools rather than charging cron pods for direct database pools; preserves per-role read-replica limits and warns about session locks/direct migrator connections. Corrects scheduled concurrency to a shared database cap; qualifies billing-disabled plan limits versus fixed security controls and removes a false 100-user throughput guarantee. Preserved: All component scaling behavior, full replicas/resources and HPA examples, metrics-server requirement, omitted replicas under HPA, PDB/shared-HPA caveat, scale-down interruption warning, and realtime disable example remain. External database two-flag requirement and app.env.DATABASE_URL override caveat; all read-replica keys/fallback/lag; all sizing values; concurrency settings; limit opt-ins/timeouts; full topology/secrets/password/ESO/PDB example remain. | | [platform/self-hosting/security.mdx](content/docs/platform/self-hosting/security.mdx) | rewritten | Moved the pre-launch checklist, signup restrictions, and proxy trust before detailed hardening. Folded repeated FAQ answers into their owners, kept all referenced anchors, and preserved key-loss/rotation and Compose password recovery. Corrected API-key storage wording and local Node-worker boundary; removed unsupported provider isolation ranking/universal SSRF guarantee. Added both inbound and outbound app self-call rules plus stored PII Logs policy and verification; kept fail-closed behavior. Kept allowlist/provenance/redirect/metadata/response-cap exceptions and GPU restricted-profile exception; condensed secret modes and validation rules. | Verified improvement. Puts checklist/access/prerequisites first; integrates duplicate FAQ answers; preserves scoped request provenance while removing the false guarantee for arbitrary sandbox network traffic. Corrects local code worker isolation, stored API-key rotation effects, immutable provider readiness errors, restricted-profile GPU exception, and broad egress override semantics. Adds stored log-redaction policy activation, both directions of app self-call NetworkPolicy, and a concrete masking check; labels timing/memory as measured rather than guaranteed. Final recheck: states that the chart default ingress peer permits any source on listed ports when ingress is enabled, and distinguishes original-key restoration from unrecoverable key loss. Preserved: All five secrets, same-session secret requirement, independent encryption backup, all three delivery strategies and shadowing/mapping consequences, ingress/egress rules, metadata-layer distinctions, optional unisolated workloads, restricted-profile exceptions and writable-root choices remain. All six egress provenance classes, exact allowlist examples/validation/limits, loopback/port/HTTP/Cloud distinctions, legacy compatibility cases, redirect statuses/header/body behavior, 100 MiB response limit and unbounded SSE exception remain. Local-worker security warning, all broker ceilings, PII failure marker/internal URL, historical default Compose password/public-port warning, SQL password rotation and loopback/SSH example, and every checklist item remain. Renamed optional-Chat-policy heading has no inbound repository references. | | [platform/self-hosting/troubleshooting.mdx](content/docs/platform/self-hosting/troubleshooting.mdx) | rewritten | Reordered all existing symptoms by deployment, auth/network, models/files, scheduler, restore, and source-only Windows lifecycle. Corrected scheduler-vs-auth diagnostics, Redis TLS scope, host alias behavior, private-host allowlist example and proxy owner links. Kept every diagnostic command and embedding/model exception; preserved encryption recovery and TLS-verification warning. Qualified CA-chain guidance and preserved UNABLE_TO_GET_ISSUER_CERT; normalized headings and Bun invocation. | Verified improvement. Reorders every symptom into deployment, login, network, model/storage, scheduler, restore, and source-only recovery order; replaces assumptions with diagnostic interpretation. Corrects shipped Linux host alias, Redis IPv4/app-only SNI scope, stale CronJob diagnosis, TLS chain/bundle caveats and cloud MiB completion threshold. Avoids deleting browser state before collecting errors and replaces printenv Redis credential exposure with presence checks. Preserved: All 23 original symptom families remain, including doctor modes/flags/exit status, database/migrations/pgvector/Kubernetes readiness/image/PVC, realtime/proxy/buffering/disconnects, Redis, TLS trust, private destinations, Ollama/LM Studio auth/prefix/URL/env propagation, CORS, embeddings width/recreation, schedules/polls/Teams, restored credentials, and Windows WSL/LF. Full Compose/Helm command examples, exact settings/env/error/status names, PostgreSQL verification and migration restart, HTTP in-pod secret expansion, original encryption-key recovery, and help links remain. Linux host-IP fallback remains available in the custom-container LM Studio guidance; shipped alias works across hosts. Renamed symptom headings have no current inbound repository references. | | [platform/self-hosting/upgrades.mdx](content/docs/platform/self-hosting/upgrades.mdx) | rewritten | Moved the upgrade procedure and rollback before advanced migration mechanics. Updated reconciled published release/chart examples and documented pinned SIM_VERSION behavior before update. Corrected SQL retry budget to 20 minutes; retained 30-minute advisory acquisition, session checks, lock/statement timeouts and connection-error retries. Separated migration DSN delivery modes and kept pooled/direct connection hazards and init-container failure recovery. Preserved all backup/rehearsal/CLI install-kind/pinning/Ollama profile/restore data-loss steps; qualified application-only rollback and password-history inference. Used shared authored tabs with padded mixed content, wrapped prose, visible horizontal scrolling, and mobile touch targets. | Verified improvement. Improves rollback safety: additive migrations alone no longer prove older-app compatibility; puts write-loss and stopped-writer consequences before restore steps. Corrects image/chart pin coupling and actual published version, migration retry budget, pin updates before sim-setup update, shell placeholders, helm-diff prerequisite and persisted-database password diagnosis. Keeps operator procedure concise and moves advanced migration mechanics into a lookup section. Final recheck: restores the concise direct MIGRATION_DATABASE_URL prerequisite before upgrade commands, including Helm/Compose delivery constraints and #migration-environment link. Preserved: All three aligned images and independent cron pin, tag/digest examples, five-step release/backup/rehearsal/apply/verify procedure, snapshots/pg_dump database names, full Helm/Compose/source/Ollama upgrade modes/profile commands, managed-file protection and maintenance downtime remain. Both rollback modes and all writer stopping/restore/old release/secret/verification consequences, all five migration failure diagnoses and do-not-skip-journal warning remain. Complete forward-only SQL/data migration behavior, replica serialization, 30-minute/5-second acquisition, direct DSN/backend PID guard, Helm/Compose propagation, SQLSTATE 55P03/20-minute retries, once-only scripts and 10 transient startup retries remain. Renamed how-migrations-run heading has no current inbound references. | diff --git a/apps/docs/content/docs/platform/self-hosting/scaling.mdx b/apps/docs/content/docs/platform/self-hosting/scaling.mdx index 10c09fbdd12..f2dfa0c5d8c 100644 --- a/apps/docs/content/docs/platform/self-hosting/scaling.mdx +++ b/apps/docs/content/docs/platform/self-hosting/scaling.mdx @@ -88,8 +88,10 @@ Each process can open more than one database pool. Budget the sum of active pool | `exec` | 10 | 4 | Execution subpool profile; current `dbFor` client opens a primary pool | | `cleanup` | 5 | 2 | Cleanup subpool profile; current `dbFor` client opens a primary pool | | `search` | 5 | 0 | File-search subpool | +| `billing` | 4 | 0 | Usage-ledger and threshold-billing subpool | +| `auth` | 3 | 0 | Session and API-key lookup subpool | -App subpools are created lazily, so a web process using all three can open up to 30 primary connections in addition to its configured read-replica pool. Count the actual pools and URLs used by each process. CronJob pods make HTTP calls to the app; they do not open their own PostgreSQL pools. +App subpools are created lazily, so a web process using all five can open up to 37 primary connections in addition to its configured read-replica pool. Count the actual pools and URLs used by each process. CronJob pods make HTTP calls to the app; they do not open their own PostgreSQL pools. A pooler such as **PgBouncer** can separate app replica count from database connection count. Before using transaction pooling, verify the application's session-lock requirements and give the migrator a [direct, non-pooled connection](/platform/self-hosting/upgrades#migration-environment). Point the chart at the chosen endpoint with `externalDatabase.enabled: true`, `externalDatabase.host`, and `externalDatabase.port`, alongside `postgresql.enabled: false`. Both halves are required — with `postgresql.enabled: false` and `externalDatabase.enabled` still false the chart renders an empty `DATABASE_URL`. Setting `app.env.DATABASE_URL` does nothing — the chart computes `DATABASE_URL` itself and inlines it on the container, where it overrides anything from the Secret. Set the pooler's server-connection budget for your database and verify the resulting workload before increasing replicas. diff --git a/apps/docs/scripts/fetch-git-history.ts b/apps/docs/scripts/fetch-git-history.ts index dd87c8f8012..0eb33efa089 100644 --- a/apps/docs/scripts/fetch-git-history.ts +++ b/apps/docs/scripts/fetch-git-history.ts @@ -3,6 +3,8 @@ * shallow clone (every file untouched since the boundary commit would get that commit's date), so * the docs sitemap shipped with no `lastModified` at all. Fetch the commit history before the * build. `--filter=blob:none` skips file contents: `git log --name-only` needs commits and trees. + * Vercel's clone has no `origin` remote, so the history comes from the public repository URL its + * system environment names, pinned to the commit being built. * * A failed fetch leaves the clone shallow, so pages ship undated as before. That is logged rather * than failing the deploy, because dates are an enhancement and the docs must still ship. @@ -11,7 +13,8 @@ import { execFileSync } from 'node:child_process' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -const logger = createLogger('DocsGitHistory') +/** A build step's outcome belongs in the build log, which the production default (errors only) would hide. */ +const logger = createLogger('DocsGitHistory', { logLevel: 'INFO' }) /** Bounds the fetch so a stalled remote costs at most this much of the deployment, never all of it. */ const FETCH_TIMEOUT_MS = 120_000 @@ -24,11 +27,40 @@ function git(args: string[], timeout?: number): string { }).trim() } -if (process.env.VERCEL) { +const { + VERCEL, + VERCEL_GIT_PROVIDER, + VERCEL_GIT_REPO_OWNER, + VERCEL_GIT_REPO_SLUG, + VERCEL_GIT_COMMIT_SHA, +} = process.env + +if (VERCEL) { try { if (git(['rev-parse', '--is-shallow-repository']) === 'true') { - git(['fetch', '--unshallow', '--filter=blob:none', '--no-tags', 'origin'], FETCH_TIMEOUT_MS) + if ( + VERCEL_GIT_PROVIDER !== 'github' || + !VERCEL_GIT_REPO_OWNER || + !VERCEL_GIT_REPO_SLUG || + !VERCEL_GIT_COMMIT_SHA + ) { + throw new Error('Vercel did not provide the GitHub repository and commit for this build') + } + const repositoryUrl = `https://github.com/${VERCEL_GIT_REPO_OWNER}/${VERCEL_GIT_REPO_SLUG}.git` + git( + [ + 'fetch', + '--unshallow', + '--filter=blob:none', + '--no-tags', + repositoryUrl, + VERCEL_GIT_COMMIT_SHA, + ], + FETCH_TIMEOUT_MS + ) logger.info('Fetched full commit history for docs last-modified dates') + } else { + logger.info('Clone already has full commit history') } } catch (error) { logger.warn( diff --git a/apps/docs/scripts/vercel-ignore-build.sh b/apps/docs/scripts/vercel-ignore-build.sh new file mode 100755 index 00000000000..321715f109b --- /dev/null +++ b/apps/docs/scripts/vercel-ignore-build.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Vercel's Ignored Build Step for the docs project (`ignoreCommand` in vercel.json), run from +# apps/docs before install. Exit 0 skips the build; exit 1 builds. +# +# Vercel's "skip unaffected projects" already drops commits that touch neither docs nor a +# workspace package it depends on. Of the rest, production and the staging integration branch +# always build, so a shared-package change that breaks the docs build still fails on staging +# before it reaches main. Any other preview branch builds only when it changes docs itself. +# +# Anything this script cannot decide builds: a branch's first deployment (no previous SHA) and a +# previous deployment outside the shallow clone. +set -uo pipefail + +if [[ "${VERCEL_ENV:-}" == production || "${VERCEL_GIT_COMMIT_REF:-}" == staging ]]; then + exit 1 +fi + +previous=${VERCEL_GIT_PREVIOUS_SHA:-} +if [[ -z "$previous" ]] || ! git cat-file -e "${previous}^{commit}" 2> /dev/null; then + echo "No previous deployment in this clone; building." + exit 1 +fi + +if git diff --quiet "$previous" HEAD -- .; then + echo "No docs changes since ${previous}; skipping this preview." + exit 0 +fi +exit 1 diff --git a/apps/docs/vercel.json b/apps/docs/vercel.json new file mode 100644 index 00000000000..63ab788af4b --- /dev/null +++ b/apps/docs/vercel.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://openapi.vercel.sh/vercel.json", + "ignoreCommand": "bash scripts/vercel-ignore-build.sh" +} diff --git a/apps/realtime/package.json b/apps/realtime/package.json index 4f3914782d2..e1ab2cd0645 100644 --- a/apps/realtime/package.json +++ b/apps/realtime/package.json @@ -5,7 +5,7 @@ "license": "Apache-2.0", "type": "module", "engines": { - "bun": ">=1.4.2", + "bun": ">=1.4.4", "node": ">=20.0.0" }, "scripts": { diff --git a/apps/sim/app/(landing)/blog/[slug]/page.tsx b/apps/sim/app/(landing)/blog/[slug]/page.tsx index 89385e4b472..0fe6519f7d9 100644 --- a/apps/sim/app/(landing)/blog/[slug]/page.tsx +++ b/apps/sim/app/(landing)/blog/[slug]/page.tsx @@ -24,7 +24,7 @@ export async function generateMetadata({ const { slug } = await params const post = await getPostBySlug(slug) if (!post || post.draft) return {} - return buildPostMetadata(post) + return buildPostMetadata(post, BLOG_SECTION) } export const revalidate = 86400 @@ -33,7 +33,7 @@ export default async function Page({ params }: { params: Promise<{ slug: string const { slug } = await params const post = await getPostBySlug(slug) if (!post || post.draft) notFound() - const related = await getRelatedPosts(slug, 3) + const related = await getRelatedPosts(slug) return ( { const { slug } = await params const entry = await getEntryBySlug(slug) - return entry ? buildPostMetadata(entry) : {} + return entry ? buildPostMetadata(entry, CHANGELOG_SECTION) : {} } export default async function ChangelogEntryPage({ params }: ChangelogEntryPageProps) { diff --git a/apps/sim/app/(landing)/comparisons/components/comparison-link-row/comparison-link-row.tsx b/apps/sim/app/(landing)/comparisons/components/comparison-link-row/comparison-link-row.tsx index 4f7728df801..7c0502594a3 100644 --- a/apps/sim/app/(landing)/comparisons/components/comparison-link-row/comparison-link-row.tsx +++ b/apps/sim/app/(landing)/comparisons/components/comparison-link-row/comparison-link-row.tsx @@ -1,6 +1,7 @@ import Link from 'next/link' import type { CompetitorProfile } from '@/lib/compare/data' import { BrandIconTile } from '@/app/(landing)/comparisons/components/brand-icon-tile' +import { comparisonPath } from '@/app/(landing)/comparisons/utils' import { ChevronArrow } from '@/app/(landing)/components/chevron-arrow' interface ComparisonLinkRowProps { @@ -12,7 +13,7 @@ export function ComparisonLinkRow({ competitor }: ComparisonLinkRowProps) { const Icon = competitor.brand?.icon return ( diff --git a/apps/sim/app/(landing)/comparisons/library-links.test.ts b/apps/sim/app/(landing)/comparisons/library-links.test.ts new file mode 100644 index 00000000000..67eac032eb9 --- /dev/null +++ b/apps/sim/app/(landing)/comparisons/library-links.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest' +import type { ContentMeta } from '@/lib/content/schema' +import { getComparisonsForPost } from '@/app/(landing)/comparisons/library-links' + +function post(title: string, description: string, tags: string[] = []): ContentMeta { + return { slug: 'post', title, description, tags } as ContentMeta +} + +describe('getComparisonsForPost', () => { + it('ranks a bare common-word competitor first when the title names its alternatives', () => { + const [first] = getComparisonsForPost( + post('Best Make Alternatives for AI Agents', 'How Zapier and n8n compare for AI agents.'), + new Set() + ) + expect(first?.name).toBe('Make') + }) + + it('does not read the bare common word elsewhere in a title as that competitor', () => { + const comparisons = getComparisonsForPost( + post('How to Make and Ship AI Agent Alternatives', 'A guide to building AI agents.'), + new Set() + ) + expect(comparisons.map(({ name }) => name)).not.toContain('Make') + }) +}) diff --git a/apps/sim/app/(landing)/comparisons/library-links.ts b/apps/sim/app/(landing)/comparisons/library-links.ts index 990cb23dd0b..3b6ddf616d9 100644 --- a/apps/sim/app/(landing)/comparisons/library-links.ts +++ b/apps/sim/app/(landing)/comparisons/library-links.ts @@ -1,31 +1,49 @@ import { escapeRegExp } from '@sim/utils/string' import type { CompetitorProfile } from '@/lib/compare/data' import type { ContentMeta } from '@/lib/content/schema' -import { getAllPostMeta } from '@/lib/library/registry' -import { ALL_COMPETITORS } from '@/app/(landing)/comparisons/utils' +import { getAllPostMeta, getPostsLinkingTo } from '@/lib/library/registry' +import { ALL_COMPETITORS, comparisonPath } from '@/app/(landing)/comparisons/utils' /** Most links either side renders, so the section stays a short "read next" list. */ -const MAX_LINKS = 3 +const MAX_LINKS = 6 -/** A competitor named in an article's title or tags is its subject; one named only in the description is a mention. */ -const SUBJECT_SCORE = 2 +/** + * An article titled "{Competitor} Alternatives" answers the same buyer question as the + * comparison page, so it leads, then articles about the competitor, articles linking to its + * comparison page, and articles naming it only in the description. + */ +const ALTERNATIVES_SCORE = 4 +const SUBJECT_SCORE = 3 +const LINKS_TO_COMPARISON_SCORE = 2 const MENTION_SCORE = 1 interface MentionPatterns { + /** + * Matches a title naming the competitor's alternatives ("n8n Alternatives", "LangChain and LangGraph + * Alternatives"). A bare common-word name ("Make") counts only directly before "Alternatives". + */ + alternatives: RegExp /** Matches Title Case titles and tags, where a bare common-word name ("Make") is ambiguous. */ subject: RegExp /** Matches sentence-case descriptions, where the bare name is unambiguous. */ mention: RegExp } +function phraseAlternation(phrases: string[]): string { + return `(?:${phrases.map(escapeRegExp).join('|')})` +} + /** Whole-word, case-sensitive match on any of `phrases`. */ function wordPattern(phrases: string[]): RegExp { - return new RegExp(`\\b(?:${phrases.map(escapeRegExp).join('|')})\\b`) + return new RegExp(`\\b${phraseAlternation(phrases)}\\b`) } function buildMentionPatterns(competitor: CompetitorProfile): MentionPatterns { const aliases = competitor.mentions ?? [competitor.name] return { + alternatives: new RegExp( + `\\b(?:${phraseAlternation(aliases)}(?: and [^,:]+?)?|${escapeRegExp(competitor.name)}) Alternatives\\b` + ), subject: wordPattern(aliases), mention: wordPattern([competitor.name, ...aliases]), } @@ -36,8 +54,14 @@ const COMPETITOR_PATTERNS = ALL_COMPETITORS.map((competitor) => ({ patterns: buildMentionPatterns(competitor), })) -function scoreRelevance(post: ContentMeta, { subject, mention }: MentionPatterns): number { +function scoreRelevance( + post: ContentMeta, + { alternatives, subject, mention }: MentionPatterns, + linksToComparison: boolean +): number { + if (alternatives.test(post.title)) return ALTERNATIVES_SCORE if (subject.test(post.title) || post.tags.some((tag) => subject.test(tag))) return SUBJECT_SCORE + if (linksToComparison) return LINKS_TO_COMPARISON_SCORE return mention.test(post.description) ? MENTION_SCORE : 0 } @@ -52,22 +76,30 @@ function topByScore(items: T[], score: (item: T) => number): T[] { } /** - * Library articles about or naming `competitor`, articles where it is the - * subject first, then newest first. + * Library articles about, linking to, or naming `competitor`, strongest match first, then + * newest first. */ export async function getLibraryPostsForCompetitor( competitor: CompetitorProfile ): Promise { const patterns = buildMentionPatterns(competitor) - return topByScore(await getAllPostMeta(), (post) => scoreRelevance(post, patterns)) + const [posts, linkingPosts] = await Promise.all([ + getAllPostMeta(), + getPostsLinkingTo(comparisonPath(competitor)), + ]) + const linkingSlugs = new Set(linkingPosts.map((post) => post.slug)) + return topByScore(posts, (post) => scoreRelevance(post, patterns, linkingSlugs.has(post.slug))) } /** - * Comparison pages for the competitors a library article is about or names, - * subjects first, then in {@link ALL_COMPETITORS} order. + * Comparison pages for the competitors a library article is about, links to (`linkedPaths`, + * from its body), or names, strongest match first, then in {@link ALL_COMPETITORS} order. */ -export function getComparisonsForPost(post: ContentMeta): CompetitorProfile[] { - return topByScore(COMPETITOR_PATTERNS, ({ patterns }) => scoreRelevance(post, patterns)).map( - ({ competitor }) => competitor - ) +export function getComparisonsForPost( + post: ContentMeta, + linkedPaths: ReadonlySet +): CompetitorProfile[] { + return topByScore(COMPETITOR_PATTERNS, ({ competitor, patterns }) => + scoreRelevance(post, patterns, linkedPaths.has(comparisonPath(competitor))) + ).map(({ competitor }) => competitor) } diff --git a/apps/sim/app/(landing)/comparisons/utils.ts b/apps/sim/app/(landing)/comparisons/utils.ts index 2c1a962e142..94c9d0fd6d2 100644 --- a/apps/sim/app/(landing)/comparisons/utils.ts +++ b/apps/sim/app/(landing)/comparisons/utils.ts @@ -54,6 +54,11 @@ export const ALL_COMPETITORS: CompetitorProfile[] = [ const COMPETITOR_BY_SLUG = new Map(ALL_COMPETITORS.map((c) => [c.id, c])) +/** Path of the Sim vs `competitor` comparison page. */ +export function comparisonPath(competitor: CompetitorProfile): string { + return `/comparisons/${competitor.id}` +} + export function getCompetitorBySlug(slug: string): CompetitorProfile | null { return COMPETITOR_BY_SLUG.get(slug) ?? null } diff --git a/apps/sim/app/(landing)/components/content-index-page/content-index-page.tsx b/apps/sim/app/(landing)/components/content-index-page/content-index-page.tsx index e4ca2bcf9b5..c0d735ab346 100644 --- a/apps/sim/app/(landing)/components/content-index-page/content-index-page.tsx +++ b/apps/sim/app/(landing)/components/content-index-page/content-index-page.tsx @@ -36,8 +36,12 @@ export function ContentIndexPage({ }: ContentIndexPageProps) { const { featured, remaining, totalPages } = paginateContentPosts(posts, { tag, page }) - const pageHref = (targetPage: number) => - `${basePath}?page=${targetPage}${tag ? `&tag=${encodeURIComponent(tag)}` : ''}` + const tagQuery = tag ? `tag=${encodeURIComponent(tag)}` : '' + const pageHref = (targetPage: number) => { + if (targetPage === 1) return tagQuery ? `${basePath}?${tagQuery}` : basePath + return `${basePath}?page=${targetPage}${tagQuery ? `&${tagQuery}` : ''}` + } + const pageNumbers = Array.from({ length: totalPages }, (_, index) => index + 1) return ( <> @@ -138,7 +142,7 @@ export function ContentIndexPage({ {totalPages > 1 && (