feat: schedule ingestion engine

.claude/settings.local.json

@@ -19,7 +19,10 @@
       "Bash(npx oxlint:*)",
       "WebSearch",
       "WebFetch(domain:tanstack.com)",
-      "Bash(node:*)"
+      "Bash(node:*)",
+      "Bash(deno test *)",
+      "Bash(git commit -m ' *)",
+      "Bash(pnpm vitest *)"
     ],
     "deny": []
   }

.github/workflows/_db_migrate.yml

@@ -28,9 +28,7 @@ jobs:
 
       - uses: actions/checkout@v4
 
-      - uses: supabase/setup-cli@v1
-        with:
-          version: 2.58.5
+      - uses: supabase/setup-cli@v2
 
       - name: Push migrations
         env:

.github/workflows/_deploy_edge_functions.yml

@@ -28,13 +28,11 @@ jobs:
 
       - uses: actions/checkout@v4
 
-      - uses: supabase/setup-cli@v1
-        with:
-          version: 2.58.5
+      - uses: supabase/setup-cli@v2
 
       - name: Deploy functions
         env:
           SUPABASE_ACCESS_TOKEN: ${{ secrets.SUPABASE_ACCESS_TOKEN }}
           PROJECT_REF: ${{ inputs.target == 'prod' && vars.PROD_PROJECT_REF || vars.STAGING_PROJECT_REF }}
         run: |
-          supabase functions deploy --project-ref "$PROJECT_REF"
+          supabase functions deploy --use-api --project-ref "$PROJECT_REF"

.github/workflows/unit-tests.yml

@@ -42,3 +42,26 @@ jobs:
           name: coverage-report
           path: coverage/
           retention-days: 7
+
+  deno-test:
+    name: Run Edge Function Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v2.x
+
+      - name: Run Deno tests for Edge Functions
+        run: |
+          set -e
+          for fn in supabase/functions/*/; do
+            if compgen -G "$fn"*.test.ts > /dev/null; then
+              echo "::group::deno test $fn"
+              (cd "$fn" && deno test --allow-env --allow-net --allow-read)
+              echo "::endgroup::"
+            fi
+          done

.gitignore

@@ -40,4 +40,4 @@ dev-dist
 dump.sql
 
 # Sync script credentials (DB connection strings — never commit)
-scripts/.env.sync
+scripts/.env.sync

package.json

@@ -24,8 +24,7 @@
     "test:e2e:report": "playwright show-report",
     "test:setup": "bash scripts/setup-test-env.sh",
     "test:setup:full": "bash scripts/setup-local-supabase.sh",
-    "types:generate": "supabase gen types typescript --project-id qssmazlqrmxiudxckxvi > src/integrations/supabase/types.ts",
-    "types:generate:local": "supabase gen types typescript --local > src/integrations/supabase/types.ts",
+    "types:generate": "bash scripts/gen-types.sh",
     "db:sync:staging": "bash scripts/sync-from-prod.sh staging",
     "db:sync:local": "bash scripts/sync-from-prod.sh local",
     "db:recreate:staging": "bash scripts/recreate-staging.sh",
@@ -61,7 +60,7 @@
     "@radix-ui/react-toggle": "^1.1.0",
     "@radix-ui/react-toggle-group": "^1.1.0",
     "@radix-ui/react-tooltip": "^1.1.4",
-    "@supabase/supabase-js": "^2.81.1",
+    "@supabase/supabase-js": "^2.105.0",
     "@tailwindcss/line-clamp": "^0.4.4",
     "@tanstack/query-async-storage-persister": "^5.86.0",
     "@tanstack/react-query": "^5.56.2",
@@ -84,6 +83,7 @@
     "lucide-react": "^0.462.0",
     "marked": "^16.3.0",
     "next-themes": "^0.3.0",
+    "papaparse": "^5.5.3",
     "posthog-js": "^1.277.0",
     "react": "^18.3.1",
     "react-day-picker": "^9.8.0",
@@ -102,15 +102,15 @@
     "zod": "^3.23.8"
   },
   "devDependencies": {
-    "vite-plugin-pwa": "^1.2.0",
     "@playwright/test": "^1.54.1",
     "@tailwindcss/typography": "^0.5.15",
+    "@tanstack/router-devtools": "^1.136.18",
+    "@tanstack/router-plugin": "^1.136.18",
     "@testing-library/jest-dom": "^6.9.1",
     "@testing-library/react": "^16.3.0",
     "@testing-library/user-event": "^14.6.1",
-    "@tanstack/router-devtools": "^1.136.18",
-    "@tanstack/router-plugin": "^1.136.18",
     "@types/node": "^22.5.5",
+    "@types/papaparse": "^5.5.2",
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
     "@vitejs/plugin-react-swc": "^4.2.2",
@@ -130,6 +130,7 @@
     "tsx": "^4.20.3",
     "typescript": "^5.5.3",
     "vite": "^7.2.7",
+    "vite-plugin-pwa": "^1.2.0",
     "vitest": "^4.0.15"
   },
   "lint-staged": {

scripts/gen-types.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Generate Supabase TypeScript types for both the app and the Edge Functions.
+# Pass --local to generate from the local database instead of the remote project.
+
+set -euo pipefail
+
+PROJECT_ID="qssmazlqrmxiudxckxvi"
+APP_TYPES="src/integrations/supabase/types.ts"
+EDGE_TYPES="supabase/functions/_shared/database.types.ts"
+
+if [[ "${1:-}" == "--local" ]]; then
+  supabase gen types typescript --local | tee "$APP_TYPES" > "$EDGE_TYPES"
+else
+  supabase gen types typescript --project-id "$PROJECT_ID" | tee "$APP_TYPES" > "$EDGE_TYPES"
+fi