diff --git a/.github/workflows/integration-checks.yml b/.github/workflows/integration-checks.yml new file mode 100644 index 0000000..0b03151 --- /dev/null +++ b/.github/workflows/integration-checks.yml @@ -0,0 +1,83 @@ +name: Integration Checks + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: integration-${{ github.ref }} + cancel-in-progress: true + +jobs: + indexer: + name: Indexer · codegen · build · tests · manifest sync + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Cache Bun dependencies + uses: actions/cache@v4 + with: + path: ~/.bun/install/cache + key: ${{ runner.os }}-bun-${{ hashFiles('bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Indexer codegen + run: bun run --cwd apps/s03-indexer codegen + + - name: Indexer build + run: bun run --cwd apps/s03-indexer build + + - name: Indexer mapping unit tests + run: bun run --cwd apps/s03-indexer test + + - name: Manifest sync (offline fixtures) + env: + SO4_CONTRACTS_REPO: ${{ github.workspace }}/apps/s03-indexer/tests/fixtures/contracts-repo + run: bun run --cwd apps/s03-indexer sync:contracts:local + + - name: Validate generated manifest + run: bash scripts/validate-manifest.sh apps/s03-indexer/config/contracts.local.json + + web: + name: Web · typecheck · build + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Cache Bun dependencies + uses: actions/cache@v4 + with: + path: ~/.bun/install/cache + key: ${{ runner.os }}-bun-${{ hashFiles('bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Web typecheck + run: bun run --cwd apps/web typecheck + + - name: Web build + run: bun run --cwd apps/web build diff --git a/README.md b/README.md index f4cf6ee..e20ed43 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,14 @@ cd so4-monorepo bun install ``` +### Full local stack (contracts + indexer + web) + +If you need the indexer and contracts running against the web app, follow the +[Local Full-Stack Integration Guide](./docs/local-full-stack.md). It documents +prerequisites, deploy/bootstrap, manifest sync, indexer start, smoke flow, +GraphQL verification, UI verification, troubleshooting, and the definition of +done. Run `bun run check:integration` before pushing to mirror the CI matrix. + ### Running the development server ```bash diff --git a/apps/s03-indexer/README.md b/apps/s03-indexer/README.md index 5a01bc2..7318872 100644 --- a/apps/s03-indexer/README.md +++ b/apps/s03-indexer/README.md @@ -4,6 +4,12 @@ This Bun workspace package contains the SubQuery Stellar indexer scaffold for the S03 app. It indexes Stellar testnet payments, account credit/debit effects, and Soroban `transfer` events. +> **New contributor?** Start with the top-level +> [Local Full-Stack Integration Guide](../../docs/local-full-stack.md). It +> covers prerequisites, the deploy → sync → index → query → UI path end-to-end, +> troubleshooting, and the definition of done. Use the rest of this README as +> the indexer-specific reference. + ## Workspace Commands Run these commands from the repository root: diff --git a/apps/s03-indexer/tests/fixtures/contracts-repo/.deployed/frontend-local.env b/apps/s03-indexer/tests/fixtures/contracts-repo/.deployed/frontend-local.env new file mode 100644 index 0000000..c314d79 --- /dev/null +++ b/apps/s03-indexer/tests/fixtures/contracts-repo/.deployed/frontend-local.env @@ -0,0 +1,7 @@ +VITE_NETWORK=local +VITE_NETWORK_PASSPHRASE=Standalone Network ; February 2017 +VITE_HORIZON_URL=http://host.docker.internal:8000 +VITE_RPC_URL=http://host.docker.internal:8000/soroban/rpc +VITE_EXCHANGE_ROUTER=CBD6BQSQFROWIIT5QCYN7KL5LJJWUIH7CEWUSZIFMUJO6NPXE6CVGYNW +VITE_READER=CC6OZUHF3LVO6PNP3V2EB36ORB3YSVYSH3LWD3RFLO4NUO3BYCXSWSYC +VITE_DATA_STORE=CCZ3VKBEDLNBO2JM3EXL3SNBDJOV5BTN52FVQPER7F6D5GCE53PITQ3J diff --git a/apps/s03-indexer/tests/fixtures/contracts-repo/.deployed/frontend-local.ts b/apps/s03-indexer/tests/fixtures/contracts-repo/.deployed/frontend-local.ts new file mode 100644 index 0000000..ceb3a90 --- /dev/null +++ b/apps/s03-indexer/tests/fixtures/contracts-repo/.deployed/frontend-local.ts @@ -0,0 +1,3 @@ +export const networkPassphrase = "Standalone Network ; February 2017"; +export const rpcUrl = "http://host.docker.internal:8000/soroban/rpc"; +export const horizonUrl = "http://host.docker.internal:8000"; diff --git a/apps/s03-indexer/tests/fixtures/contracts-repo/.deployed/local.env b/apps/s03-indexer/tests/fixtures/contracts-repo/.deployed/local.env new file mode 100644 index 0000000..07a8584 --- /dev/null +++ b/apps/s03-indexer/tests/fixtures/contracts-repo/.deployed/local.env @@ -0,0 +1,17 @@ +NETWORK=local +ROLE_STORE=CBSUAIAMIFFS4AXQYZ7KR7FNO7IMKAPS5WF4DXANVXDTPKH2F7YUIN6Q +DATA_STORE=CCZ3VKBEDLNBO2JM3EXL3SNBDJOV5BTN52FVQPER7F6D5GCE53PITQ3J +ORACLE=CBEMTV23SIJJBIST3V5HTMWHR4MHYGHNBIG4M26U4LGUJTWZXTFSVQEY +MARKET_FACTORY=CBGX3EJFI3JRHSN5B533O2L5P57JFPTCRS55IPWFS5BNDXLJLXDWA5Z2 +DEPOSIT_HANDLER=CDWOFIP4YQJGMCYAOWLSRBAWN2OTJUG2I5WOFC32O2TX2SRU56RWBE5C +WITHDRAWAL_HANDLER=CBRWM6PNRRFL5RSTJH6HWEXBTMWCGLQRO45NTRDB6BBABWXZ4ZE7DGTO +ORDER_HANDLER=CC35OFZVWUTAZPV3B6UKSDVAVORZEWUUMOMTHO33H4YR4C5FKPEFODKY +LIQUIDATION_HANDLER=CBXUAR5GCHIRFQL75WTZS3FLA6SMWDPIKG4EKNPWVQVNGVFXBHGTJHTM +ADL_HANDLER=CACFPG3QAKG6DCAJSOP7YGDTM44NV6NPI3SKAG7GUGIV6DMGXPCAMMME +FEE_HANDLER=CC4P3FJ7EAH6F3RYJPQ2T7VIB4I7UJ4EEYGVWTZVXTAUN647QRVSDHS4 +REFERRAL_STORAGE=CDHTPQO4RRJ6OUBIW3GDXTIVLVOMIKPJC65PGDJH2G5OLDJRE5KTROWK +READER=CC6OZUHF3LVO6PNP3V2EB36ORB3YSVYSH3LWD3RFLO4NUO3BYCXSWSYC +EXCHANGE_ROUTER=CBD6BQSQFROWIIT5QCYN7KL5LJJWUIH7CEWUSZIFMUJO6NPXE6CVGYNW +MARKET_TOKEN_TETH_TUSDC=CCBUUSYZJTGVA6PYUNQDFPZFHTBZ2QSHOUO7YAGRQVA46T3ZLSIYULS4 +MARKET_TOKEN_TWBTC_TUSDC=CDDVSLBGGDV2UOFN5W72R4LW7ABYL7H7ZWVSFHGMXXB3D52ZYANC5G3L +MARKET_TOKEN_TXLM_TUSDC=CDIBR7BDCDWGAG3CC6PBKRSLMISPYKNDGE57DCZO5TMTLZK34TMGKFQQ diff --git a/apps/s03-indexer/tests/fixtures/contracts-repo/.deployed/tokens-local.env b/apps/s03-indexer/tests/fixtures/contracts-repo/.deployed/tokens-local.env new file mode 100644 index 0000000..79a5521 --- /dev/null +++ b/apps/s03-indexer/tests/fixtures/contracts-repo/.deployed/tokens-local.env @@ -0,0 +1,5 @@ +TUSDC=CBAN5YU3KRDKPTQ2H76D6S7HQFPRBGUD524F65BUM2RQCITPTRLKWKES +TWBTC=CCFTOPHUPSUDO2MB4X5D3XYJ2HRJ7NJPAW4UVPAVN7ZLE63EZLSMXDUO +TETH=CAJ6BZKGFT47ALGMVFZZGAOXBV2RWIVYVCU4WJCQIURKRNXU346RWVAU +TXLM=CAHNXBBSXVMGI6G3FUBY3OTNWKQ7434FDDEEE7ZT733WIW6NUZL4ONU6 +FAUCET=CCWXXBKXHHP5DXC6TYVIL22XUNHD5A75O6WM5D2KM5PY45IOV5VDMARJ diff --git a/apps/s03-indexer/tests/fixtures/contracts-repo/.stellar/contract-ids/local.json b/apps/s03-indexer/tests/fixtures/contracts-repo/.stellar/contract-ids/local.json new file mode 100644 index 0000000..7383b2e --- /dev/null +++ b/apps/s03-indexer/tests/fixtures/contracts-repo/.stellar/contract-ids/local.json @@ -0,0 +1,19 @@ +{ + "network": "local", + "network_passphrase": "Standalone Network ; February 2017", + "contracts": { + "role_store": "CBSUAIAMIFFS4AXQYZ7KR7FNO7IMKAPS5WF4DXANVXDTPKH2F7YUIN6Q", + "data_store": "CCZ3VKBEDLNBO2JM3EXL3SNBDJOV5BTN52FVQPER7F6D5GCE53PITQ3J", + "oracle": "CBEMTV23SIJJBIST3V5HTMWHR4MHYGHNBIG4M26U4LGUJTWZXTFSVQEY", + "market_factory": "CBGX3EJFI3JRHSN5B533O2L5P57JFPTCRS55IPWFS5BNDXLJLXDWA5Z2", + "deposit_handler": "CDWOFIP4YQJGMCYAOWLSRBAWN2OTJUG2I5WOFC32O2TX2SRU56RWBE5C", + "withdrawal_handler": "CBRWM6PNRRFL5RSTJH6HWEXBTMWCGLQRO45NTRDB6BBABWXZ4ZE7DGTO", + "order_handler": "CC35OFZVWUTAZPV3B6UKSDVAVORZEWUUMOMTHO33H4YR4C5FKPEFODKY", + "liquidation_handler": "CBXUAR5GCHIRFQL75WTZS3FLA6SMWDPIKG4EKNPWVQVNGVFXBHGTJHTM", + "adl_handler": "CACFPG3QAKG6DCAJSOP7YGDTM44NV6NPI3SKAG7GUGIV6DMGXPCAMMME", + "fee_handler": "CC4P3FJ7EAH6F3RYJPQ2T7VIB4I7UJ4EEYGVWTZVXTAUN647QRVSDHS4", + "referral_storage": "CDHTPQO4RRJ6OUBIW3GDXTIVLVOMIKPJC65PGDJH2G5OLDJRE5KTROWK", + "reader": "CC6OZUHF3LVO6PNP3V2EB36ORB3YSVYSH3LWD3RFLO4NUO3BYCXSWSYC", + "exchange_router": "CBD6BQSQFROWIIT5QCYN7KL5LJJWUIH7CEWUSZIFMUJO6NPXE6CVGYNW" + } +} diff --git a/apps/s03-indexer/tests/fixtures/contracts-repo/README.md b/apps/s03-indexer/tests/fixtures/contracts-repo/README.md new file mode 100644 index 0000000..80018be --- /dev/null +++ b/apps/s03-indexer/tests/fixtures/contracts-repo/README.md @@ -0,0 +1,37 @@ +# Fixture contracts repo + +This directory mirrors the on-disk layout of the real contracts repo +(`../contracts` relative to this interface repo) so the indexer sync script and +manifest validation checks can run **without a live Stellar deploy**. + +It is used by: + +- `scripts/check-integration.sh` — local pre-PR integration check. +- `.github/workflows/integration-checks.yml` — CI integration check. +- Manual reproduction: + + ```bash + SO4_CONTRACTS_REPO=apps/s03-indexer/tests/fixtures/contracts-repo \ + bun run --cwd apps/s03-indexer sync:contracts:local + ``` + +The IDs here are real testnet contract IDs reused as deterministic fixtures — +they are **not** intended to be deployed against or signed for. Treat them as +shape-valid sample data. + +## Layout + +``` +contracts-repo/ +├── .deployed/ +│ ├── local.env # core contracts + market token triplets +│ ├── tokens-local.env # TUSDC / TWBTC / TETH / TXLM / FAUCET +│ ├── frontend-local.env # frontend export env +│ └── frontend-local.ts # frontend export TypeScript (passphrase, RPC) +└── .stellar/ + └── contract-ids/ + └── local.json # combined network passphrase + contract IDs +``` + +Update these fixtures whenever the contracts repo adds a new core contract or +deployment file shape — they are the canonical sample input for offline checks. diff --git a/docs/local-full-stack.md b/docs/local-full-stack.md new file mode 100644 index 0000000..498a112 --- /dev/null +++ b/docs/local-full-stack.md @@ -0,0 +1,401 @@ +# SO4 Local Full-Stack Integration Guide + +This guide walks a new contributor through running the SO4 stack — **contracts**, +**indexer**, and **web app** — locally, from a clean clone, using a single +documented path. Follow it top to bottom: each step builds on the previous one, +and the verification commands at the end exist to catch regressions before +review. + +> **Scope:** `local` (a standalone/quickstart Stellar node + locally deployed +> contracts), `testnet` (public SDF testnet), and a forward note on `mainnet`. +> Local is the default development path. + +--- + +## Table of contents + +- [Prerequisites](#prerequisites) +- [Repository layout](#repository-layout) +- [1. Build the contracts](#1-build-the-contracts) +- [2. Deploy and bootstrap contracts](#2-deploy-and-bootstrap-contracts) +- [3. Sync manifests into the indexer](#3-sync-manifests-into-the-indexer) +- [4. Start the indexer services](#4-start-the-indexer-services) +- [5. Run the local smoke scenario](#5-run-the-local-smoke-scenario) +- [6. Start the web app](#6-start-the-web-app) +- [7. Verify GraphQL data](#7-verify-graphql-data) +- [8. Verify the UI is reading indexed data](#8-verify-the-ui-is-reading-indexed-data) +- [Expected URLs and ports](#expected-urls-and-ports) +- [Local vs testnet vs mainnet](#local-vs-testnet-vs-mainnet) +- [Troubleshooting](#troubleshooting) +- [Automated checks](#automated-checks) +- [Definition of done](#definition-of-done) + +--- + +## Prerequisites + +| Tool | Version / notes | +| --- | --- | +| [Bun](https://bun.sh) | `>= 1.3` — the workspace package manager and task runner. The root `package.json` pins `bun@1.3.13` via `packageManager`. | +| [Docker Engine](https://docs.docker.com/engine/) | With Docker Compose v2. The indexer stack (`postgres`, `subquery-node`, `graphql-engine`) runs in containers. | +| [Stellar CLI](https://developers.stellar.org/docs/build/smart-contracts/getting-started/setup#install-the-stellar-cli) | `stellar` must be on `PATH` for contract deploys and smoke runs. | +| [Rust toolchain](https://www.rust-lang.org/tools/install) | Stable Rust with the `wasm32-unknown-unknown` target for Soroban contracts: `rustup target add wasm32-unknown-unknown`. | +| `make` | Used by the contracts repo deploy/bootstrap targets. | +| Node.js | `>= 20` (Bun bundles its own runtime, but some tooling still expects Node on `PATH`). | + +### Stellar keys (local + testnet) + +The local smoke flow uses local-only Stellar CLI keys; production keys are never +touched. From the contracts repo (default sibling path +`/home/sunny/zero/so4-market-project/contracts`): + +```bash +stellar keys generate so4-local --network local --fund +stellar keys generate so4-testnet --network testnet --fund +``` + +These keys live under `~/.config/stellar/identity/` and are friendbot-funded +automatically when generated against a known network. + +--- + +## Repository layout + +``` +so4-market-project/ +├── interface/ # this repo (web app, indexer, docs) +│ ├── apps/ +│ │ ├── s03-indexer/ # SubQuery indexer (Bun workspace) +│ │ └── web/ # React + Vite trading UI +│ └── docs/ # you are here +└── contracts/ # SO4 Soroban contracts (sibling repo) + ├── .deployed/ # generated deployment env files + └── .stellar/ # generated contract ID / wasm hash JSON +``` + +The indexer's `sync:contracts:*` script reads `../contracts` by default. Override +with `SO4_CONTRACTS_REPO=/abs/path` if your layout differs. + +--- + +## 1. Build the contracts + +From the contracts repo: + +```bash +cd ../contracts +make build # cargo build + soroban contract build for every workspace contract +``` + +> Expected: `target/wasm32-unknown-unknown/release/*.wasm` exist for every +> handler/store/router contract. + +## 2. Deploy and bootstrap contracts + +```bash +# Local standalone network (default for development) +make deploy NETWORK=local +make bootstrap NETWORK=local + +# Or against public testnet +make deploy NETWORK=testnet +make bootstrap NETWORK=testnet +``` + +The deploy step writes: + +- `.deployed/.env` — every protocol contract ID + role config +- `.deployed/tokens-.env` — `TUSDC`, `TWBTC`, `TETH`, `TXLM`, `faucet` +- `.deployed/frontend-.env` and `.deployed/frontend-.ts` — + frontend-friendly exports +- `.stellar/contract-ids/.json` — combined contract ID + network passphrase + +Bootstrap creates markets and writes `MARKET_TOKEN_*` values back into the env +file. If you skip bootstrap, the indexer sync will warn (not fail) about missing +market tokens. + +## 3. Sync manifests into the indexer + +```bash +cd ../interface +bun run --cwd apps/s03-indexer sync:contracts:local # for local +bun run --cwd apps/s03-indexer sync:contracts:testnet # for testnet +``` + +This writes `apps/s03-indexer/config/contracts..json` containing +network metadata, Horizon/Soroban RPC endpoints, core contracts, test tokens, +and market triplets. `project.ts` picks the right manifest by reading +`INDEXER_NETWORK` (or the explicit `INDEXER_CONTRACTS_CONFIG` override) from +`apps/s03-indexer/.env`. + +## 4. Start the indexer services + +```bash +# One-shot clean checkout path: codegen + build + docker compose up +bun run indexer:dev +``` + +Or, step by step: + +```bash +bun run indexer:codegen # subql codegen — regenerates src/types/ +bun run indexer:build # subql build — produces dist/ mappings +bun run indexer:start # docker compose pull && up --remove-orphans +``` + +Services come up on: + +| Service | URL | +| --- | --- | +| GraphQL playground / query API | | +| Postgres (indexed entities + metadata) | `postgres://postgres:postgres@localhost:5432/postgres` | +| Stellar Horizon (local quickstart) | | +| Soroban RPC (local quickstart) | | + +## 5. Run the local smoke scenario + +`smoke:local` is the single end-to-end check that ties everything together. + +```bash +bun run --cwd apps/s03-indexer build +bun run --cwd apps/s03-indexer smoke:local +``` + +It runs preflight → service checks → deploy (if needed) → manifest sync → +indexer rebuild → contract actions → GraphQL assertions, and writes +`.smoke/report.json`. Re-run against a clean DB with +`bun run --cwd apps/s03-indexer smoke:clean`. + +See `apps/s03-indexer/README.md` for the full flag reference (`--mode`, +`--source`, `--keeper`, `--report`, `--skip-*` flags, and the `SMOKE_*` env-var +equivalents). + +## 6. Start the web app + +```bash +bun run --cwd apps/web dev +``` + +The web app comes up at in dev mode. (The indexer +GraphQL service also defaults to port `3000` — when you need both at once, run +the web app with `bun run --cwd apps/web dev --port 3001` or stop the indexer's +`graphql-engine` container.) + +For production builds: + +```bash +bun run --cwd apps/web build +``` + +## 7. Verify GraphQL data + +Open the GraphQL playground at and run: + +```graphql +{ + query { + markets(first: 5) { + totalCount + nodes { id marketToken indexToken longToken shortToken } + } + deposits(first: 5, orderBy: LEDGER_DESC) { + nodes { id account market amount ledger } + } + positions(first: 5) { + nodes { id account market sizeUsd entryPrice } + } + } +} +``` + +Expected after a successful smoke run: + +- `markets.totalCount >= 1` (at least the bootstrapped market) +- `deposits` shows the smoke-run deposit +- `positions` shows the long opened by the `MarketIncrease` step + +If counts are zero, jump to [Troubleshooting → indexer behind latest +ledger](#indexer-is-behind-latest-ledger). + +## 8. Verify the UI is reading indexed data + +With the web app running: + +1. Open (or `:3001` if you remapped). +2. Connect a wallet funded against the same network you deployed to. +3. The **Earn** page should list the bootstrapped market with non-zero pool + metadata sourced from `pools` / `markets` GraphQL queries. +4. The **Trade** page chart should resolve the market by `marketToken` and + surface any open positions / orders for the connected account. + +If the UI shows empty state while GraphQL has data, check the browser network +tab — the most common cause is the web app pointing at the wrong GraphQL +endpoint (see `apps/web/src/lib/*` and any `VITE_GRAPHQL_*` env vars). + +--- + +## Expected URLs and ports + +| What | Where | +| --- | --- | +| Web app (Vite dev) | (default) or `:3001` when remapped | +| SubQuery GraphQL playground | | +| Postgres | `localhost:5432` (user/db: `postgres`) | +| Stellar Horizon (local) | | +| Soroban RPC (local) | | +| Stellar Horizon (testnet) | | +| Soroban RPC (testnet) | | +| Docker services | `docker compose -f apps/s03-indexer/docker-compose.yml ps` | + +--- + +## Local vs testnet vs mainnet + +| Environment | When to use | Endpoint defaults | Indexer config | +| --- | --- | --- | --- | +| **local** | Fast iteration, fixture data, deterministic test runs | `host.docker.internal:8000` Horizon + Soroban RPC | `config/contracts.local.json`, `INDEXER_NETWORK=local` | +| **testnet** | Shared remote network, multi-contributor reproduction | `horizon-testnet.stellar.org` + `soroban-testnet.stellar.org` | `config/contracts.testnet.json`, `INDEXER_NETWORK=testnet` | +| **mainnet** | **Not yet supported.** Future: separate manifest + RPC endpoints + signed deploys; never reuse local/testnet keys. | + +Keep local and testnet manifests in version control side-by-side — +`sync:contracts:*` writes to network-specific files and never overwrites the +other. + +--- + +## Troubleshooting + +### Missing `.deployed/.env` + +**Symptom:** `sync:contracts:*` fails with `deployment file not found` or +`smoke:local` exits at the `contracts-deploy` step. + +**Fix:** You have not deployed contracts for that network yet. Run +`make deploy NETWORK=` in the contracts repo. If you deployed but the +file is in an unexpected location, set `SO4_CONTRACTS_REPO=/abs/path` so the +sync script can find it. + +### Wrong network passphrase + +**Symptom:** `Soroban RPC rejects transaction: NetworkMismatch`, or the indexer +silently produces zero events even though Horizon shows transactions. + +**Fix:** The passphrase in `.env` / generated config must match the network you +deployed against. Local standalone uses +`Standalone Network ; February 2017`; testnet uses +`Test SDF Network ; September 2015`. Re-run +`bun run --cwd apps/s03-indexer sync:contracts:` to regenerate from +the source of truth and double-check `INDEXER_NETWORK` in +`apps/s03-indexer/.env`. + +### Stale market token IDs + +**Symptom:** Web app shows the right markets but UI rows look empty, or smoke +asserts the wrong number of `position` entities. + +**Fix:** Market tokens are created during `make bootstrap`. If you re-bootstrap +without re-syncing, the indexer keeps watching the previous tokens. Always run +`sync:contracts:` after `make bootstrap`, then rebuild the indexer +(`bun run indexer:build`) and restart it. + +### SubQuery database not ready + +**Symptom:** `subquery-node` logs `connection refused` or `relation does not +exist` errors on startup. + +**Fix:** Postgres needs a few seconds to initialize on first boot. Stop the +stack, then start it again: + +```bash +docker compose -f apps/s03-indexer/docker-compose.yml down +docker compose -f apps/s03-indexer/docker-compose.yml up +``` + +If the error persists, wipe state with +`bun run --cwd apps/s03-indexer smoke:clean` (drops the DB volume) and re-start. + +### Indexer is behind latest ledger + +**Symptom:** GraphQL returns zero rows for an event you just submitted, or the +smoke run times out in the `graphql-query` step. + +**Fix:** The indexer cursors lag the chain by a few ledgers under normal load. +Check `subquery-node` logs for the current cursor vs Horizon's latest ledger: + +```bash +docker compose -f apps/s03-indexer/docker-compose.yml logs -f subquery-node +``` + +If the lag is growing instead of catching up, you likely have the wrong +endpoint, wrong passphrase, or a malformed manifest. Re-run sync and rebuild. + +### Missing local token balances or approvals + +**Symptom:** Smoke step `contract-action` fails on the deposit/order with +`InsufficientBalance` or `AllowanceMissing`. + +**Fix:** The faucet has not been claimed for the smoke-run key, or token +allowances against `exchange_router` / `deposit_handler` were not set. The +default smoke flow claims the faucet and approves tokens automatically; if you +ran it with `--skip-*` flags, do the missed steps manually using the contracts +repo `make faucet` / `make approve` targets. As a last resort, regenerate keys +and re-run with `--mode fresh`. + +### Web app shows "no data" with healthy GraphQL + +**Symptom:** GraphQL playground returns rows; the UI still renders empty +states. + +**Fix:** The web app is pointing at the wrong endpoint or has cached an old +schema. Check `apps/web/.env` and any `VITE_GRAPHQL_*` variables; then hard +reload the browser (the TanStack Query cache and service worker can hold stale +nulls). + +--- + +## Automated checks + +Run these from the repo root before opening a PR. They mirror the CI workflow +under `.github/workflows/` and catch the most common schema / mapping / config +regressions: + +```bash +bun install # workspace install +bun run --cwd apps/s03-indexer codegen # schema -> types +bun run --cwd apps/s03-indexer build # mappings build +bun run --cwd apps/s03-indexer test # mapping unit tests +bun run --cwd apps/web typecheck # web app type safety +bun run --cwd apps/web build # web app production build +``` + +Manifest sync also has a fixture-based check that does not require a live +Stellar deploy. See `scripts/check-integration.sh` for the bundled local check +script and `apps/s03-indexer/tests/fixtures/` for offline deployment fixtures. + +--- + +## Definition of done + +A local integration is "done" when **every** box below is true: + +- [ ] `bun install` completes from a clean clone with no warnings about + `npm`/`yarn`/`pnpm` lockfiles. +- [ ] Contracts deploy + bootstrap against the chosen network (`local` or + `testnet`) and produce a complete `.deployed/.env`. +- [ ] `bun run --cwd apps/s03-indexer sync:contracts:` writes a fresh + `config/contracts..json` with the matching network passphrase, + every core contract, every test token, and the bootstrapped market + triplets. +- [ ] `bun run --cwd apps/s03-indexer codegen` succeeds. +- [ ] `bun run --cwd apps/s03-indexer build` succeeds. +- [ ] `bun run --cwd apps/s03-indexer test` passes (mapping unit tests). +- [ ] `bun run --cwd apps/s03-indexer smoke:local` exits zero and the resulting + `.smoke/report.json` shows passing assertions for every layer. +- [ ] GraphQL playground at returns non-zero rows for + `markets`, `deposits`, and `positions`. +- [ ] `bun run --cwd apps/web typecheck` passes. +- [ ] `bun run --cwd apps/web build` produces a clean production bundle. +- [ ] `bun run --cwd apps/web dev` shows the same markets, deposits, and + positions in the UI that the GraphQL playground returned. +- [ ] The contributor can hand off the running stack to a teammate using only + this document plus the smoke `report.json` for diagnostics. diff --git a/package.json b/package.json index 2b90670..71d4acc 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "indexer:codegen": "bun run --cwd apps/s03-indexer codegen", "indexer:build": "bun run --cwd apps/s03-indexer build", "indexer:dev": "bun run --cwd apps/s03-indexer dev", - "indexer:start": "bun run --cwd apps/s03-indexer start" + "indexer:start": "bun run --cwd apps/s03-indexer start", + "check:integration": "bash scripts/check-integration.sh" }, "devDependencies": { "@playwright/test": "^1.61.1", diff --git a/scripts/check-integration.sh b/scripts/check-integration.sh new file mode 100755 index 0000000..f0e57b7 --- /dev/null +++ b/scripts/check-integration.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# scripts/check-integration.sh +# +# Local pre-PR integration check. Mirrors the matrix in +# .github/workflows/integration-checks.yml so contributors can catch schema, +# mapping, and manifest-sync regressions before pushing. +# +# Usage: +# bash scripts/check-integration.sh +# bash scripts/check-integration.sh --skip-web +# bash scripts/check-integration.sh --skip-sync +# +# Requires: Bun, and a clean workspace install (`bun install`). + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +cd "$repo_root" + +skip_web=0 +skip_sync=0 + +for arg in "$@"; do + case "$arg" in + --skip-web) skip_web=1 ;; + --skip-sync) skip_sync=1 ;; + -h|--help) + sed -n '2,16p' "$0" + exit 0 + ;; + *) + echo "unknown option: $arg" >&2 + exit 2 + ;; + esac +done + +step() { + printf '\n\033[1;34m==> %s\033[0m\n' "$1" +} + +step "Indexer codegen (subql codegen)" +bun run --cwd apps/s03-indexer codegen + +step "Indexer build (subql build)" +bun run --cwd apps/s03-indexer build + +step "Indexer mapping unit tests" +bun run --cwd apps/s03-indexer test + +if [ "$skip_sync" -eq 0 ]; then + step "Manifest sync against offline fixtures" + SO4_CONTRACTS_REPO="$repo_root/apps/s03-indexer/tests/fixtures/contracts-repo" \ + bun run --cwd apps/s03-indexer sync:contracts:local + bash scripts/validate-manifest.sh apps/s03-indexer/config/contracts.local.json +fi + +if [ "$skip_web" -eq 0 ]; then + step "Web typecheck" + bun run --cwd apps/web typecheck + + step "Web production build" + bun run --cwd apps/web build +fi + +printf '\n\033[1;32mAll integration checks passed.\033[0m\n' diff --git a/scripts/validate-manifest.sh b/scripts/validate-manifest.sh new file mode 100755 index 0000000..1b2f224 --- /dev/null +++ b/scripts/validate-manifest.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# scripts/validate-manifest.sh .json> +# +# Validates that a synced indexer manifest contains every required field. Used +# by scripts/check-integration.sh and the CI integration job to catch sync +# regressions without needing a live Stellar deploy. + +set -euo pipefail + +manifest="${1:-}" +if [ -z "$manifest" ] || [ ! -f "$manifest" ]; then + echo "usage: $0 .json>" >&2 + exit 2 +fi + +required_contracts=( + role_store + data_store + oracle + market_factory + deposit_handler + withdrawal_handler + order_handler + liquidation_handler + adl_handler + fee_handler + referral_storage + reader + exchange_router +) + +required_tokens=(TUSDC TWBTC TETH TXLM faucet) + +missing=() + +has_field() { + # JSON field-presence probe using bun -e so we don't add a jq dependency. + bun -e " + const data = JSON.parse(require('fs').readFileSync(process.argv[1], 'utf8')); + const path = process.argv[2].split('.'); + let v = data; + for (const p of path) { + if (v == null || !(p in v)) process.exit(1); + v = v[p]; + } + if (v == null || v === '') process.exit(1); + " "$manifest" "$1" +} + +for f in network.name network.passphrase network.horizonEndpoint network.sorobanRpcEndpoint; do + has_field "$f" || missing+=("$f") +done + +for c in "${required_contracts[@]}"; do + has_field "contracts.$c" || missing+=("contracts.$c") +done + +for t in "${required_tokens[@]}"; do + has_field "tokens.$t" || missing+=("tokens.$t") +done + +if [ "${#missing[@]}" -gt 0 ]; then + echo "Manifest $manifest is missing required fields:" >&2 + for m in "${missing[@]}"; do echo " - $m" >&2; done + exit 1 +fi + +echo "Manifest $manifest passed validation."