diff --git a/CHANGELOG.md b/CHANGELOG.md index 204ffe4..cc4884b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,61 @@ Versions follow [Semantic Versioning](https://semver.org/) — minor bump per su --- +## [1.3.0] — 2026-07-13 + +### Added +- `LOOKUP` column qualifier — language grammar and storage layer (#58). Any column can + declare a constraint on its legal values: `LOOKUP . [DISPLAY ]` + for a live table-driven lookup, or `LOOKUP ("a","b",...)` for a literal list. Parsed by + `CREATE TABLE`/`ALTER TABLE ADD`/`ALTER TABLE ALTER`, persisted per-column in + `ColumnMetaStore` via an additive migration (existing declared types are never touched + or dropped), and resolvable to concrete `{value,label}` options via the new + `src/interpreter/LookupResolver.ts` (degrades to free entry — never truncates — when the + source is missing, empty, or exceeds 1000 distinct values). This is a WebBase-III + extension with no dBASE III ancestor. +- `LOOKUP` enforcement + BROWSE dropdown (#60). `REPLACE` and the BROWSE grid's `grid-edit` + now reject a value outside a column's declared `LOOKUP`, re-resolving the constraint fresh + against the live database on every write (so a value that only just became legal, or that + just stopped being legal, is judged correctly — never a stale cached list). An unresolvable + lookup (source table dropped, empty, or over 1000 values) degrades to free entry with a + warning rather than locking the column. BROWSE renders a lookup column as a dropdown — + `DISPLAY` labels shown while editing, the stored code shown once committed, matching + `LIST`/report output. +- **Field-bound `@ SAY GET`** (#59). `@ r,c SAY "…" GET ` now binds directly to the + active table's column when one matches — dBASE III's actual behavior — instead of only + ever collecting into a memory variable. Fields take precedence over a memory variable of + the same name (why the `m_` prefix convention exists). A field-bound `GET` needs a current + record and prefills from it; a lookup column renders the same picker BROWSE does. `READ`'s + submit validates every field-bound value (declared type + lookup membership) before + writing any of them — a rejection sends a new `form-error` message that keeps the form + open with the bad fields outlined, rather than silently discarding the valid ones. + Writes target the row captured at `GET` time, mirroring how `grid-edit` already writes by + rowid. This is the PR that promotes `LOOKUP` to the README command reference in full — + both BROWSE and forms now declare, enforce, and render it end to end. +- **Demos adopt `LOOKUP`** (#61). `demos/overtime.prg` gains a `SCHEDULES` catalog table + (`SCHEDID`, `DESCR`) as the lookup source for `EMPLOYEES.SCHEDID` — Add Employee is now a + check-first, two-form flow where the schedule is picked from a dropdown showing the + description ("Standard 40h (08:00-16:30)") instead of typed from memory. `demos/crm.prg`'s + `DEALS.STAGE` is constrained to a literal `LOOKUP` list matching its own seeded vocabulary + exactly, exercising the other lookup kind in a real, working demo. +- **Assistant wizard support** (#62). The New-table and Modify-structure wizards gain a + per-column "lookup (optional)" field accepting `TABLE.COLUMN [DISPLAY COLUMN]` or a quoted + list, so declaring a `LOOKUP` no longer requires dropping into raw W3Script syntax. This + closes out the v1.3.0 lookup-columns milestone (#58–#62): the language, storage, resolver, + BROWSE/REPLACE enforcement, field-bound forms, two demo apps, and the GUI wizards all now + agree on one constraint, declared once, on the column. + +### Fixed +- A memory-variable `@ SAY GET` no longer shows blank when the variable already holds a + value. Field-binding (#59) rewrote the non-field fallback path to hardcode an empty + prefill instead of reading the variable's current value, silently breaking every demo + form that pre-fills a default via `STORE`: `overtime.prg`'s Week Monday date (Open/Prep + Week, Recalculate Week) and leave date (Register Leave Taken), `INVENTORY.prg`'s stock/ + reorder/price/quantity defaults (Add Product, Stock In, Stock Out), and `crm.prg`'s deal + value default (Add Deal). Found by exercising the running app, not by the test suite — + every existing test for this path used a variable with no prior value, where an empty + prefill happens to be correct either way. + ## [1.2.0] — 2026-07-09 — TIME columns, WEEK()/DATEADD(), BROWSE cell validation, Overtime demo ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 841a324..1a2ca58 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,6 +63,9 @@ src/ Executor.ts Async AST runner; manages state (db/table/filter/vars/rowPtr/activeIndex). Emits fire-and-forget client side-effects (CSV download, report preview, CSV upload picker) via onSideEffect so they work inside program blocks IndexCommands.ts Index command handlers (extracted from Executor) ReportCommands.ts Report command handlers delegating to ReportRunner + LookupResolver.ts Resolves a column's LOOKUP constraint (literal list, or live table+column+DISPLAY) + to concrete {value,label} options against IDatabaseBridge; degrades to null + (never truncates) on a missing source, empty result, or >1000 distinct values terminal/ Terminal.ts REPL UI — command history, multi-line block accumulation @@ -194,6 +197,42 @@ WebBase-III supports **unlimited work areas** (no DOS 10-area limit). Cross-area Declared types are recorded per `(database, table, column)` in `server/ColumnMetaStore.ts`, because SQLite only keeps a storage affinity: `TIME`/`DATE`/`CHAR` are all `TEXT`, `LOGICAL`/`INT` are both `INTEGER`, and a `NUM(p,s)` qualifier is lost entirely. +#### `LOOKUP` column qualifier (v1.3.0) + +Any column may add a `LOOKUP` clause after its type, constraining it to a set of legal +values — a WebBase-III extension with no dBASE III ancestor (dBASE III+ only had +`PICTURE "@M a,b,c"`, a literal spacebar-cycled list with no table-driven form): + +``` +SCHEDID CHAR(4) LOOKUP SCHEDULES.SCHEDID DISPLAY DESCR -- live table lookup +STAGE CHAR(12) LOOKUP ("Lead","Won","Lost") -- literal list +``` + +`CREATE TABLE`/`ALTER TABLE ADD`/`ALTER TABLE ALTER` parse and store it (`ColumnMetaStore`, +same additive-migration discipline as the type columns above). `src/interpreter/LookupResolver.ts` +turns a stored `LOOKUP` into concrete `{value,label}` options against the live database, +degrading to `null` (never truncating) when the source is missing, empty, or exceeds 1000 +distinct values. + +BROWSE renders a lookup column as a dropdown (`Grid.ts`'s `startEdit`) fed by the options +`Session.sendGridData` resolves at `grid-open`; the stored value shows once committed, +matching `LIST`/report output — only the edit-mode dropdown shows `DISPLAY` labels. +`REPLACE` and `grid-edit` both enforce membership, re-resolving fresh at write time (not +trusting a client-held list) so a value that just became legal or just stopped being legal +is judged correctly; an unresolvable lookup degrades to free entry with a warning rather +than locking the column. + +`@ r,c SAY "…" GET ` binds to the active table's column when one matches — dBASE III's +actual behavior, and why the field takes precedence over a memory variable of the same name +(the `m_` prefix convention exists for this reason). A field-bound `GET` requires a current +record (`APPEND RECORD` first — `** Error: GET : no current record` otherwise), +prefills from the record, and renders a picker when the column has a resolvable lookup. +`READ`'s submit is all-or-nothing across every field-bound `GET` in the form: every value is +validated (declared type + lookup membership) before any is written, and a rejection sends a +`form-error` message that keeps the form open with the offending fields outlined — Escape +still writes nothing. Writes target the rowid captured at `GET` time (`fetchCurrentRow`), +mirroring `grid-edit`, so pointer motion between `GET` and submit can't retarget the write. + #### Cell validation (`BROWSE`) `src/shared/cellValidation.ts` holds the rules and runs on **both** sides: `Grid.ts` checks before commit (an invalid edit keeps the cell in edit mode, outlined red, with the reason shown; the error clears as the value becomes valid), and `Session`'s `grid-edit` handler re-checks authoritatively before writing — a WS message can reach the server without passing through the grid. @@ -207,7 +246,7 @@ Declared types are recorded per `(database, table, column)` in `server/ColumnMet | `LOGICAL` | `.T.`/`.F.`/`.TRUE.`/`.FALSE.`/`T`/`F`/`TRUE`/`FALSE`/`1`/`0` | | `CHAR` / `MEMO` | anything (length is not enforced) | -An empty value is always allowed (clears the cell). Columns with no recorded declared type are unconstrained. `REPLACE` enforces only `TIME` — widening it would change the semantics of existing programs. +An empty value is always allowed (clears the cell). Columns with no recorded declared type are unconstrained. `REPLACE` enforces `TIME` and, since v1.3.0, lookup membership on any column that declares one (additive — no pre-v1.3.0 column declares a `LOOKUP`, so no existing program's behavior changes); widening validation beyond that would change the semantics of existing programs. ### Indexing & search | Command | What it does | @@ -339,7 +378,7 @@ npm run coverage # Vitest + v8 coverage report (reporting only, no thresh npx playwright test # E2E browser tests — requires dev server on :5173/:3000 ``` -Playwright suites (94 tests): `tests/assistant.spec.ts` (23 tests — sidebar, wizards, report designer, MODIFY STRUCTURE round-trip, `TIME(15)` column + REPLACE validation, `NUM(p,s)` wizard, Browse-action grid validation, program run, CSV/SORT/SUM-AVERAGE/REINDEX/PACK actions, demo launchers), `tests/integration.spec.ts` (20 tests — full REPL scenario), `tests/overtime.spec.ts` (9 tests — overtime.prg: menu, seeding, DATEADD week prep, TIME(15) grid rejection, recalculation, live balance, quarter-hour leave check, report, CSV), `tests/inventory.spec.ts` (8 tests — INVENTORY.prg menu + valuation/low-stock report/sort/CSV/JOIN), `tests/crm.spec.ts` (6 tests — CRM demo menu, pipeline summary, sort, report, CSV, JOIN), `tests/parity-commands.spec.ts` (6 tests — `?`/`??`, built-in functions, `WEEK()`, `DATEADD()`, `SUM`/`AVERAGE`, `SORT ON … TO`), `tests/multiarea.spec.ts` (4 tests — multi-work-area, relations, alias.field), `tests/demos.spec.ts` (4 tests — demo program + report seeding), `tests/grid-validation.spec.ts` (3 tests — BROWSE per-cell validation: TIME(15), NUM(p,s)/DATE, Esc abandons), `tests/schema-errors.spec.ts` (3 tests — malformed CREATE TABLE errors, NUM(p,s) column count, bare INPUT stores its value), `tests/copycsv.spec.ts` (2 tests — COPY TO download + APPEND FROM upload), `tests/splash.spec.ts` (2 tests — version banner + demo discoverability), `tests/join.spec.ts` (1 test — JOIN materialization), `tests/propagation.spec.ts` (1 test — live multiuser refresh), `tests/program-side-effects.spec.ts` (1 test — CSV/report side-effects fire from inside a program block). +Playwright suites (95 tests): `tests/assistant.spec.ts` (23 tests — sidebar, wizards, report designer, MODIFY STRUCTURE round-trip, `TIME(15)` column + REPLACE validation, `NUM(p,s)` wizard, Browse-action grid validation, program run, CSV/SORT/SUM-AVERAGE/REINDEX/PACK actions, demo launchers), `tests/integration.spec.ts` (20 tests — full REPL scenario), `tests/overtime.spec.ts` (10 tests — overtime.prg: menu, seeding, DATEADD week prep, TIME(15) grid rejection, recalculation, live balance, quarter-hour leave check, report, CSV, Add-Employee lookup dropdown), `tests/inventory.spec.ts` (8 tests — INVENTORY.prg menu + valuation/low-stock report/sort/CSV/JOIN), `tests/crm.spec.ts` (6 tests — CRM demo menu, pipeline summary, sort, report, CSV, JOIN), `tests/parity-commands.spec.ts` (6 tests — `?`/`??`, built-in functions, `WEEK()`, `DATEADD()`, `SUM`/`AVERAGE`, `SORT ON … TO`), `tests/multiarea.spec.ts` (4 tests — multi-work-area, relations, alias.field), `tests/demos.spec.ts` (4 tests — demo program + report seeding), `tests/grid-validation.spec.ts` (3 tests — BROWSE per-cell validation: TIME(15), NUM(p,s)/DATE, Esc abandons), `tests/schema-errors.spec.ts` (3 tests — malformed CREATE TABLE errors, NUM(p,s) column count, bare INPUT stores its value), `tests/copycsv.spec.ts` (2 tests — COPY TO download + APPEND FROM upload), `tests/splash.spec.ts` (2 tests — version banner + demo discoverability), `tests/join.spec.ts` (1 test — JOIN materialization), `tests/propagation.spec.ts` (1 test — live multiuser refresh), `tests/program-side-effects.spec.ts` (1 test — CSV/report side-effects fire from inside a program block). ## Test discipline diff --git a/README.md b/README.md index bc951ad..1c88c1f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **dBASE III is back. In your browser. `USE customers` like it's 1984.** -![WebBase-III demo — USE, LIST, SEEK, BROWSE](docs/screenshots/demo.gif) +![WebBase-III demo — USE, LIST, SEEK, BROWSE, and a LOOKUP column dropdown](docs/screenshots/demo.gif) Remember the dot prompt? Before SQL won, before ORMs, before anyone said "full-stack" — there was dBASE III. You typed `USE customers`, then `LIST`, and your data was just *there*. WebBase-III brings that whole world back: the terminal, the language, `BROWSE`, `@ SAY GET` forms, `.prg` programs, indexes, reports — rebuilt from scratch as a modern web app with its own interpreter in TypeScript, backed by Node.js, WebSockets, and SQLite. @@ -98,6 +98,20 @@ as you fix it, and `Esc` abandons the edit. Here a `TIME(15)` column rejects `08 --- +### `LOOKUP` columns — pick a value instead of typing it + +A column declared `LOOKUP
. DISPLAY ` edits through a dropdown in +both BROWSE and forms — showing the friendlier label, storing the code. Here `SCHEDID` +is constrained to `SCHEDULES.SCHEDID`, displaying `DESCR`. + +![BROWSE editing a LOOKUP column via dropdown](docs/screenshots/screenshot-lookup-browse.png) + +A field-bound `@ SAY GET` (one whose name matches a table column) renders the same picker: + +![A form field bound to a LOOKUP column](docs/screenshots/screenshot-lookup-form.png) + +--- + ### Aggregate commands & dBASE III parity `SUM`, `AVERAGE`, `? ROUND(…)`, `? MAX(…)`, and `SORT ON … TO` — numeric aggregates and sorted copies, honouring the active filter. @@ -140,6 +154,8 @@ build indexes, search, reindex, pack the database, design and run reports, run p table structure. Every click generates a real W3Script command that echoes into the terminal — watch it to learn the language. Wizards (New table, Filter, Sort, Sum/Average, Modify structure, report designer, …) open in the main area and show a live preview of the command they will run. +New table and Modify structure both offer an optional per-column "lookup" field — `LOOKUP` +without typing raw syntax. --- @@ -244,6 +260,43 @@ WebBase-III supports **unlimited work areas** — each independently holding a t **Column types**: `CHAR(n)` (aliases `CHARACTER`/`VARCHAR`/`STRING`/`MEMO`), `NUM`/`NUM(p,s)` (`NUMERIC`/`FLOAT`/`DOUBLE`/`DECIMAL`), `INT`/`INTEGER`, `LOGICAL`/`BOOLEAN`, `DATE`, and `TIME`/`TIME(n)`. `TIME` stores `HH:MM` (24-hour); the optional `TIME(n)` qualifier (e.g. `TIME(15)`) requires minutes to be a multiple of `n`. `REPLACE ... WITH` rejects a malformed or off-granularity `TIME` value instead of silently coercing it, and `LIST STRUCTURE` prints the declared type (`NUM(8,2)`, `TIME(15)`) rather than SQLite's storage class. +### `LOOKUP` columns — constrain a column to legal values + +Any column may add a `LOOKUP` clause after its type, declaring the values it's allowed to hold: + +``` +CREATE TABLE EMPLOYEES (EMPID CHAR(4), NAME CHAR(30), + SCHEDID CHAR(4) LOOKUP SCHEDULES.SCHEDID DISPLAY DESCR) +CREATE TABLE DEALS (STAGE CHAR(12) LOOKUP ("Lead","Qualified","Proposal","Won","Lost")) +``` + +Two forms: `LOOKUP
. [DISPLAY ]` looks up live values from another +table (optionally showing a friendlier label while storing the code), or +`LOOKUP ("a","b",...)` is a fixed list. `CREATE TABLE`/`ALTER TABLE ADD`/`ALTER TABLE ALTER` +all accept it. + +BROWSE renders a lookup column as a dropdown — `DISPLAY` labels while editing, the stored +code once committed, matching `LIST`/report output. `REPLACE` and BROWSE edits both reject +a value outside the list, re-checked fresh against the live database on every write (a +value that just became legal, or just stopped being legal, is judged correctly — never a +stale cached list). A lookup that can't be resolved (source table dropped, empty, or over +1000 distinct values) degrades to free entry with a warning instead of locking the column. + +Forms pick it up too: `@ r,c SAY "…" GET ` binds to the active table's column when +one matches — a field takes precedence over a memory variable of the same name, which is +why programs use an `m_` prefix for scratch variables. A field-bound `GET` needs a current +record (`APPEND RECORD` first) and prefills from it; if the column has a lookup, the form +renders the same picker BROWSE does. `READ` validates every field-bound value before +writing any of them — a rejection keeps the form open with the bad field outlined instead +of silently dropping the others. + +> **Deviation from dBASE III:** `LOOKUP` has no dBASE III ancestor — dBASE III+ only offered +> `PICTURE "@M a,b,c"`, a literal value list cycled with the spacebar, with no table-driven +> form and no display label. This is a WebBase-III extension, in the same spirit as +> unlimited work areas and `alias.field` dot notation. Field-bound `GET` *is* authentic +> dBASE III behavior — WebBase-III's own memory-variable-only forms were the deviation, +> now corrected. + > **CSV format (`COPY TO` / `APPEND FROM`):** Unlike dBASE III's headerless, > positional `DELIMITED`/`SDF` formats, WebBase-III uses modern **header-based CSV** > (RFC-4180, mapped by column name). Export downloads through the browser and @@ -318,7 +371,7 @@ WebBase-III supports **unlimited work areas** — each independently holding a t | `? [, ...]` | Evaluate expression(s) and print the result (numbers right-justified; bare `?` prints a blank line; `??` also accepted) | | `STORE TO ` | Assign a variable | | `INPUT "prompt" TO ` | Collect keyboard input | -| `@ r,c SAY "text" GET ` | Define a form field | +| `@ r,c SAY "text" GET ` | Define a form field; a name matching a column of the active table binds that column (lookup columns render a picker) | | `READ` | Display the form and wait for submit | ### Control flow diff --git a/demos/crm.prg b/demos/crm.prg index 449560e..75c8b05 100644 --- a/demos/crm.prg +++ b/demos/crm.prg @@ -45,7 +45,7 @@ IF RECCOUNT() == 0 SELECT DEAL DROP TABLE DEALS - CREATE TABLE DEALS (DEALID CHAR(6), COMPID CHAR(5), TITLE CHAR(40), STAGE CHAR(12), VALUE NUM(12,2), CLOSEMONTH NUM(6)) + CREATE TABLE DEALS (DEALID CHAR(6), COMPID CHAR(5), TITLE CHAR(40), STAGE CHAR(12) LOOKUP ("Lead","Qualified","Proposal","Won","Lost"), VALUE NUM(12,2), CLOSEMONTH NUM(6)) INDEX ON DEALID TO BYDEAL INDEX ON COMPID TO DEALCOMP APPEND RECORD diff --git a/demos/overtime.prg b/demos/overtime.prg index 237146b..c0808db 100644 --- a/demos/overtime.prg +++ b/demos/overtime.prg @@ -31,6 +31,10 @@ SELECT SCH USE DATABASE OVERTIME USE SCHEDULEDAYS +SELECT SCD +USE DATABASE OVERTIME +USE SCHEDULES + SELECT TS USE DATABASE OVERTIME USE TIMESHEET @@ -52,8 +56,17 @@ USE LEAVETAKEN * wipe TIMESHEET and WEEKSUMMARY — on every run until someone used it. SELECT EMP IF RECCOUNT() == 0 + SELECT SCD + DROP TABLE SCHEDULES + CREATE TABLE SCHEDULES (SCHEDID CHAR(4), DESCR CHAR(30)) + APPEND RECORD + REPLACE SCHEDID WITH "S001", DESCR WITH "Standard 40h (08:00-16:30)" + APPEND RECORD + REPLACE SCHEDID WITH "S002", DESCR WITH "Short 31.25h (09:00-16:00)" + + SELECT EMP DROP TABLE EMPLOYEES - CREATE TABLE EMPLOYEES (EMPID CHAR(4), NAME CHAR(30), SCHEDID CHAR(4)) + CREATE TABLE EMPLOYEES (EMPID CHAR(4), NAME CHAR(30), SCHEDID CHAR(4) LOOKUP SCHEDULES.SCHEDID DISPLAY DESCR) INDEX ON EMPID TO BYEMP APPEND RECORD REPLACE EMPID WITH "E001", NAME WITH "Ada Lovelace", SCHEDID WITH "S001" @@ -139,21 +152,24 @@ DO WHILE running CASE UPPER(TRIM(choice)) == "1" CLEAR @ 2, 5 SAY "--- ADD EMPLOYEE ---" - STORE SPACE(4) TO m_emp - STORE SPACE(30) TO m_name - STORE SPACE(4) TO m_sch - @ 4, 5 SAY "Employee ID (4): " GET m_emp - @ 5, 5 SAY "Name (30): " GET m_name - @ 6, 5 SAY "Schedule ID (4): " GET m_sch - READ SELECT EMP SET INDEX TO BYEMP + STORE SPACE(4) TO m_emp + @ 4, 5 SAY "Employee ID (4): " GET m_emp + READ SEEK TRIM(m_emp) IF FOUND() @ 8, 5 SAY "Employee already exists: " + TRIM(m_emp) ELSE + * Create in natural order: writing the key under an active index + * would move the record out from under the form. + SET INDEX TO APPEND RECORD - REPLACE EMPID WITH TRIM(m_emp), NAME WITH TRIM(m_name), SCHEDID WITH TRIM(m_sch) + REPLACE EMPID WITH TRIM(m_emp) + @ 5, 5 SAY "Name (30): " GET NAME + @ 6, 5 SAY "Schedule : " GET SCHEDID + READ + SET INDEX TO BYEMP @ 8, 5 SAY "Employee added: " + TRIM(m_emp) ENDIF INPUT "Press Enter to continue" TO pause diff --git a/docs/screenshots/demo.gif b/docs/screenshots/demo.gif index 7f0dd96..2abc4cc 100644 Binary files a/docs/screenshots/demo.gif and b/docs/screenshots/demo.gif differ diff --git a/docs/screenshots/screenshot-assistant-wizard.png b/docs/screenshots/screenshot-assistant-wizard.png index 0edd42e..18f882d 100644 Binary files a/docs/screenshots/screenshot-assistant-wizard.png and b/docs/screenshots/screenshot-assistant-wizard.png differ diff --git a/docs/screenshots/screenshot-assistant.png b/docs/screenshots/screenshot-assistant.png index 52d8c26..eafb182 100644 Binary files a/docs/screenshots/screenshot-assistant.png and b/docs/screenshots/screenshot-assistant.png differ diff --git a/docs/screenshots/screenshot-csv.png b/docs/screenshots/screenshot-csv.png index 25fa95e..8811f0b 100644 Binary files a/docs/screenshots/screenshot-csv.png and b/docs/screenshots/screenshot-csv.png differ diff --git a/docs/screenshots/screenshot-index.png b/docs/screenshots/screenshot-index.png index 8884642..d61869a 100644 Binary files a/docs/screenshots/screenshot-index.png and b/docs/screenshots/screenshot-index.png differ diff --git a/docs/screenshots/screenshot-list.png b/docs/screenshots/screenshot-list.png index c635a5b..bd06029 100644 Binary files a/docs/screenshots/screenshot-list.png and b/docs/screenshots/screenshot-list.png differ diff --git a/docs/screenshots/screenshot-lookup-browse.png b/docs/screenshots/screenshot-lookup-browse.png new file mode 100644 index 0000000..fa8d10c Binary files /dev/null and b/docs/screenshots/screenshot-lookup-browse.png differ diff --git a/docs/screenshots/screenshot-lookup-form.png b/docs/screenshots/screenshot-lookup-form.png new file mode 100644 index 0000000..3237adc Binary files /dev/null and b/docs/screenshots/screenshot-lookup-form.png differ diff --git a/docs/screenshots/screenshot-modify-structure.png b/docs/screenshots/screenshot-modify-structure.png index bd10f65..b9957d4 100644 Binary files a/docs/screenshots/screenshot-modify-structure.png and b/docs/screenshots/screenshot-modify-structure.png differ diff --git a/docs/screenshots/screenshot-parity.png b/docs/screenshots/screenshot-parity.png index 4ade517..4cfcad6 100644 Binary files a/docs/screenshots/screenshot-parity.png and b/docs/screenshots/screenshot-parity.png differ diff --git a/docs/screenshots/screenshot-repl-session.png b/docs/screenshots/screenshot-repl-session.png index f8bb091..22f3fb0 100644 Binary files a/docs/screenshots/screenshot-repl-session.png and b/docs/screenshots/screenshot-repl-session.png differ diff --git a/docs/screenshots/screenshot-terminal.png b/docs/screenshots/screenshot-terminal.png index 124140a..60c8602 100644 Binary files a/docs/screenshots/screenshot-terminal.png and b/docs/screenshots/screenshot-terminal.png differ diff --git a/docs/superpowers/plans/2026-07-11-lookup-columns.md b/docs/superpowers/plans/2026-07-11-lookup-columns.md new file mode 100644 index 0000000..3ffc971 --- /dev/null +++ b/docs/superpowers/plans/2026-07-11-lookup-columns.md @@ -0,0 +1,2431 @@ +# Lookup Columns + Field-Bound GET (v1.3.0) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Declare a column's legal values once (`LOOKUP SCHEDULES.SCHEDID DISPLAY DESCR` or `LOOKUP ("Lead","Won")`), and have forms, the BROWSE grid, `REPLACE`, and `grid-edit` all offer a picker and enforce membership. + +**Architecture:** The lookup is stored per-column in `ColumnMetaStore` (additive `ADD COLUMN` migration — never drop-recreate, v1.2.0 shipped). A pure `resolveLookup()` turns a lookup into `{value,label}[]` options (or `null` → degrade to free text, never truncate). `@ SAY GET ` binds to the active table's column when one matches (fields shadow memvars — dBASE III's own rule), captures the record's rowid at GET time, and `form-submit` writes by rowid exactly like `grid-edit`, all-or-nothing, with a new `form-error` message on rejection. + +**Tech Stack:** TypeScript, better-sqlite3, Vitest, Playwright. No new dependencies. + +**Spec:** `docs/superpowers/specs/2026-07-10-lookup-columns-design.md` (approved). GitHub issues #58 (qualifier/storage/resolver), #59 (field-bound GET), #60 (grid/REPLACE enforcement), #61 (demos), #62 (Assistant wizards). + +--- + +## Ground rules for this repo (read before Task 1) + +- **Branch:** all work on `feature/lookup-columns`, branched off `release/v1.3.0`. One PR back into `release/v1.3.0` closing #58–#62. **Never** commit to `main`. + ```bash + git checkout release/v1.3.0 && git pull && git checkout -b feature/lookup-columns + ``` +- **NEVER add a `Co-Authored-By: Claude` (or any AI attribution) trailer to commits.** Repo rule, overrides all defaults. +- **Run suites serially, never concurrently:** `npm test` (vitest) and `npx playwright test` both mutate `data/`. Playwright needs the dev server (`npm run dev`) on :5173/:3000, or let `playwright.config.ts`'s `webServer` start it. +- **Test discipline:** assert exact structure with `toEqual` (never only `toContain`); anything that must be *readable* in the browser asserts `toBeInViewport()`; name-keyed state gets a two-database test. +- Run a single vitest file with: `npx vitest run tests/.test.ts` + +## File map (what changes where) + +| File | Change | +|---|---| +| `src/shared/cellValidation.ts` | `Lookup`, `LookupOption` types; `ColumnMeta.lookup/options`; membership check | +| `src/shared/types.ts` | re-export `Lookup`/`LookupOption`; `FormField.target/value/options`; `form-error` ServerMessage | +| `src/interpreter/Parser.ts` | `LOOKUP` clause on CREATE/ALTER; `ColDef.lookup`; ALTER AST carries lookup | +| `src/interpreter/LookupResolver.ts` | **new** — resolve a `Lookup` to options against an `IDatabaseBridge` | +| `server/ColumnMetaStore.ts` | 5 lookup columns, additive migration, persist/parse lookup | +| `src/interpreter/Executor.ts` | CREATE/ALTER record lookup; REPLACE membership; field-bound `doAtSayGet` | +| `server/Session.ts` | grid-open options; grid-edit membership; form-submit field writes + `form-error`; retained field list | +| `src/ui/FormLayout.ts` | `` cell editor for lookup columns | +| `src/styles/main.css` | select styling + invalid-form styling | +| `src/ui/wizards/TableWizard.ts`, `ModStructWizard.ts` | per-column Lookup input | +| `demos/overtime.prg` | `SCHEDULES` table, `LOOKUP … DISPLAY`, two-form Add Employee | +| `demos/crm.prg` | `STAGE … LOOKUP ("Lead","Qualified","Proposal","Won","Lost")` | +| Tests | extend `CellValidation`, `CreateTableParse`, `ColumnMetaStore`, `GridMessages`-style; new `LookupResolver.test.ts`, `LookupEnforcement.test.ts`, `FormFieldBinding.test.ts`, `lookup.spec.ts`; update `DemoSchemas.test.ts`, `overtime.spec.ts`, `assistant.spec.ts` | + +--- + +### Task 1: Shared lookup types + membership validation + +**Files:** +- Modify: `src/shared/cellValidation.ts` +- Modify: `src/shared/types.ts:54-58` +- Test: `tests/CellValidation.test.ts` + +- [ ] **Step 1.1: Write the failing tests** — append to `tests/CellValidation.test.ts`: + +```ts +describe('lookup membership', () => { + const meta = { + baseType: 'CHAR', qualifier: 12 as number | null, scale: null as number | null, + lookup: { kind: 'list' as const, values: ['Lead', 'Won', 'Lost'] }, + options: [ + { value: 'Lead', label: 'Lead' }, + { value: 'Won', label: 'Won' }, + { value: 'Lost', label: 'Lost' }, + ], + }; + + it('accepts a value that is in the resolved options', () => { + expect(validateCellValue('STAGE', 'Won', meta)).toBeNull(); + }); + + it('rejects a value that is not in the resolved options', () => { + expect(validateCellValue('STAGE', 'Maybe', meta)).toMatch(/not one of the allowed values/); + }); + + it('is case-sensitive: "won" is not "Won"', () => { + expect(validateCellValue('STAGE', 'won', meta)).toMatch(/not one of the allowed values/); + }); + + it('still allows clearing the cell', () => { + expect(validateCellValue('STAGE', '', meta)).toBeNull(); + }); + + it('skips membership when options are absent (unresolvable lookup degrades)', () => { + const degraded = { ...meta, options: undefined }; + expect(validateCellValue('STAGE', 'Anything', degraded)).toBeNull(); + }); + + it('runs the declared-type check before membership', () => { + const intMeta = { + baseType: 'INT', qualifier: null, scale: null, + lookup: { kind: 'list' as const, values: ['1', '2'] }, + options: [{ value: '1', label: '1' }, { value: '2', label: '2' }], + }; + expect(validateCellValue('N', 'abc', intMeta)).toMatch(/whole number/); + expect(validateCellValue('N', '3', intMeta)).toMatch(/not one of the allowed values/); + expect(validateCellValue('N', '2', intMeta)).toBeNull(); + }); +}); +``` + +- [ ] **Step 1.2: Run to verify failure** + +Run: `npx vitest run tests/CellValidation.test.ts` +Expected: FAIL — TS error (`lookup` not on `ColumnMeta`) or membership assertions fail. + +- [ ] **Step 1.3: Implement.** In `src/shared/cellValidation.ts`, replace the `ColumnMeta` interface (lines 9-13) with: + +```ts +/** A column's legal-values constraint — a WebBase-III extension, no dBASE III ancestor. */ +export type Lookup = + | { kind: 'list'; values: string[] } + | { kind: 'table'; table: string; column: string; display?: string }; + +export interface LookupOption { value: string; label: string } + +export interface ColumnMeta { + baseType: string; + qualifier: number | null; // CHAR(n) length, TIME(n) minute granularity, NUM(p,s) precision + scale: number | null; // NUM(p,s) scale + lookup?: Lookup | null; // declared constraint (may be unresolvable) + options?: LookupOption[]; // resolved values — absent when the lookup degraded +} +``` + +Then rename the existing `validateCellValue` body: change the `switch` so every `return null` inside a case falls through to a shared membership check. Concretely, replace the whole function with: + +```ts +export function validateCellValue( + colName: string, + value: string, + meta: ColumnMeta | null | undefined, +): string | null { + if (!meta) return null; // untracked column — no constraint + const v = value.trim(); + if (v === '') return null; // clearing a cell is always allowed + const typeErr = declaredTypeError(colName, v, meta); + if (typeErr) return typeErr; + // Membership runs only when the lookup resolved; an unresolvable lookup + // degrades to free text rather than locking the column. + if (meta.options && meta.options.length && !meta.options.some(o => o.value === v)) { + return `${colName}: "${v}" is not one of the allowed values`; + } + return null; +} + +function declaredTypeError(colName: string, v: string, meta: ColumnMeta): string | null { + switch (meta.baseType.toUpperCase()) { + /* …the existing switch body from the old validateCellValue, verbatim… */ + } +} +``` + +Move the old `switch (meta.baseType.toUpperCase()) { … }` block (old lines 34-88) into `declaredTypeError` unchanged. + +In `src/shared/types.ts`, directly under the `ColumnTypeInfo` line (58), add: + +```ts +export type { Lookup, LookupOption } from './cellValidation'; +``` + +- [ ] **Step 1.4: Run to verify pass** + +Run: `npx vitest run tests/CellValidation.test.ts` → PASS. Then `npm test` → all green (nothing else touches the shape yet). + +- [ ] **Step 1.5: Commit** + +```bash +git add src/shared/cellValidation.ts src/shared/types.ts tests/CellValidation.test.ts +git commit -m "feat(#58): Lookup types + membership check in shared cell validation" +``` + +--- + +### Task 2: Parser — the LOOKUP clause + +**Files:** +- Modify: `src/interpreter/Parser.ts` (`ColDef` line 79, ALTER AST lines 62-63, `parseCreate` line 467, `parseAlter` line 540) +- Test: `tests/CreateTableParse.test.ts` + +- [ ] **Step 2.1: Write the failing tests** — append to `tests/CreateTableParse.test.ts`: + +```ts +describe('LOOKUP column qualifier', () => { + it('parses a table lookup with DISPLAY', () => { + expect(cols('CREATE TABLE e (SCHEDID CHAR(4) LOOKUP SCHEDULES.SCHEDID DISPLAY DESCR)')).toEqual([ + { name: 'SCHEDID', colType: 'CHAR', size: 4, + lookup: { kind: 'table', table: 'SCHEDULES', column: 'SCHEDID', display: 'DESCR' } }, + ]); + }); + + it('parses a table lookup without DISPLAY', () => { + expect(cols('CREATE TABLE e (SCHEDID CHAR(4) LOOKUP SCHEDULES.SCHEDID)')).toEqual([ + { name: 'SCHEDID', colType: 'CHAR', size: 4, + lookup: { kind: 'table', table: 'SCHEDULES', column: 'SCHEDID' } }, + ]); + }); + + it('parses a literal list, preserving case', () => { + expect(cols('CREATE TABLE d (STAGE CHAR(12) LOOKUP ("Lead","Won","Lost"))')).toEqual([ + { name: 'STAGE', colType: 'CHAR', size: 12, + lookup: { kind: 'list', values: ['Lead', 'Won', 'Lost'] } }, + ]); + }); + + it('a LOOKUP column can be followed by more columns', () => { + expect(cols('CREATE TABLE d (STAGE CHAR(12) LOOKUP ("A","B"), VALUE NUM(8,2))').map((c: any) => c.name)) + .toEqual(['STAGE', 'VALUE']); + }); + + it('rejects an empty LOOKUP list', () => { + expect(() => parse('CREATE TABLE d (STAGE CHAR(12) LOOKUP ())')).toThrow(/LOOKUP/i); + }); + + it('rejects unquoted values in a LOOKUP list', () => { + expect(() => parse('CREATE TABLE d (STAGE CHAR(12) LOOKUP (Lead,Won))')).toThrow(/LOOKUP/i); + }); + + it('rejects LOOKUP with no source at all', () => { + expect(() => parse('CREATE TABLE d (STAGE CHAR(12) LOOKUP)')).toThrow(/LOOKUP/i); + }); + + it('rejects a table lookup missing the .column part', () => { + expect(() => parse('CREATE TABLE d (STAGE CHAR(12) LOOKUP SCHEDULES)')).toThrow(/LOOKUP/i); + }); + + it('carries LOOKUP through ALTER TABLE ADD', () => { + const ast = parse('ALTER TABLE t ADD STAGE CHAR(12) LOOKUP ("A","B")')[0] as any; + expect(ast.lookup).toEqual({ kind: 'list', values: ['A', 'B'] }); + }); + + it('carries LOOKUP through ALTER TABLE ALTER', () => { + const ast = parse('ALTER TABLE t ALTER STAGE CHAR LOOKUP S.C DISPLAY D')[0] as any; + expect(ast.lookup).toEqual({ kind: 'table', table: 'S', column: 'C', display: 'D' }); + }); +}); +``` + +- [ ] **Step 2.2: Run to verify failure** + +Run: `npx vitest run tests/CreateTableParse.test.ts` +Expected: FAIL — `lookup` is undefined on parsed cols / no throw for malformed forms. + +- [ ] **Step 2.3: Implement.** In `src/interpreter/Parser.ts`: + +At the top, add the import: +```ts +import type { Lookup } from '../shared/cellValidation'; +``` + +Change line 79 to: +```ts +export interface ColDef { name: string; colType: string; size?: number; scale?: number; lookup?: Lookup; } +``` + +Change the ALTER AST variants (lines 62-63) to: +```ts + | { type: 'ALTER_TABLE'; name: string; op: 'ADD'; col: string; colType: string; lookup: Lookup | null } + | { type: 'ALTER_TABLE'; name: string; op: 'ALTER'; col: string; colType: string; lookup: Lookup | null } +``` + +In `parseCreate` (line 467), replace the `cols.push(...)` line (491) and the lines just above it so the loop body reads: + +```ts + const cname = this.colName(); + const ctype = this.colType(cname); + let size: number | undefined; + let scale: number | undefined; + if (this.peek().type === 'LPAREN') { + this.adv(); + size = this.typeArg(cname); + if (this.peek().type === 'COMMA') { + this.adv(); + scale = this.typeArg(cname); + } + this.expectRParen(`type qualifier for column '${cname}'`); + } + let lookup: Lookup | undefined; + if (this.peekKw('LOOKUP')) lookup = this.parseLookupClause(cname); + cols.push(lookup !== undefined + ? { name: cname, colType: ctype, size, scale, lookup } + : { name: cname, colType: ctype, size, scale }); +``` + +(The conditional push keeps `toEqual` shapes in the pre-existing tests intact — no `lookup: undefined` key.) + +Below `expectRParen` (line 532), add: + +```ts + // LOOKUP
. [DISPLAY ] | LOOKUP ("a","b",...) + // A WebBase-III extension (documented deviation — dBASE III had no lookup). + private parseLookupClause(colName: string): Lookup { + this.adv(); // LOOKUP + if (this.peek().type === 'LPAREN') { + this.adv(); + const values: string[] = []; + while (!this.end() && this.peek().type !== 'RPAREN') { + if (this.peek().type !== 'STR') { + this.createErr(`expected a quoted string in the LOOKUP list for column '${colName}'`); + } + values.push(this.adv().val); + if (this.peek().type === 'COMMA') this.adv(); + else break; + } + this.expectRParen(`LOOKUP list for column '${colName}'`); + if (!values.length) this.createErr(`the LOOKUP list for column '${colName}' is empty`); + return { kind: 'list', values }; + } + const t = this.peek(); + if (t.type !== 'ID' && t.type !== 'KW') { + this.createErr(`expected
. or a ("…") list after LOOKUP for column '${colName}'`); + } + const table = this.adv().val; + if (this.peek().type !== 'DOT') { + this.createErr(`expected
. after LOOKUP for column '${colName}'`); + } + this.adv(); // DOT + const cTok = this.peek(); + if (cTok.type !== 'ID' && cTok.type !== 'KW') { + this.createErr(`expected a column after '${table}.' in the LOOKUP for column '${colName}'`); + } + const column = this.adv().val; + if (this.peekKw('DISPLAY')) { + this.adv(); + const dTok = this.peek(); + if (dTok.type !== 'ID' && dTok.type !== 'KW') { + this.createErr(`expected a display column after DISPLAY in the LOOKUP for column '${colName}'`); + } + return { kind: 'table', table, column, display: this.adv().val }; + } + return { kind: 'table', table, column }; + } +``` + +In `parseAlter` (line 540), change the ADD and ALTER lines to parse an optional clause: + +```ts + if (this.peekKw('ADD')) { this.adv(); this.skipKw('COLUMN'); const col = this.ident(); const colType = this.ident(); this.skipTypeSize(); const lookup = this.peekKw('LOOKUP') ? this.parseLookupClause(col) : null; return { type: 'ALTER_TABLE', name, op: 'ADD', col, colType, lookup }; } + if (this.peekKw('ALTER')) { this.adv(); this.skipKw('COLUMN'); const col = this.ident(); const colType = this.ident(); this.skipTypeSize(); const lookup = this.peekKw('LOOKUP') ? this.parseLookupClause(col) : null; return { type: 'ALTER_TABLE', name, op: 'ALTER', col, colType, lookup }; } +``` + +Note: `DISPLAY` is already a lexer keyword (`Lexer.ts:15`); `LOOKUP` lexes as `ID`, and `peekKw` matches both — **no lexer change**. + +- [ ] **Step 2.4: Run to verify pass** + +Run: `npx vitest run tests/CreateTableParse.test.ts` → PASS. Then `npm test` — `Executor.ts` will fail to compile until the ALTER node's new `lookup` property is consumed; if `npm test` reports type errors in `doAlterTable`, that is expected and fixed in Task 5. If vitest doesn't typecheck (it often doesn't), everything passes now. + +- [ ] **Step 2.5: Commit** + +```bash +git add src/interpreter/Parser.ts tests/CreateTableParse.test.ts +git commit -m "feat(#58): parse LOOKUP column qualifier in CREATE TABLE and ALTER TABLE" +``` + +--- + +### Task 3: ColumnMetaStore — persist lookups, additive migration + +**Files:** +- Modify: `server/ColumnMetaStore.ts` +- Modify: `src/shared/types.ts:61-68` (`IColumnMetaStore`) +- Test: `tests/ColumnMetaStore.test.ts` + +- [ ] **Step 3.1: Write the failing tests** — append to `tests/ColumnMetaStore.test.ts`: + +```ts +describe('lookup persistence', () => { + it('round-trips a table lookup with display', () => { + const store = new ColumnMetaStore(tmpDbPath()); + store.setColumnType('DB', 'EMPLOYEES', 'SCHEDID', 'CHAR', 4, null, + { kind: 'table', table: 'SCHEDULES', column: 'SCHEDID', display: 'DESCR' }); + expect(store.getColumnType('DB', 'EMPLOYEES', 'SCHEDID')).toEqual({ + baseType: 'CHAR', qualifier: 4, scale: null, + lookup: { kind: 'table', table: 'SCHEDULES', column: 'SCHEDID', display: 'DESCR' }, + }); + }); + + it('round-trips a literal list preserving order and case', () => { + const store = new ColumnMetaStore(tmpDbPath()); + store.setColumnType('DB', 'DEALS', 'STAGE', 'CHAR', 12, null, + { kind: 'list', values: ['Lead', 'Won', 'lost'] }); + expect(store.getColumnType('DB', 'DEALS', 'STAGE')?.lookup) + .toEqual({ kind: 'list', values: ['Lead', 'Won', 'lost'] }); + }); + + it('omits the lookup key entirely when none was declared', () => { + const store = new ColumnMetaStore(tmpDbPath()); + store.setColumnType('DB', 'T', 'PLAIN', 'CHAR', 10, null); + expect(store.getColumnType('DB', 'T', 'PLAIN')).toEqual({ baseType: 'CHAR', qualifier: 10, scale: null }); + }); + + it('re-declaring a column without a lookup clears the stored one', () => { + const store = new ColumnMetaStore(tmpDbPath()); + store.setColumnType('DB', 'T', 'C', 'CHAR', 4, null, { kind: 'list', values: ['A'] }); + store.setColumnType('DB', 'T', 'C', 'CHAR', 4, null); + expect(store.getColumnType('DB', 'T', 'C')?.lookup).toBeUndefined(); + }); + + it('scopes lookups per database (two DBs, same table+column)', () => { + const store = new ColumnMetaStore(tmpDbPath()); + store.setColumnType('A', 'T', 'C', 'CHAR', 4, null, { kind: 'list', values: ['X'] }); + store.setColumnType('B', 'T', 'C', 'CHAR', 4, null, { kind: 'list', values: ['Y'] }); + expect(store.getColumnType('A', 'T', 'C')?.lookup).toEqual({ kind: 'list', values: ['X'] }); + expect(store.getColumnType('B', 'T', 'C')?.lookup).toEqual({ kind: 'list', values: ['Y'] }); + }); + + it('listColumnTypes carries lookups', () => { + const store = new ColumnMetaStore(tmpDbPath()); + store.setColumnType('DB', 'T', 'C', 'CHAR', 4, null, { kind: 'list', values: ['A'] }); + expect(store.listColumnTypes('DB', 'T').C.lookup).toEqual({ kind: 'list', values: ['A'] }); + }); + + // A v1.2.0 store has db_name and scale but no lookup columns. It SHIPPED — + // it must be migrated additively, never dropped (dropping erases users' + // declared TIME(15)/NUM(p,s) types). + it('adds lookup columns to a v1.2.0 store without losing existing rows', () => { + const p = tmpDbPath(); + const v120 = new Database(p); + v120.exec(` + CREATE TABLE column_types ( + db_name TEXT NOT NULL, + table_name TEXT NOT NULL, + col_name TEXT NOT NULL, + base_type TEXT NOT NULL, + qualifier INTEGER, + scale INTEGER, + PRIMARY KEY (db_name, table_name, col_name) + ); + `); + v120.prepare('INSERT INTO column_types VALUES (?,?,?,?,?,?)') + .run('OVERTIME', 'SCHEDULEDAYS', 'TIMEIN', 'TIME', 15, null); + v120.close(); + + const store = new ColumnMetaStore(p); + // The pre-existing row SURVIVES: + expect(store.getColumnType('OVERTIME', 'SCHEDULEDAYS', 'TIMEIN')) + .toEqual({ baseType: 'TIME', qualifier: 15, scale: null }); + // And the new columns work: + store.setColumnType('OVERTIME', 'EMPLOYEES', 'SCHEDID', 'CHAR', 4, null, + { kind: 'table', table: 'SCHEDULES', column: 'SCHEDID' }); + expect(store.getColumnType('OVERTIME', 'EMPLOYEES', 'SCHEDID')?.lookup) + .toEqual({ kind: 'table', table: 'SCHEDULES', column: 'SCHEDID' }); + }); +}); +``` + +- [ ] **Step 3.2: Run to verify failure** + +Run: `npx vitest run tests/ColumnMetaStore.test.ts` +Expected: FAIL — `setColumnType` takes 6 args / no lookup columns. + +- [ ] **Step 3.3: Implement.** In `src/shared/types.ts`, change the `IColumnMetaStore.setColumnType` signature (line 62) to: + +```ts + setColumnType(dbName: string, tableName: string, colName: string, baseType: string, qualifier: number | null, scale: number | null, lookup?: Lookup | null): void; +``` + +and add `Lookup` to the type import at the top of the interface's file — it's the same file; the re-export from Task 1 makes `Lookup` available; add a direct import so the interface can reference it: + +```ts +import type { Lookup as _Lookup } from './cellValidation'; +``` + +(Or simply reference `import('./cellValidation').Lookup` inline — match the style used for `ColumnTypeInfo` on line 58:) + +```ts + setColumnType(dbName: string, tableName: string, colName: string, baseType: string, qualifier: number | null, scale: number | null, lookup?: import('./cellValidation').Lookup | null): void; +``` + +In `server/ColumnMetaStore.ts`, replace the whole file body with: + +```ts +import Database from 'better-sqlite3'; +import fs from 'fs'; +import path from 'path'; +import type { IColumnMetaStore, ColumnTypeInfo, Lookup } from '../src/shared/types.js'; + +const DATA_DIR = path.join(process.cwd(), 'data'); +const DB_PATH = path.join(DATA_DIR, 'system.sqlite3'); + +const LOOKUP_COLS = ['lookup_kind', 'lookup_table', 'lookup_col', 'lookup_display', 'lookup_values'] as const; + +interface Row { + baseType: string; qualifier: number | null; scale: number | null; + lookup_kind: string | null; lookup_table: string | null; lookup_col: string | null; + lookup_display: string | null; lookup_values: string | null; +} + +function rowToMeta(r: Row): ColumnTypeInfo { + const meta: ColumnTypeInfo = { baseType: r.baseType, qualifier: r.qualifier, scale: r.scale }; + if (r.lookup_kind === 'list') { + meta.lookup = { kind: 'list', values: JSON.parse(r.lookup_values ?? '[]') as string[] }; + } else if (r.lookup_kind === 'table' && r.lookup_table && r.lookup_col) { + meta.lookup = r.lookup_display + ? { kind: 'table', table: r.lookup_table, column: r.lookup_col, display: r.lookup_display } + : { kind: 'table', table: r.lookup_table, column: r.lookup_col }; + } + return meta; +} + +const SELECT_COLS = `base_type AS baseType, qualifier, scale, + lookup_kind, lookup_table, lookup_col, lookup_display, lookup_values`; + +/** + * Declared column types (+ optional lookup constraint), keyed by + * (database, table, column). + * + * SQLite only records a storage affinity (TEXT/REAL/INTEGER), which cannot tell + * TIME from DATE from CHAR, LOGICAL from INT, or recover a NUM(p,s) qualifier. + * The grid, forms and REPLACE need the declared type + lookup to validate writes. + * + * Scoping by database matters: two databases may each hold a table of the same + * name with different column types. + */ +export class ColumnMetaStore implements IColumnMetaStore { + private db: Database.Database; + + constructor(dbPath = DB_PATH) { + fs.mkdirSync(path.dirname(dbPath), { recursive: true }); + this.db = new Database(dbPath); + this.db.pragma('journal_mode = WAL'); + this.db.exec(` + CREATE TABLE IF NOT EXISTS column_types ( + db_name TEXT NOT NULL, + table_name TEXT NOT NULL, + col_name TEXT NOT NULL, + base_type TEXT NOT NULL, + qualifier INTEGER, + scale INTEGER, + lookup_kind TEXT, + lookup_table TEXT, + lookup_col TEXT, + lookup_display TEXT, + lookup_values TEXT, + PRIMARY KEY (db_name, table_name, col_name) + ); + `); + // v1.2.0 dev migration: the first cut of this table (#43) had neither db_name + // nor scale. Those rows never shipped in a release, so rebuilding was safe. + let cols = this.db.prepare('PRAGMA table_info(column_types)').all() as { name: string }[]; + if (!cols.some(c => c.name === 'db_name') || !cols.some(c => c.name === 'scale')) { + this.db.exec(` + DROP TABLE column_types; + CREATE TABLE column_types ( + db_name TEXT NOT NULL, + table_name TEXT NOT NULL, + col_name TEXT NOT NULL, + base_type TEXT NOT NULL, + qualifier INTEGER, + scale INTEGER, + lookup_kind TEXT, + lookup_table TEXT, + lookup_col TEXT, + lookup_display TEXT, + lookup_values TEXT, + PRIMARY KEY (db_name, table_name, col_name) + ); + `); + cols = this.db.prepare('PRAGMA table_info(column_types)').all() as { name: string }[]; + } + // v1.3.0 lookup migration (#58): v1.2.0 SHIPPED, so this one must be + // additive — dropping the table here would silently erase every released + // user's declared TIME(15)/NUM(p,s) types. + for (const col of LOOKUP_COLS) { + if (!cols.some(c => c.name === col)) { + this.db.exec(`ALTER TABLE column_types ADD COLUMN ${col} TEXT`); + } + } + } + + setColumnType(dbName: string, tableName: string, colName: string, baseType: string, qualifier: number | null, scale: number | null, lookup: Lookup | null = null): void { + const kind = lookup?.kind ?? null; + const lTable = lookup?.kind === 'table' ? lookup.table : null; + const lCol = lookup?.kind === 'table' ? lookup.column : null; + const lDisplay = lookup?.kind === 'table' ? (lookup.display ?? null) : null; + const lValues = lookup?.kind === 'list' ? JSON.stringify(lookup.values) : null; + this.db.prepare(` + INSERT INTO column_types (db_name, table_name, col_name, base_type, qualifier, scale, + lookup_kind, lookup_table, lookup_col, lookup_display, lookup_values) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(db_name, table_name, col_name) DO UPDATE SET + base_type = excluded.base_type, qualifier = excluded.qualifier, scale = excluded.scale, + lookup_kind = excluded.lookup_kind, lookup_table = excluded.lookup_table, + lookup_col = excluded.lookup_col, lookup_display = excluded.lookup_display, + lookup_values = excluded.lookup_values + `).run(dbName, tableName, colName, baseType, qualifier, scale, kind, lTable, lCol, lDisplay, lValues); + } + + getColumnType(dbName: string, tableName: string, colName: string): ColumnTypeInfo | null { + const row = this.db.prepare( + `SELECT ${SELECT_COLS} FROM column_types WHERE db_name = ? AND table_name = ? AND col_name = ?` + ).get(dbName, tableName, colName) as Row | undefined; + return row ? rowToMeta(row) : null; + } + + listColumnTypes(dbName: string, tableName: string): Record { + const rows = this.db.prepare( + `SELECT col_name AS colName, ${SELECT_COLS} FROM column_types WHERE db_name = ? AND table_name = ?` + ).all(dbName, tableName) as Array; + const out: Record = {}; + for (const r of rows) out[r.colName] = rowToMeta(r); + return out; + } + + renameColumn(dbName: string, tableName: string, oldName: string, newName: string): void { + this.db.prepare( + 'UPDATE column_types SET col_name = ? WHERE db_name = ? AND table_name = ? AND col_name = ?' + ).run(newName, dbName, tableName, oldName); + } + + dropColumn(dbName: string, tableName: string, colName: string): void { + this.db.prepare('DELETE FROM column_types WHERE db_name = ? AND table_name = ? AND col_name = ?') + .run(dbName, tableName, colName); + } + + dropTable(dbName: string, tableName: string): void { + this.db.prepare('DELETE FROM column_types WHERE db_name = ? AND table_name = ?').run(dbName, tableName); + } +} + +export const columnMetaStore = new ColumnMetaStore(); +``` + +- [ ] **Step 3.4: Run to verify pass** + +Run: `npx vitest run tests/ColumnMetaStore.test.ts` → PASS (including the pre-existing tests, whose `toEqual({baseType, qualifier, scale})` assertions survive because `rowToMeta` omits `lookup` when null). Run `npx vitest run tests/ColumnMeta.test.ts` too — it exercises the store through Session. + +- [ ] **Step 3.5: Commit** + +```bash +git add server/ColumnMetaStore.ts src/shared/types.ts tests/ColumnMetaStore.test.ts +git commit -m "feat(#58): persist lookups in ColumnMetaStore via additive ADD COLUMN migration" +``` + +--- + +### Task 4: LookupResolver + +**Files:** +- Create: `src/interpreter/LookupResolver.ts` +- Test: `tests/LookupResolver.test.ts` + +Note: the spec names `server/LookupResolver.ts`, but the Executor (in `src/interpreter/`) must call it for REPLACE enforcement and GET options, and `src/` must not import from `server/`. It lives in `src/interpreter/` — same behavior, dependency-clean. Record this deviation in the PR description. + +- [ ] **Step 4.1: Write the failing tests** — create `tests/LookupResolver.test.ts`: + +```ts +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { ServerDatabaseBridge, __closeAndEvictForTest } from '../server/ServerDatabaseBridge'; +import { resolveLookup, LOOKUP_MAX_VALUES } from '../src/interpreter/LookupResolver'; +import fs from 'fs'; +import path from 'path'; + +const TEST_DB = 'test_lookupresolver'; +const DATA_DIR = path.join(process.cwd(), 'data'); +const DB_PATH = path.join(DATA_DIR, `${TEST_DB}.sqlite3`); + +describe('resolveLookup', () => { + let bridge: ServerDatabaseBridge; + + beforeEach(async () => { + bridge = new ServerDatabaseBridge(); + await bridge.openDatabase(TEST_DB); + }); + + afterEach(async () => { + await bridge.closeDatabase(); + __closeAndEvictForTest(TEST_DB); + for (const f of [DB_PATH, DB_PATH + '-shm', DB_PATH + '-wal']) { + if (fs.existsSync(f)) fs.unlinkSync(f); + } + }); + + it('resolves a literal list verbatim, value === label', async () => { + expect(await resolveLookup(bridge, { kind: 'list', values: ['Lead', 'Won'] })).toEqual([ + { value: 'Lead', label: 'Lead' }, + { value: 'Won', label: 'Won' }, + ]); + }); + + it('resolves a table lookup to DISTINCT ordered values', async () => { + await bridge.exec('CREATE TABLE SCHEDULES (SCHEDID TEXT, DESCR TEXT)'); + await bridge.exec("INSERT INTO SCHEDULES VALUES ('S002','Short day'),('S001','Std day'),('S001','Std day')"); + expect(await resolveLookup(bridge, { kind: 'table', table: 'SCHEDULES', column: 'SCHEDID' })).toEqual([ + { value: 'S001', label: 'S001' }, + { value: 'S002', label: 'S002' }, + ]); + }); + + it('uses the DISPLAY column as the label, storing the value', async () => { + await bridge.exec('CREATE TABLE SCHEDULES (SCHEDID TEXT, DESCR TEXT)'); + await bridge.exec("INSERT INTO SCHEDULES VALUES ('S001','Std day'),('S002','Short day')"); + expect(await resolveLookup(bridge, { kind: 'table', table: 'SCHEDULES', column: 'SCHEDID', display: 'DESCR' })).toEqual([ + { value: 'S001', label: 'Std day' }, + { value: 'S002', label: 'Short day' }, + ]); + }); + + it('returns null when the source table is missing', async () => { + expect(await resolveLookup(bridge, { kind: 'table', table: 'NOPE', column: 'X' })).toBeNull(); + }); + + it('returns null when the source column is missing', async () => { + await bridge.exec('CREATE TABLE SCHEDULES (SCHEDID TEXT)'); + expect(await resolveLookup(bridge, { kind: 'table', table: 'SCHEDULES', column: 'MISSING' })).toBeNull(); + }); + + it('returns null when the display column is missing', async () => { + await bridge.exec('CREATE TABLE SCHEDULES (SCHEDID TEXT)'); + expect(await resolveLookup(bridge, { kind: 'table', table: 'SCHEDULES', column: 'SCHEDID', display: 'MISSING' })).toBeNull(); + }); + + it('returns null when the source is empty', async () => { + await bridge.exec('CREATE TABLE SCHEDULES (SCHEDID TEXT)'); + expect(await resolveLookup(bridge, { kind: 'table', table: 'SCHEDULES', column: 'SCHEDID' })).toBeNull(); + }); + + it('degrades (null), never truncates, over the ceiling', async () => { + await bridge.exec('CREATE TABLE BIG (V TEXT)'); + const values = Array.from({ length: LOOKUP_MAX_VALUES + 1 }, (_, i) => `('v${String(i).padStart(5, '0')}')`); + await bridge.exec(`INSERT INTO BIG VALUES ${values.join(',')}`); + expect(await resolveLookup(bridge, { kind: 'table', table: 'BIG', column: 'V' })).toBeNull(); + }); + + it('skips NULL/empty values from the source', async () => { + await bridge.exec('CREATE TABLE SCHEDULES (SCHEDID TEXT)'); + await bridge.exec("INSERT INTO SCHEDULES VALUES ('S001'),(NULL),('')"); + expect(await resolveLookup(bridge, { kind: 'table', table: 'SCHEDULES', column: 'SCHEDID' })).toEqual([ + { value: 'S001', label: 'S001' }, + ]); + }); +}); +``` + +- [ ] **Step 4.2: Run to verify failure** + +Run: `npx vitest run tests/LookupResolver.test.ts` +Expected: FAIL — module not found. + +- [ ] **Step 4.3: Implement.** Create `src/interpreter/LookupResolver.ts`: + +```ts +import type { IDatabaseBridge } from '../shared/types'; +import type { Lookup, LookupOption } from '../shared/cellValidation'; + +export const LOOKUP_MAX_VALUES = 1000; + +function q(name: string): string { + return `"${name.replace(/"/g, '""')}"`; +} + +/** + * Resolve a lookup to concrete {value,label} options, or null when it cannot + * be honoured (missing table/column, empty source, or over the ceiling). + * Callers degrade to free text and skip membership enforcement on null — + * an unresolvable lookup is a warning, not a lock. Never truncates: a clipped + * option list would hide legal values while membership validation rejected them. + */ +export async function resolveLookup(db: IDatabaseBridge, lookup: Lookup): Promise { + if (lookup.kind === 'list') { + return lookup.values.map(v => ({ value: v, label: v })); + } + if (!(await db.tableExists(lookup.table))) return null; + const cols = await db.getStructure(lookup.table); + const valueCol = cols.find(c => c.name.toUpperCase() === lookup.column.toUpperCase()); + if (!valueCol) return null; + let displayCol; + if (lookup.display) { + displayCol = cols.find(c => c.name.toUpperCase() === lookup.display!.toUpperCase()); + if (!displayCol) return null; + } + const sel = displayCol ? `${q(valueCol.name)}, ${q(displayCol.name)}` : q(valueCol.name); + const rows = await db.query( + `SELECT DISTINCT ${sel} FROM ${q(lookup.table)} ORDER BY 1 LIMIT ${LOOKUP_MAX_VALUES + 1}` + ); + const options: LookupOption[] = []; + for (const r of rows) { + const value = String(r[valueCol.name] ?? ''); + if (value === '') continue; // NULL/empty is not a pickable value + const label = displayCol ? String(r[displayCol.name] ?? value) : value; + options.push({ value, label }); + } + if (options.length === 0 || options.length > LOOKUP_MAX_VALUES) return null; + return options; +} +``` + +- [ ] **Step 4.4: Run to verify pass** + +Run: `npx vitest run tests/LookupResolver.test.ts` → PASS. + +- [ ] **Step 4.5: Commit** + +```bash +git add src/interpreter/LookupResolver.ts tests/LookupResolver.test.ts +git commit -m "feat(#58): LookupResolver — options or degrade, never truncate" +``` + +--- + +### Task 5: Executor — record lookups on CREATE/ALTER, enforce on REPLACE + +**Files:** +- Modify: `src/interpreter/Executor.ts` (`doCreateTable` line 762, `doAlterTable` lines 916-923 and 950-976, `doReplaceAll` line 427) +- Test: `tests/LookupEnforcement.test.ts` (new) + +- [ ] **Step 5.1: Write the failing tests** — create `tests/LookupEnforcement.test.ts` (Session-harness style, same as `tests/GridMessages.test.ts`): + +```ts +import { describe, it, expect, afterEach } from 'vitest'; +import { Session } from '../server/Session'; +import type { ServerMessage } from '../src/shared/types'; +import fs from 'fs'; +import path from 'path'; + +let dbCounter = 0; +function uniqueDb() { return `test_lookupenf_${Date.now()}_${++dbCounter}`; } + +afterEach(() => { + const dataDir = path.join(process.cwd(), 'data'); + if (fs.existsSync(dataDir)) { + fs.readdirSync(dataDir) + .filter(f => f.toLowerCase().startsWith('test_lookupenf_')) + .forEach(f => fs.unlinkSync(path.join(dataDir, f))); + } +}); + +async function setup() { + const sent: ServerMessage[] = []; + const session = new Session((m) => sent.push(m)); + const run = async (text: string) => { + sent.length = 0; + await session.handleMessage({ type: 'command', text }); + const out = sent.filter(m => m.type === 'output') as any[]; + return out.flatMap(o => o.lines).map((l: any) => l.text).join('\n'); + }; + await run(`USE DATABASE ${uniqueDb()}`); + return { session, sent, run }; +} + +describe('REPLACE enforces lookup membership', () => { + it('rejects an off-list value on a literal-lookup column and does not write it', async () => { + const { run } = await setup(); + await run('CREATE TABLE DEALS (TITLE CHAR(20), STAGE CHAR(12) LOOKUP ("Lead","Won","Lost"))'); + await run('USE DEALS'); + await run('APPEND RECORD'); + const out = await run('REPLACE STAGE WITH "Maybe"'); + expect(out).toMatch(/not one of the allowed values/); + expect(await run('LIST')).not.toContain('Maybe'); + }); + + it('accepts an on-list value (exact case)', async () => { + const { run } = await setup(); + await run('CREATE TABLE DEALS (TITLE CHAR(20), STAGE CHAR(12) LOOKUP ("Lead","Won","Lost"))'); + await run('USE DEALS'); + await run('APPEND RECORD'); + expect(await run('REPLACE STAGE WITH "Won"')).toContain('Replaced'); + expect(await run('LIST')).toContain('Won'); + }); + + it('rejects the wrong case', async () => { + const { run } = await setup(); + await run('CREATE TABLE DEALS (STAGE CHAR(12) LOOKUP ("Won"))'); + await run('USE DEALS'); + await run('APPEND RECORD'); + expect(await run('REPLACE STAGE WITH "won"')).toMatch(/not one of the allowed values/); + }); + + it('enforces a table lookup against live source rows', async () => { + const { run } = await setup(); + await run('CREATE TABLE SCHEDULES (SCHEDID CHAR(4), DESCR CHAR(30))'); + await run('USE SCHEDULES'); + await run('APPEND RECORD'); + await run('REPLACE SCHEDID WITH "S001", DESCR WITH "Std day"'); + await run('CREATE TABLE EMPLOYEES (EMPID CHAR(4), SCHEDID CHAR(4) LOOKUP SCHEDULES.SCHEDID DISPLAY DESCR)'); + await run('USE EMPLOYEES'); + await run('APPEND RECORD'); + expect(await run('REPLACE SCHEDID WITH "S999"')).toMatch(/not one of the allowed values/); + expect(await run('REPLACE SCHEDID WITH "S001"')).toContain('Replaced'); + }); + + it('a new source row becomes legal immediately (fresh re-resolve at write time)', async () => { + const { run } = await setup(); + await run('CREATE TABLE SCHEDULES (SCHEDID CHAR(4))'); + await run('USE SCHEDULES'); + await run('APPEND RECORD'); + await run('REPLACE SCHEDID WITH "S001"'); + await run('CREATE TABLE EMPLOYEES (SCHEDID CHAR(4) LOOKUP SCHEDULES.SCHEDID)'); + await run('USE EMPLOYEES'); + await run('APPEND RECORD'); + expect(await run('REPLACE SCHEDID WITH "S002"')).toMatch(/not one of the allowed values/); + await run('USE SCHEDULES'); + await run('APPEND RECORD'); + await run('REPLACE SCHEDID WITH "S002"'); + await run('USE EMPLOYEES'); + expect(await run('REPLACE SCHEDID WITH "S002"')).toContain('Replaced'); + }); + + it('an unresolvable lookup degrades: the write is allowed', async () => { + const { run } = await setup(); + await run('CREATE TABLE EMPLOYEES (SCHEDID CHAR(4) LOOKUP GHOSTTABLE.SCHEDID)'); + await run('USE EMPLOYEES'); + await run('APPEND RECORD'); + expect(await run('REPLACE SCHEDID WITH "ANY"')).toContain('Replaced'); + }); + + it('ALTER TABLE ADD carries the lookup', async () => { + const { run } = await setup(); + await run('CREATE TABLE T (A CHAR(4))'); + await run('USE T'); + await run('ALTER TABLE T ADD STAGE CHAR LOOKUP ("X","Y")'); + await run('APPEND RECORD'); + expect(await run('REPLACE STAGE WITH "Z"')).toMatch(/not one of the allowed values/); + expect(await run('REPLACE STAGE WITH "X"')).toContain('Replaced'); + }); + + it('two databases keep independent lookups for the same table+column', async () => { + const sent: ServerMessage[] = []; + const session = new Session((m) => sent.push(m)); + const run = async (text: string) => { + sent.length = 0; + await session.handleMessage({ type: 'command', text }); + const out = sent.filter(m => m.type === 'output') as any[]; + return out.flatMap(o => o.lines).map((l: any) => l.text).join('\n'); + }; + const dbA = uniqueDb(); const dbB = uniqueDb(); + await run(`USE DATABASE ${dbA}`); + await run('CREATE TABLE T (C CHAR(4) LOOKUP ("A"))'); + await run(`USE DATABASE ${dbB}`); + await run('CREATE TABLE T (C CHAR(4) LOOKUP ("B"))'); + await run('USE T'); + await run('APPEND RECORD'); + expect(await run('REPLACE C WITH "A"')).toMatch(/not one of the allowed values/); + expect(await run('REPLACE C WITH "B"')).toContain('Replaced'); + }); +}); +``` + +- [ ] **Step 5.2: Run to verify failure** + +Run: `npx vitest run tests/LookupEnforcement.test.ts` +Expected: FAIL — off-list REPLACE succeeds (`Replaced` where a rejection is expected). + +- [ ] **Step 5.3: Implement.** In `src/interpreter/Executor.ts`: + +Add the import (top of file, next to the `cellValidation` import): +```ts +import { resolveLookup } from './LookupResolver'; +``` + +In `doCreateTable` (line 782-786), pass the lookup through: +```ts + for (const c of cols) { + this.columnMetaStore?.setColumnType( + this.metaDb, name, c.name, c.colType.toUpperCase(), c.size ?? null, c.scale ?? null, c.lookup ?? null, + ); + } +``` + +In `doAlterTable`, the ADD branch (line 920) and ALTER branch (line 971) each change their `setColumnType` call: +```ts + this.columnMetaStore?.setColumnType(this.metaDb, name, node.col, node.colType.toUpperCase(), null, null, node.lookup); +``` +(both places — the node now carries `lookup: Lookup | null` from Task 2). + +In `doReplaceAll` (lines 431-441), replace the TIME-only validation loop with: + +```ts + // Declared-type enforcement on REPLACE is deliberately narrow: TIME since + // #43, plus lookup membership (#58) — membership is additive, because no + // pre-#58 column declares a lookup, so no existing program changes behavior. + // The grid still validates every declared type (#45). + for (const p of pairs) { + if (p.value === null || p.value === undefined) continue; + const info = this.columnMetaStore?.getColumnType(this.metaDb, this.area.table!, p.field); + if (!info) continue; + if (info.baseType === 'TIME') { + const err = validateCellValue(p.field, String(p.value), info); + if (err) throw new Error(err); + } + if (info.lookup) { + // Re-resolve at write time: a value that became legal after any cached + // list was built is accepted; a vanished one is rejected. Unresolvable + // (null) skips membership — degradation, not a lock. + const options = await resolveLookup(this.db, info.lookup); + if (options) { + const err = validateCellValue(p.field, String(p.value), { ...info, options }); + if (err) throw new Error(err); + } + } + } +``` + +- [ ] **Step 5.4: Run to verify pass** + +Run: `npx vitest run tests/LookupEnforcement.test.ts` → PASS. Then `npm test` → all green (fix any TypeScript fallout from the ALTER node change — the compiler will point at every site). + +- [ ] **Step 5.5: Commit** + +```bash +git add src/interpreter/Executor.ts tests/LookupEnforcement.test.ts +git commit -m "feat(#58,#60): record lookups on CREATE/ALTER; REPLACE enforces membership" +``` + +--- + +### Task 6: Session — grid-open ships options; grid-edit enforces membership + +**Files:** +- Modify: `server/Session.ts` (`sendGridData` line 359, `grid-edit` case line 74) +- Test: `tests/LookupEnforcement.test.ts` (extend) + +- [ ] **Step 6.1: Write the failing tests** — append to `tests/LookupEnforcement.test.ts`: + +```ts +describe('grid messages with lookups', () => { + it('grid-open ships resolved options for a lookup column — exact list', async () => { + const { session, sent, run } = await setup(); + await run('CREATE TABLE DEALS (STAGE CHAR(12) LOOKUP ("Lead","Won","Lost"))'); + await run('USE DEALS'); + sent.length = 0; + await session.handleMessage({ type: 'command', text: 'BROWSE' }); + const grid = sent.find(m => m.type === 'grid-open') as any; + expect(grid.columnTypes.STAGE.options).toEqual([ + { value: 'Lead', label: 'Lead' }, + { value: 'Won', label: 'Won' }, + { value: 'Lost', label: 'Lost' }, + ]); + }); + + it('grid-open degrades an unresolvable lookup: no options, a warning line', async () => { + const { session, sent, run } = await setup(); + await run('CREATE TABLE E (SCHEDID CHAR(4) LOOKUP GHOST.SCHEDID)'); + await run('USE E'); + sent.length = 0; + await session.handleMessage({ type: 'command', text: 'BROWSE' }); + const grid = sent.find(m => m.type === 'grid-open') as any; + expect(grid.columnTypes.SCHEDID.options).toBeUndefined(); + const out = sent.filter(m => m.type === 'output') as any[]; + expect(out.flatMap(o => o.lines).map((l: any) => l.text).join('\n')).toMatch(/lookup for SCHEDID/i); + }); + + it('grid-edit rejects an off-list value server-side (forged message path)', async () => { + const { session, sent, run } = await setup(); + await run('CREATE TABLE DEALS (STAGE CHAR(12) LOOKUP ("Lead","Won"))'); + await run('USE DEALS'); + await run('APPEND RECORD'); + sent.length = 0; + await session.handleMessage({ type: 'command', text: 'BROWSE' }); + const grid = sent.find(m => m.type === 'grid-open') as any; + + sent.length = 0; + await session.handleMessage({ type: 'grid-edit', rowid: grid.rows[0]._rowid, col: 'STAGE', value: 'Hacked' }); + const out = sent.find(m => m.type === 'output') as any; + expect(out.lines.map((l: any) => l.text).join('\n')).toMatch(/not one of the allowed values/); + expect(await run('LIST')).not.toContain('Hacked'); + }); + + it('grid-edit accepts an on-list value', async () => { + const { session, sent, run } = await setup(); + await run('CREATE TABLE DEALS (STAGE CHAR(12) LOOKUP ("Lead","Won"))'); + await run('USE DEALS'); + await run('APPEND RECORD'); + sent.length = 0; + await session.handleMessage({ type: 'command', text: 'BROWSE' }); + const grid = sent.find(m => m.type === 'grid-open') as any; + await session.handleMessage({ type: 'grid-edit', rowid: grid.rows[0]._rowid, col: 'STAGE', value: 'Won' }); + expect(await run('LIST')).toContain('Won'); + }); +}); +``` + +- [ ] **Step 6.2: Run to verify failure** + +Run: `npx vitest run tests/LookupEnforcement.test.ts` +Expected: FAIL — `options` undefined on grid-open; forged grid-edit writes 'Hacked'. + +- [ ] **Step 6.3: Implement.** In `server/Session.ts`: + +Add imports: +```ts +import { resolveLookup } from '../src/interpreter/LookupResolver.js'; +import type { ClientMessage, ServerMessage, ColInfo, OutputLine, FormField } from '../src/shared/types.js'; +``` +(extends the existing type import on line 11; `FormField` is used in Task 8.) + +Replace `sendGridData` (lines 359-369) with: + +```ts + private async sendGridData(): Promise { + const area = this.executor.area; + if (!area.table) { + this.send({ type: 'output', lines: [{ text: 'No table selected', cls: 'error' }] }); + return; + } + const columns = await this.bridge.getStructure(area.table); + const columnTypes = columnMetaStore.listColumnTypes(area.db ?? '', area.table); + const warns: OutputLine[] = []; + for (const [col, meta] of Object.entries(columnTypes)) { + if (!meta.lookup) continue; + const options = await resolveLookup(this.bridge, meta.lookup); + if (options) meta.options = options; + else warns.push({ text: `** Warning: lookup for ${col} could not be resolved — free entry`, cls: 'warn' }); + } + if (warns.length) this.send({ type: 'output', lines: warns }); + const rows = await this.executor.getOrderedRowsWithIds(2000); + this.send({ type: 'grid-open', table: area.table, filter: area.filter, columns, columnTypes, rows }); + } +``` + +In the `grid-edit` case (lines 74-93), replace the validation lines (80-86) with: + +```ts + const db = this.executor.area.db ?? ''; + let meta = columnMetaStore.getColumnType(db, table, col); + if (meta?.lookup) { + // Fresh re-resolve at write time — the option list the client got + // at grid-open may be stale, and a forged message never saw one. + const options = await resolveLookup(this.bridge, meta.lookup); + meta = { ...meta, options: options ?? undefined }; + } + const err = validateCellValue(col, value, meta); + if (err) { + this.send({ type: 'output', lines: [{ text: `** ${err}`, cls: 'error' }] }); + await this.sendGridData(); + break; + } +``` + +- [ ] **Step 6.4: Run to verify pass** + +Run: `npx vitest run tests/LookupEnforcement.test.ts` → PASS. Then `npx vitest run tests/GridMessages.test.ts tests/ColumnMeta.test.ts` → still green. + +- [ ] **Step 6.5: Commit** + +```bash +git add server/Session.ts tests/LookupEnforcement.test.ts +git commit -m "feat(#60): grid-open resolves lookup options; grid-edit enforces membership fresh" +``` + +--- + +### Task 7: Executor — field-bound @ SAY GET + +**Files:** +- Modify: `src/shared/types.ts:12-17` (`FormField`) +- Modify: `src/interpreter/Executor.ts:542-549` (`doAtSayGet`) +- Test: `tests/FormFieldBinding.test.ts` (new) + +- [ ] **Step 7.1: Write the failing tests** — create `tests/FormFieldBinding.test.ts`: + +```ts +import { describe, it, expect, afterEach } from 'vitest'; +import { Session } from '../server/Session'; +import type { ServerMessage } from '../src/shared/types'; +import fs from 'fs'; +import path from 'path'; + +let dbCounter = 0; +function uniqueDb() { return `test_formbind_${Date.now()}_${++dbCounter}`; } + +afterEach(() => { + const dataDir = path.join(process.cwd(), 'data'); + if (fs.existsSync(dataDir)) { + fs.readdirSync(dataDir) + .filter(f => f.toLowerCase().startsWith('test_formbind_')) + .forEach(f => fs.unlinkSync(path.join(dataDir, f))); + } +}); + +async function setup() { + const sent: ServerMessage[] = []; + const session = new Session((m) => sent.push(m)); + const run = async (text: string) => { + sent.length = 0; + await session.handleMessage({ type: 'command', text }); + const out = sent.filter(m => m.type === 'output') as any[]; + return out.flatMap(o => o.lines).map((l: any) => l.text).join('\n'); + }; + await run(`USE DATABASE ${uniqueDb()}`); + await run('CREATE TABLE EMPLOYEES (EMPID CHAR(4), NAME CHAR(30), SCHEDID CHAR(4) LOOKUP ("S001","S002"))'); + await run('USE EMPLOYEES'); + return { session, sent, run }; +} + +/** Run a multi-line block as one command (the terminal sends blocks whole). */ +async function runBlock(session: Session, sent: ServerMessage[], src: string) { + sent.length = 0; + await session.handleMessage({ type: 'command', text: src }); +} + +describe('field-bound @ SAY GET', () => { + it('binds a GET whose name matches a column: field target, prefill, options', async () => { + const { session, sent, run } = await setup(); + await run('APPEND RECORD'); + await run('REPLACE EMPID WITH "E001", NAME WITH "Ada", SCHEDID WITH "S001"'); + + await runBlock(session, sent, '@ 4, 5 SAY "Name: " GET NAME\n@ 5, 5 SAY "Sched: " GET SCHEDID\nREAD'); + const form = sent.find(m => m.type === 'form-open') as any; + expect(form).toBeDefined(); + const nameField = form.fields.find((f: any) => f.varName === 'NAME'); + expect(nameField.target.kind).toBe('field'); + expect(nameField.target.column).toBe('NAME'); + expect(typeof nameField.target.rowid).toBe('number'); + expect(nameField.value).toBe('Ada'); + const schedField = form.fields.find((f: any) => f.varName === 'SCHEDID'); + expect(schedField.options).toEqual([ + { value: 'S001', label: 'S001' }, + { value: 'S002', label: 'S002' }, + ]); + expect(schedField.value).toBe('S001'); + }); + + it('fields shadow memory variables: an existing var of the same name loses', async () => { + const { session, sent, run } = await setup(); + await run('APPEND RECORD'); + await run('STORE "not-the-field" TO NAME'); + await runBlock(session, sent, '@ 4, 5 SAY "Name: " GET NAME\nREAD'); + const form = sent.find(m => m.type === 'form-open') as any; + expect(form.fields[0].target.kind).toBe('field'); + }); + + it('falls back to a memory variable when no column matches', async () => { + const { session, sent, run } = await setup(); + await run('APPEND RECORD'); + await runBlock(session, sent, '@ 4, 5 SAY "Id: " GET M_EMP\nREAD'); + const form = sent.find(m => m.type === 'form-open') as any; + expect(form.fields[0].target.kind).toBe('var'); + expect(form.fields[0].value).toBe(''); + }); + + it('is a variable GET when no table is in use', async () => { + const sent: ServerMessage[] = []; + const session = new Session((m) => sent.push(m)); + await session.handleMessage({ type: 'command', text: '@ 4, 5 SAY "X: " GET WHATEVER\nREAD' }); + const form = sent.find(m => m.type === 'form-open') as any; + expect(form.fields[0].target.kind).toBe('var'); + }); + + it('errors on a field-bound GET with no current record', async () => { + const { session, sent } = await setup(); // table is empty + await runBlock(session, sent, '@ 4, 5 SAY "Name: " GET NAME\nREAD'); + const out = sent.filter(m => m.type === 'output') as any[]; + expect(out.flatMap(o => o.lines).map((l: any) => l.text).join('\n')) + .toMatch(/GET NAME: no current record/); + expect(sent.find(m => m.type === 'form-open')).toBeUndefined(); + }); + + it('degrades a dead lookup on a field GET: no options, warn line, form still opens', async () => { + const { session, sent, run } = await setup(); + await run('ALTER TABLE EMPLOYEES ADD BADCOL CHAR LOOKUP GHOST.X'); + await run('APPEND RECORD'); + await runBlock(session, sent, '@ 4, 5 SAY "B: " GET BADCOL\nREAD'); + const form = sent.find(m => m.type === 'form-open') as any; + expect(form.fields[0].options).toBeUndefined(); + const out = sent.filter(m => m.type === 'output') as any[]; + expect(out.flatMap(o => o.lines).map((l: any) => l.text).join('\n')).toMatch(/lookup for BADCOL/i); + }); +}); +``` + +- [ ] **Step 7.2: Run to verify failure** + +Run: `npx vitest run tests/FormFieldBinding.test.ts` +Expected: FAIL — `target` undefined on fields. + +- [ ] **Step 7.3: Implement.** In `src/shared/types.ts`, replace `FormField` (lines 12-17) with: + +```ts +export interface FormField { + row: number; + col: number; + label: string; + /** The submit key. For a field-bound GET this is the column name. */ + varName: string; + /** What form-submit writes. Absent (legacy INPUT/@SAY paths) means 'var'. */ + target?: + | { kind: 'var' } + | { kind: 'field'; column: string; table: string; db: string; rowid: number }; + /** Prefill. Field GETs carry the record's value; var GETs stay '' (unchanged UX). */ + value?: string; + /** Resolved lookup options — render a ` editor for lookup columns + +**Files:** +- Modify: `src/ui/Grid.ts:166-197` (`startEdit`) +- Modify: `src/styles/main.css:255` + +- [ ] **Step 10.1: Implement.** In `src/ui/Grid.ts`, inside `startEdit`, after `const cur = String(this.rows[ri][colName] ?? '');` (line 173) and the `td.classList` line (174), branch on options — replace lines 175-196 with: + +```ts + const meta = this.columnTypes[colName]; + if (meta?.options?.length) { + // Lookup column: the dropdown IS the validation on the happy path + // (the server still re-checks). Static cells keep showing the stored + // value — only this editor shows display labels. + const sel = document.createElement('select'); + sel.className = 'cell-ed'; + const blank = document.createElement('option'); + blank.value = ''; blank.textContent = ''; + sel.appendChild(blank); + for (const o of meta.options) { + const op = document.createElement('option'); + op.value = o.value; + op.textContent = o.label === o.value ? o.value : `${o.label} (${o.value})`; + sel.appendChild(op); + } + sel.value = cur; + td.textContent = ''; + td.appendChild(sel); + sel.focus(); + this.editingCell = { r: ri, c: ci }; + + sel.addEventListener('keydown', (e) => { + if (e.key === 'Enter' || e.key === 'Tab') { + e.preventDefault(); e.stopPropagation(); + if (!this.commitEdit(sel.value)) return; + if (e.key === 'Tab') this.selectCell(ri, ci + 2); + } else if (e.key === 'Escape') { + e.preventDefault(); e.stopPropagation(); + this.cancelEdit(); + } + }); + return; + } + + const inp = document.createElement('input'); + inp.className = 'cell-ed'; inp.value = cur; + td.textContent = ''; + td.appendChild(inp); + inp.focus(); inp.select(); + this.editingCell = { r: ri, c: ci }; + + // Clear a stale error as soon as the value becomes valid again. + inp.addEventListener('input', () => { + if (!validateCellValue(colName, inp.value, this.columnTypes[colName])) this.clearCellError(td); + }); + + inp.addEventListener('keydown', (e) => { + if (e.key === 'Enter' || e.key === 'Tab') { + e.preventDefault(); e.stopPropagation(); + if (!this.commitEdit(inp.value)) return; // invalid — stay in edit mode + if (e.key === 'Tab') this.selectCell(ri, ci + 2); + } else if (e.key === 'Escape') { + e.preventDefault(); e.stopPropagation(); + this.cancelEdit(); + } + }); +``` + +`commitEdit` needs no change — `validateCellValue` now sees `meta.options` and membership passes for picked values. + +- [ ] **Step 10.2: CSS.** In `src/styles/main.css`, extend line 255 so selects get the invalid styling too (edit in place): + +```css +#grid-table td.cell-invalid input.cell-ed, +#grid-table td.cell-invalid select.cell-ed { border-color: #cc0000; background: #2a0000; } +``` + +Also add, next to the `.cell-ed` base rule (grep `cell-ed` in the file): + +```css +select.cell-ed { min-width: 110px; } +``` + +- [ ] **Step 10.3: Verify** — `npm run build` clean. Real-browser assertions land in Task 14. + +- [ ] **Step 10.4: Commit** + +```bash +git add src/ui/Grid.ts src/styles/main.css +git commit -m "feat(#60): BROWSE edits lookup columns through a dropdown" +``` + +--- + +### Task 11: overtime.prg — SCHEDULES table, lookup, two-form Add Employee + +**Files:** +- Modify: `demos/overtime.prg` (seed block lines 53-96, CASE "1" lines 139-160, area setup lines 26-44) +- Modify: `tests/DemoSchemas.test.ts:28` (golden) +- Modify: `tests/overtime.spec.ts` (beforeEach table list line ~83; new test) + +- [ ] **Step 11.1: Update the golden first (failing test).** In `tests/DemoSchemas.test.ts`, add to `DEMO_SCHEMAS` after the `EMPLOYEES` line: + +```ts + SCHEDULES: ['SCHEDID', 'DESCR'], +``` + +Run: `npx vitest run tests/DemoSchemas.test.ts` → FAIL (`no CREATE TABLE SCHEDULES found in demos/*.prg`). + +- [ ] **Step 11.2: Edit `demos/overtime.prg`.** + +(a) Add a work area for SCHEDULES — after the `SELECT SCH … USE SCHEDULEDAYS` block (lines 30-32), insert: + +``` +SELECT SCD +USE DATABASE OVERTIME +USE SCHEDULES +``` + +(b) In the seed block, right after `IF RECCOUNT() == 0` (line 54) and before `DROP TABLE EMPLOYEES`, seed the schedule catalog (the lookup source): + +``` + SELECT SCD + DROP TABLE SCHEDULES + CREATE TABLE SCHEDULES (SCHEDID CHAR(4), DESCR CHAR(30)) + APPEND RECORD + REPLACE SCHEDID WITH "S001", DESCR WITH "Standard 40h (08:00-16:30)" + APPEND RECORD + REPLACE SCHEDID WITH "S002", DESCR WITH "Short 31.25h (09:00-16:00)" + + SELECT EMP +``` + +(c) Change the `CREATE TABLE EMPLOYEES` line (56) to declare the lookup: + +``` + CREATE TABLE EMPLOYEES (EMPID CHAR(4), NAME CHAR(30), SCHEDID CHAR(4) LOOKUP SCHEDULES.SCHEDID DISPLAY DESCR) +``` + +(d) Replace the whole `CASE UPPER(TRIM(choice)) == "1"` arm (lines 139-160) with the check-first, two-form flow. The id stays a memory variable (it is a search term until the record exists); the create runs in natural order so writing the key cannot move the new record; NAME and SCHEDID are field-bound, and SCHEDID's picker comes from the column's lookup: + +``` + CASE UPPER(TRIM(choice)) == "1" + CLEAR + @ 2, 5 SAY "--- ADD EMPLOYEE ---" + SELECT EMP + SET INDEX TO BYEMP + STORE SPACE(4) TO m_emp + @ 4, 5 SAY "Employee ID (4): " GET m_emp + READ + SEEK TRIM(m_emp) + IF FOUND() + @ 8, 5 SAY "Employee already exists: " + TRIM(m_emp) + ELSE + * Create in natural order: writing the key under an active index + * would move the record out from under the form. + SET INDEX TO + APPEND RECORD + REPLACE EMPID WITH TRIM(m_emp) + @ 5, 5 SAY "Name (30): " GET NAME + @ 6, 5 SAY "Schedule : " GET SCHEDID + READ + SET INDEX TO BYEMP + @ 8, 5 SAY "Employee added: " + TRIM(m_emp) + ENDIF + INPUT "Press Enter to continue" TO pause + SELECT EMP +``` + +- [ ] **Step 11.3: Update `tests/overtime.spec.ts`.** + +(a) In the `beforeEach` clean-slate loop (~line 83), add `'SCHEDULES'` to the dropped-tables list: + +```ts + for (const t of ['EMPLOYEES', 'SCHEDULES', 'SCHEDULEDAYS', 'TIMESHEET', 'WEEKSUMMARY', 'LEAVETAKEN']) { +``` + +(b) Add a new test (after the seeding test): + +```ts + test('Add Employee: the schedule is picked from a lookup dropdown, not typed', async ({ page }) => { + await menuChoice(page, '1'); // Add Employee → form 1 (id) + await expect(page.locator('#form-view')).toContainText('ADD EMPLOYEE', { timeout: 6000 }); + const idInput = page.locator('#form-view input.f-get').last(); + await idInput.fill('E003'); + await idInput.press('Enter'); + + // Form 2: NAME is a text field, SCHEDID is a `; membership is enforced on both client and server (`grid-edit`) and by `REPLACE`. +- **`form-error` message**: a rejected form submit keeps the form open with the offending fields outlined — writes are all-or-nothing. +- **`SCHEDULES` table in the overtime demo** (#61): Add Employee picks the schedule from a dropdown showing descriptions, storing the code. CRM deal stages are constrained to the real stage list. +- **Wizard support** (#62): Table and Modify-structure wizards take an optional per-column lookup. + +### Changed +- Form GETs no longer close on submit; the server validates first (success returns to the terminal, rejection keeps the form open). +- `REPLACE` now enforces lookup membership on columns that declare one (additive — no pre-1.3.0 column does). +``` + +- [ ] **Step 15.2: README.md** — three edits: + +(a) Column-types table: add a row / note under it: + +```markdown +Any column may add `LOOKUP
. [DISPLAY ]` or `LOOKUP ("a","b",…)` — see *Lookup columns*. +``` + +(b) New section after the column-types section: + +```markdown +### Lookup columns + +Declare a column's legal values **once, on the column**: + + CREATE TABLE EMPLOYEES (EMPID CHAR(4), NAME CHAR(30), + SCHEDID CHAR(4) LOOKUP SCHEDULES.SCHEDID DISPLAY DESCR) + CREATE TABLE DEALS (STAGE CHAR(12) LOOKUP ("Lead","Qualified","Proposal","Won","Lost")) + +Everything inherits it: BROWSE edits the column through a dropdown, a +field-bound form `GET` renders a picker (`DISPLAY` shows a label, the code is +stored), and `REPLACE`/`grid-edit` reject off-list values. A lookup that cannot +be resolved (source table dropped, empty, or over 1000 distinct values) +degrades to free text with a warning — it never locks the column and never +truncates the list. + +`@ r,c SAY "…" GET ` binds to the active table's column when one matches +(the current record must exist — `APPEND RECORD` first); otherwise it remains a +memory-variable GET. **Fields shadow memory variables**, as in dBASE III. + +#### Deviations from dBASE III + +`LOOKUP`/`DISPLAY` are WebBase-III inventions — dBASE III had no table-driven +lookup at all; dBASE III+ only offered `PICTURE "@M a,b,c"`, a literal list +cycled with the spacebar, which we deliberately do not implement. Field-bound +`GET` *is* authentic dBASE III behavior. This joins the other documented +deviations: unlimited work areas and `alias.field` (not `alias->field`). +``` + +(c) Variables & I/O command table: change the `@ r,c SAY … GET` row description to: `Define a form field; a name matching a column of the active table binds that column (lookup columns render a picker)`. + +- [ ] **Step 15.3: CLAUDE.md** — keep it truthful: + - Architecture block: add `LookupResolver.ts` under `src/interpreter/` ("resolves a column's LOOKUP to {value,label} options; degrades, never truncates"). + - Column types section: add the `LOOKUP` clause syntax + one-paragraph summary and the deviation note. + - Cell-validation table: add row `| lookup columns | value must be one of the resolved options (case-sensitive) |`, and update the closing sentence "REPLACE enforces only `TIME`" → "`REPLACE` enforces `TIME` and lookup membership (columns that declare a `LOOKUP`); widening beyond that would change the semantics of existing programs." + - Roadmap: add a "Beyond parity (v1.3.0)" block listing #58–#62 as shipped. + - Test counts in the Testing section: update after Step 15.5 with the real numbers from the runs. + +- [ ] **Step 15.4: Screenshots** — the grid and form UI changed. Check what exists (`ls docs/screenshots/`) and retake any image that shows BROWSE or a form so it reflects current UI; add one new shot of the Add-Employee schedule dropdown (capture during a paused `npx playwright test tests/overtime.spec.ts --headed`, or via a `page.screenshot` line temporarily added to the new overtime test). Commit the images. + +- [ ] **Step 15.5: Full gate — run serially, in this order:** + +```bash +npm test # all vitest green +npm run build # clean typecheck/build +npx playwright test # all e2e green (dev server via webServer config) +npm run coverage # eyeball: LookupResolver, new Session branches covered +``` + +- [ ] **Step 15.6: Commit + PR** + +```bash +git add CHANGELOG.md README.md CLAUDE.md docs/screenshots +git commit -m "docs: lookup columns — README deviations section, changelog, CLAUDE.md (v1.3.0)" +git push -u origin feature/lookup-columns +gh pr create --base release/v1.3.0 --title "Lookup columns + field-bound GET (v1.3.0)" \ + --body "Implements the approved spec (docs/superpowers/specs/2026-07-10-lookup-columns-design.md). + +Closes #58, closes #59, closes #60, closes #61, closes #62. + +Deviation from spec: the resolver lives at src/interpreter/LookupResolver.ts (not server/) because the Executor must import it and src/ cannot depend on server/." +``` + +Do **not** merge until both CI jobs (`unit`, `e2e`) are green. Do **not** tag — tags happen only when `release/v1.3.0` merges to `main`. + +--- + +## Self-review notes (already applied) + +- **Spec coverage:** qualifier syntax → T2; storage/migration → T3; resolution/degradation/ceiling → T4; single-source inheritance → T5-T8; forms surface → T7-T9; grid surface → T6/T10; enforcement → T5/T6/T8; demos → T11/T12; Assistant parity → T13; deviations documented → T15. Blank-record cleanup pattern → T11 (two-form, natural-order create). All-or-nothing + retained targets + forged test → T8. `toBeInViewport` → T11/T13/T14. Two-DB tests → T3/T5. +- **Type consistency:** `Lookup`/`LookupOption` defined once in `cellValidation.ts`, re-exported from `types.ts`; `resolveLookup(db, lookup): Promise` used identically in T5/T6/T7/T8; `FormField.target` field variant `{ column, table, db, rowid }` written in T7, consumed in T8. +- **Known accepted gaps** (spec-sanctioned): no lookup-removal syntax (re-declare the column via `ALTER … ALTER` without a clause — `setColumnType` with `null` clears it, which the wizard does not expose); `LIST STRUCTURE` does not print lookups; labels appear only in editors, never in static cells. diff --git a/docs/superpowers/specs/2026-07-10-lookup-columns-design.md b/docs/superpowers/specs/2026-07-10-lookup-columns-design.md new file mode 100644 index 0000000..23cce53 --- /dev/null +++ b/docs/superpowers/specs/2026-07-10-lookup-columns-design.md @@ -0,0 +1,333 @@ +# Lookup columns and field-bound GET — v1.3.0 design + +Status: approved, not yet implemented +Milestone: v1.3.0 +Supersedes nothing. Defers #34 (Assistant Join / Work-areas wizards) to v1.4.0. + +## Problem + +`demos/overtime.prg` asks the user to type a schedule id from memory: + +``` +@ 6, 5 SAY "Schedule ID (4): " GET m_sch +``` + +`SCHEDID` is a foreign key into `SCHEDULEDAYS`, but W3Script has no way to know +that. `FormLayout.ts` renders every `GET` as `input type="text"`. The BROWSE grid +validates a cell against its declared *type* (`TIME(15)`, `NUM(8,2)`) and never +against a set of legal values. The user must remember `EARL`/`LATE`/`NIGHT`, and +nothing stops them writing `EARLY`. + +Three surfaces need to constrain a value to a list: form `GET` fields, BROWSE +grid cells, and the `REPLACE` / `grid-edit` write paths behind them. + +## Deviations from dBASE III + +This feature has no dBASE III ancestor and does not pretend to one. + +dBASE III+ offered `@ ... GET var PICTURE "@M red,green,blue"`: a literal, +comma-separated list cycled with the spacebar. It had no table-driven lookup, no +display column, and no column-level declaration. `PICTURE` is a formatting +mini-language (`@!`, `@R`, `999.99`) that WebBase-III does not implement, so +borrowing its syntax would imply semantics we do not have. + +`LOOKUP` as a column qualifier, `DISPLAY` for a label column, and lookup +inheritance by field-bound `GET` are all WebBase-III inventions. This is a +deliberate deviation, in the same spirit as unlimited work areas (dBASE III +capped at 10) and `alias.field` dot notation (dBASE III used `alias->field`). +It must be documented as such in `README.md` and `CLAUDE.md`, not left for a +user to infer lineage from familiar-looking keywords. + +Field-bound `GET` (`GET SCHEDID` editing the current record, written back by +`READ`) *is* authentic dBASE III behavior, and replaces the memory-variable +round-trip the demos use today. + +## Data model + +One shape, in `src/shared/types.ts`, shared by both declaration sites: + +```ts +export type Lookup = + | { kind: 'list'; values: string[] } + | { kind: 'table'; table: string; column: string; display?: string }; +``` + +`ColumnTypeInfo` gains `lookup?: Lookup`. + +### Single source of truth + +The lookup is declared **once, on the column**. Nothing else may redeclare it. +A form `GET` bound to that column inherits it; a grid cell editing that column +inherits it; `REPLACE` into that column enforces it. There is no per-`GET` +lookup syntax, because a second declaration site is a second thing to keep in +sync, and it would drift. + +A `GET` on a genuine scratch variable (a menu choice, a search term) has no +column and therefore no lookup. That is accepted: such variables are not +constrained values, they are free input. + +## Syntax + +``` +CREATE TABLE EMPLOYEES (EMPID CHAR(4), NAME CHAR(30), + SCHEDID CHAR(4) LOOKUP SCHEDULES.SCHEDID DISPLAY DESCR) + +CREATE TABLE DEALS (STAGE CHAR(10) LOOKUP ("lead","demo","won","lost")) +``` + +The `LOOKUP` clause follows the type. Two right-hand forms: + +- `LOOKUP
. [DISPLAY ]` — live table lookup. +- `LOOKUP ("a","b","c")` — literal list. + +Both forms are parsed by one function, reused by `ALTER TABLE ADD` and +`ALTER TABLE ALTER`. `CREATE TABLE` is strict (see Test discipline in +`CLAUDE.md`): a malformed `LOOKUP` clause throws rather than being absorbed. + +### Field-bound GET + +``` +APPEND RECORD +@ 4, 5 SAY "Employee ID: " GET EMPID +@ 5, 5 SAY "Name : " GET NAME +@ 6, 5 SAY "Schedule : " GET SCHEDID +READ +``` + +`doAtSayGet` (`Executor.ts:542`) resolves the name against the active table's +columns. A match binds the field: the form prefills from the current record and, +if the column declares a lookup, carries its resolved options. No match falls +back to a memory variable, exactly as today. This mirrors how `@ SAY` already +resolves field names inside expressions. + +`READ` writes field-bound values on submit. Escape writes nothing. + +**Fields shadow memory variables.** If the active table has a column with the +GET's name, the field wins — even when a memory variable of that name already +exists. The alternative, letting an earlier `STORE` change what a `GET` means, +would make the binding depend on execution history. Field-over-memvar +precedence is also what dBASE III did; the community's universal `m_` prefix +convention exists because of it, and every `GET` in the demos already follows +that convention (audited against the golden schemas — no demo GET name collides +with a column). README documents the rule as a behavior change for programs +that `GET` a variable whose name matches a column of the table in use. + +At `READ`, each field-bound `GET` resolves its record once — the alias captured +at declaration, its current record via the `fetchCurrentRow` path — and keeps +the SQLite `rowid`. `form-submit` writes by that rowid, exactly as `grid-edit` +already does, so pointer motion between `READ` and submit cannot retarget the +write. + +## Resolution + +A `table` lookup resolves server-side to: + +```sql +SELECT DISTINCT [, ] FROM
ORDER BY 1 +``` + +with a 1000-distinct-value ceiling. A `list` lookup needs no resolution. + +Resolution is performed by a new `server/LookupResolver.ts`, which takes the +database bridge and a `Lookup` and returns `{value,label}[]`. It is the only +place that reads lookup source tables. + +### Degradation + +If the source table or column is missing, the query returns zero rows, or the +source exceeds the 1000-value ceiling, the field degrades to free text and the +command emits a `warn` line. It must never make a form unopenable or a column +unwritable — a user who drops the lookup source table must still be able to +edit records. + +The ceiling degrades, never truncates. A clipped option list would be worse +than none: the dropdown would hide legal values while membership validation +rejected them. + +Membership validation is skipped for a lookup that cannot be resolved, for the +same reason. An unresolvable lookup is a warning, not a lock. + +## Surfaces + +### Forms + +`FormField` grows: + +```ts +target: { kind: 'var'; name: string } | { kind: 'field'; column: string }; +value: string; // prefill +options?: { value: string; label: string }[]; +``` + +`FormLayout.ts:46` renders `` rather than a text input for a +lookup column. The dropdown *is* the validation on the happy path; the server +still re-checks. + +Outside edit mode the cell shows the stored value, matching `LIST` and report +output; only the edit dropdown shows `display` labels. Labels in static cells +would make the grid disagree with every other surface that prints the column. + +A `
. [DISPLAY ] | LOOKUP ("a","b",...) + // A WebBase-III extension (documented deviation — dBASE III had no lookup). + private parseLookupClause(colName: string): Lookup { + this.adv(); // LOOKUP + if (this.peek().type === 'LPAREN') { + this.adv(); + const values: string[] = []; + while (!this.end() && this.peek().type !== 'RPAREN') { + if (this.peek().type !== 'STR') { + this.createErr(`expected a quoted string in the LOOKUP list for column '${colName}'`); + } + values.push(this.adv().val); + if (this.peek().type === 'COMMA') this.adv(); + else break; + } + this.expectRParen(`LOOKUP list for column '${colName}'`); + if (!values.length) this.createErr(`the LOOKUP list for column '${colName}' is empty`); + return { kind: 'list', values }; + } + const t = this.peek(); + if (t.type !== 'ID' && t.type !== 'KW') { + this.createErr(`expected
. or a ("…") list after LOOKUP for column '${colName}'`); + } + const table = this.adv().val; + if (this.peek().type !== 'DOT') { + this.createErr(`expected
. after LOOKUP for column '${colName}'`); + } + this.adv(); // DOT + const cTok = this.peek(); + if (cTok.type !== 'ID' && cTok.type !== 'KW') { + this.createErr(`expected a column after '${table}.' in the LOOKUP for column '${colName}'`); + } + const column = this.adv().val; + if (this.peekKw('DISPLAY')) { + this.adv(); + const dTok = this.peek(); + if (dTok.type !== 'ID' && dTok.type !== 'KW') { + this.createErr(`expected a display column after DISPLAY in the LOOKUP for column '${colName}'`); + } + return { kind: 'table', table, column, display: this.adv().val }; + } + return { kind: 'table', table, column }; + } + private parseDrop(): ASTNode { this.adv(); this.skipKw('TABLE'); @@ -541,8 +593,8 @@ export class Parser { this.adv(); // ALTER this.skipKw('TABLE'); const name = this.ident(); - if (this.peekKw('ADD')) { this.adv(); this.skipKw('COLUMN'); const col = this.ident(); const colType = this.ident(); this.skipTypeSize(); return { type: 'ALTER_TABLE', name, op: 'ADD', col, colType }; } - if (this.peekKw('ALTER')) { this.adv(); this.skipKw('COLUMN'); const col = this.ident(); const colType = this.ident(); this.skipTypeSize(); return { type: 'ALTER_TABLE', name, op: 'ALTER', col, colType }; } + if (this.peekKw('ADD')) { this.adv(); this.skipKw('COLUMN'); const col = this.ident(); const colType = this.ident(); this.skipTypeSize(); const lookup = this.peekKw('LOOKUP') ? this.parseLookupClause(col) : null; return { type: 'ALTER_TABLE', name, op: 'ADD', col, colType, lookup }; } + if (this.peekKw('ALTER')) { this.adv(); this.skipKw('COLUMN'); const col = this.ident(); const colType = this.ident(); this.skipTypeSize(); const lookup = this.peekKw('LOOKUP') ? this.parseLookupClause(col) : null; return { type: 'ALTER_TABLE', name, op: 'ALTER', col, colType, lookup }; } if (this.peekKw('DROP')) { this.adv(); this.skipKw('COLUMN'); const col = this.ident(); return { type: 'ALTER_TABLE', name, op: 'DROP', col }; } if (this.peekKw('RENAME')) { this.adv(); this.skipKw('COLUMN'); const col = this.ident(); this.skipKw('TO'); const newName = this.ident(); return { type: 'ALTER_TABLE', name, op: 'RENAME', col, newName }; } throw new Error('Expected ADD, DROP, RENAME, or ALTER after ALTER TABLE '); diff --git a/src/shared/cellValidation.ts b/src/shared/cellValidation.ts index 6ad5579..bdf7c16 100644 --- a/src/shared/cellValidation.ts +++ b/src/shared/cellValidation.ts @@ -6,10 +6,19 @@ * Returns an error message, or null when the value is acceptable. */ +/** A column's legal-values constraint — a WebBase-III extension, no dBASE III ancestor. */ +export type Lookup = + | { kind: 'list'; values: string[] } + | { kind: 'table'; table: string; column: string; display?: string }; + +export interface LookupOption { value: string; label: string } + export interface ColumnMeta { baseType: string; qualifier: number | null; // CHAR(n) length, TIME(n) minute granularity, NUM(p,s) precision scale: number | null; // NUM(p,s) scale + lookup?: Lookup | null; // declared constraint (may be unresolvable) + options?: LookupOption[]; // resolved values — absent when the lookup degraded } const TIME_RE = /^([01]\d|2[0-3]):([0-5]\d)$/; @@ -30,7 +39,17 @@ export function validateCellValue( if (!meta) return null; // untracked column — no constraint const v = value.trim(); if (v === '') return null; // clearing a cell is always allowed + const typeErr = declaredTypeError(colName, v, meta); + if (typeErr) return typeErr; + // Membership runs only when the lookup resolved; an unresolvable lookup + // degrades to free text rather than locking the column. + if (meta.options && meta.options.length && !meta.options.some(o => o.value === v)) { + return `${colName}: "${v}" is not one of the allowed values`; + } + return null; +} +function declaredTypeError(colName: string, v: string, meta: ColumnMeta): string | null { switch (meta.baseType.toUpperCase()) { case 'TIME': { const m = TIME_RE.exec(v); diff --git a/src/shared/types.ts b/src/shared/types.ts index c9ce093..47f524d 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -13,7 +13,16 @@ export interface FormField { row: number; col: number; label: string; + /** The submit key. For a field-bound GET this is the column name. */ varName: string; + /** What form-submit writes. Absent (legacy INPUT/@SAY paths) means 'var'. */ + target?: + | { kind: 'var' } + | { kind: 'field'; column: string; table: string; db: string; rowid: number }; + /** Prefill. Field GETs carry the record's value; var GETs stay '' (unchanged UX). */ + value?: string; + /** Resolved lookup options — render a fed by SCHEDULES. + const sched = page.locator('#form-view select.f-get'); + await expect(sched).toBeVisible({ timeout: 6000 }); + await expect(sched).toBeInViewport(); + // DISPLAY label + code are both shown in the option text. + await expect(sched.locator('option', { hasText: 'Standard 40h' })).toHaveCount(1); + + const name = page.locator('#form-view input.f-get').first(); + await name.fill('Alan Turing'); + await sched.selectOption('S001'); + await sched.press('Enter'); // last control → submit + + await expect(page.locator('#form-view')).toContainText('Employee added: E003', { timeout: 6000 }); + await ack(page); + + // Verify through the table tour that the record landed with the code. + await menuChoice(page, '9'); + await expect(page.locator('#terminal-output')).toContainText('Alan Turing'); + await expect(page.locator('#terminal-output')).toContainText('S001'); + await ack(page); + }); + test('prep week derives Mon-Fri work dates with DATEADD and shows the ISO week', async ({ page }) => { await menuChoice(page, '3'); await fillForm(page, ['E001', WEEK], 2000);