feat: LOOKUP column qualifier — grammar + storage layer (#58)

CHANGELOG.md

@@ -7,6 +7,23 @@ Versions follow [Semantic Versioning](https://semver.org/) — minor bump per su
 
 ---
 
+## [Unreleased] — v1.3.0 in progress
+
+### Added
+- `LOOKUP` column qualifier — language grammar and storage layer (#58). Any column can
+  declare a constraint on its legal values: `LOOKUP <table>.<column> [DISPLAY <column>]`
+  for a live table-driven lookup, or `LOOKUP ("a","b",...)` for a literal list. Parsed by
+  `CREATE TABLE`/`ALTER TABLE ADD`/`ALTER TABLE ALTER`, persisted per-column in
+  `ColumnMetaStore` via an additive migration (existing declared types are never touched
+  or dropped), and resolvable to concrete `{value,label}` options via the new
+  `src/interpreter/LookupResolver.ts` (degrades to free entry — never truncates — when the
+  source is missing, empty, or exceeds 1000 distinct values). This is a WebBase-III
+  extension with no dBASE III ancestor.
+  **This PR is grammar + storage only** — nothing enforces or renders a `LOOKUP` yet.
+  Membership enforcement in `BROWSE`/`REPLACE` and the form/grid picker UI ship in
+  follow-up PRs (#59, #60) before the feature is documented in the README command
+  reference.
+
 ## [1.2.0] — 2026-07-09 — TIME columns, WEEK()/DATEADD(), BROWSE cell validation, Overtime demo
 
 ### Added

CLAUDE.md

@@ -63,6 +63,9 @@ src/
     Executor.ts         Async AST runner; manages state (db/table/filter/vars/rowPtr/activeIndex). Emits fire-and-forget client side-effects (CSV download, report preview, CSV upload picker) via onSideEffect so they work inside program blocks
     IndexCommands.ts    Index command handlers (extracted from Executor)
     ReportCommands.ts   Report command handlers delegating to ReportRunner
+    LookupResolver.ts   Resolves a column's LOOKUP constraint (literal list, or live table+column+DISPLAY)
+                        to concrete {value,label} options against IDatabaseBridge; degrades to null
+                        (never truncates) on a missing source, empty result, or >1000 distinct values
 
   terminal/
     Terminal.ts         REPL UI — command history, multi-line block accumulation
@@ -194,6 +197,25 @@ WebBase-III supports **unlimited work areas** (no DOS 10-area limit). Cross-area
 
 Declared types are recorded per `(database, table, column)` in `server/ColumnMetaStore.ts`, because SQLite only keeps a storage affinity: `TIME`/`DATE`/`CHAR` are all `TEXT`, `LOGICAL`/`INT` are both `INTEGER`, and a `NUM(p,s)` qualifier is lost entirely.
 
+#### `LOOKUP` column qualifier (in progress — v1.3.0)
+
+Any column may add a `LOOKUP` clause after its type, constraining it to a set of legal
+values — a WebBase-III extension with no dBASE III ancestor (dBASE III+ only had
+`PICTURE "@M a,b,c"`, a literal spacebar-cycled list with no table-driven form):
+
+```
+SCHEDID CHAR(4) LOOKUP SCHEDULES.SCHEDID DISPLAY DESCR   -- live table lookup
+STAGE   CHAR(12) LOOKUP ("Lead","Won","Lost")             -- literal list
+```
+
+`CREATE TABLE`/`ALTER TABLE ADD`/`ALTER TABLE ALTER` parse and store it (`ColumnMetaStore`,
+same additive-migration discipline as the type columns above). `src/interpreter/LookupResolver.ts`
+turns a stored `LOOKUP` into concrete `{value,label}` options against the live database,
+degrading to `null` (never truncating) when the source is missing, empty, or exceeds 1000
+distinct values. **Not yet enforced or rendered anywhere** — BROWSE/`REPLACE` membership
+checking and the form/grid picker UI land in follow-up PRs before this section is promoted
+to the README command reference.
+
 #### Cell validation (`BROWSE`)
 
 `src/shared/cellValidation.ts` holds the rules and runs on **both** sides: `Grid.ts` checks before commit (an invalid edit keeps the cell in edit mode, outlined red, with the reason shown; the error clears as the value becomes valid), and `Session`'s `grid-edit` handler re-checks authoritatively before writing — a WS message can reach the server without passing through the grid.

server/ColumnMetaStore.ts

@@ -1,17 +1,41 @@
 import Database from 'better-sqlite3';
 import fs from 'fs';
 import path from 'path';
-import type { IColumnMetaStore, ColumnTypeInfo } from '../src/shared/types.js';
+import type { IColumnMetaStore, ColumnTypeInfo, Lookup } from '../src/shared/types.js';
 
 const DATA_DIR = path.join(process.cwd(), 'data');
 const DB_PATH  = path.join(DATA_DIR, 'system.sqlite3');
 
+const LOOKUP_COLS = ['lookup_kind', 'lookup_table', 'lookup_col', 'lookup_display', 'lookup_values'] as const;
+
+interface Row {
+  baseType: string; qualifier: number | null; scale: number | null;
+  lookup_kind: string | null; lookup_table: string | null; lookup_col: string | null;
+  lookup_display: string | null; lookup_values: string | null;
+}
+
+function rowToMeta(r: Row): ColumnTypeInfo {
+  const meta: ColumnTypeInfo = { baseType: r.baseType, qualifier: r.qualifier, scale: r.scale };
+  if (r.lookup_kind === 'list') {
+    meta.lookup = { kind: 'list', values: JSON.parse(r.lookup_values ?? '[]') as string[] };
+  } else if (r.lookup_kind === 'table' && r.lookup_table && r.lookup_col) {
+    meta.lookup = r.lookup_display
+      ? { kind: 'table', table: r.lookup_table, column: r.lookup_col, display: r.lookup_display }
+      : { kind: 'table', table: r.lookup_table, column: r.lookup_col };
+  }
+  return meta;
+}
+
+const SELECT_COLS = `base_type AS baseType, qualifier, scale,
+       lookup_kind, lookup_table, lookup_col, lookup_display, lookup_values`;
+
 /**
- * Declared column types, keyed by (database, table, column).
+ * Declared column types (+ optional lookup constraint), keyed by
+ * (database, table, column).
  *
  * SQLite only records a storage affinity (TEXT/REAL/INTEGER), which cannot tell
  * TIME from DATE from CHAR, LOGICAL from INT, or recover a NUM(p,s) qualifier.
- * The grid and REPLACE need the declared type to validate writes.
+ * The grid, forms and REPLACE need the declared type + lookup to validate writes.
  *
  * Scoping by database matters: two databases may each hold a table of the same
  * name with different column types.
@@ -25,57 +49,84 @@ export class ColumnMetaStore implements IColumnMetaStore {
     this.db.pragma('journal_mode = WAL');
     this.db.exec(`
       CREATE TABLE IF NOT EXISTS column_types (
-        db_name    TEXT NOT NULL,
-        table_name TEXT NOT NULL,
-        col_name   TEXT NOT NULL,
-        base_type  TEXT NOT NULL,
-        qualifier  INTEGER,
-        scale      INTEGER,
+        db_name        TEXT NOT NULL,
+        table_name     TEXT NOT NULL,
+        col_name       TEXT NOT NULL,
+        base_type      TEXT NOT NULL,
+        qualifier      INTEGER,
+        scale          INTEGER,
+        lookup_kind    TEXT,
+        lookup_table   TEXT,
+        lookup_col     TEXT,
+        lookup_display TEXT,
+        lookup_values  TEXT,
         PRIMARY KEY (db_name, table_name, col_name)
       );
     `);
     // v1.2.0 dev migration: the first cut of this table (#43) had neither db_name
-    // nor scale. The rows only cache what CREATE TABLE re-records, so rebuilding
-    // is cheaper and safer than back-filling an unscoped key.
-    const cols = this.db.prepare('PRAGMA table_info(column_types)').all() as { name: string }[];
+    // nor scale. Those rows never shipped in a release, so rebuilding was safe.
+    let cols = this.db.prepare('PRAGMA table_info(column_types)').all() as { name: string }[];
     if (!cols.some(c => c.name === 'db_name') || !cols.some(c => c.name === 'scale')) {
       this.db.exec(`
         DROP TABLE column_types;
         CREATE TABLE column_types (
-          db_name    TEXT NOT NULL,
-          table_name TEXT NOT NULL,
-          col_name   TEXT NOT NULL,
-          base_type  TEXT NOT NULL,
-          qualifier  INTEGER,
-          scale      INTEGER,
+          db_name        TEXT NOT NULL,
+          table_name     TEXT NOT NULL,
+          col_name       TEXT NOT NULL,
+          base_type      TEXT NOT NULL,
+          qualifier      INTEGER,
+          scale          INTEGER,
+          lookup_kind    TEXT,
+          lookup_table   TEXT,
+          lookup_col     TEXT,
+          lookup_display TEXT,
+          lookup_values  TEXT,
           PRIMARY KEY (db_name, table_name, col_name)
         );
       `);
+      cols = this.db.prepare('PRAGMA table_info(column_types)').all() as { name: string }[];
+    }
+    // v1.3.0 lookup migration (#58): v1.2.0 SHIPPED, so this one must be
+    // additive — dropping the table here would silently erase every released
+    // user's declared TIME(15)/NUM(p,s) types.
+    for (const col of LOOKUP_COLS) {
+      if (!cols.some(c => c.name === col)) {
+        this.db.exec(`ALTER TABLE column_types ADD COLUMN ${col} TEXT`);
+      }
     }
   }
 
-  setColumnType(dbName: string, tableName: string, colName: string, baseType: string, qualifier: number | null, scale: number | null): void {
+  setColumnType(dbName: string, tableName: string, colName: string, baseType: string, qualifier: number | null, scale: number | null, lookup: Lookup | null = null): void {
+    const kind = lookup?.kind ?? null;
+    const lTable = lookup?.kind === 'table' ? lookup.table : null;
+    const lCol = lookup?.kind === 'table' ? lookup.column : null;
+    const lDisplay = lookup?.kind === 'table' ? (lookup.display ?? null) : null;
+    const lValues = lookup?.kind === 'list' ? JSON.stringify(lookup.values) : null;
     this.db.prepare(`
-      INSERT INTO column_types (db_name, table_name, col_name, base_type, qualifier, scale)
-      VALUES (?, ?, ?, ?, ?, ?)
+      INSERT INTO column_types (db_name, table_name, col_name, base_type, qualifier, scale,
+                                lookup_kind, lookup_table, lookup_col, lookup_display, lookup_values)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
       ON CONFLICT(db_name, table_name, col_name) DO UPDATE SET
-        base_type = excluded.base_type, qualifier = excluded.qualifier, scale = excluded.scale
-    `).run(dbName, tableName, colName, baseType, qualifier, scale);
+        base_type = excluded.base_type, qualifier = excluded.qualifier, scale = excluded.scale,
+        lookup_kind = excluded.lookup_kind, lookup_table = excluded.lookup_table,
+        lookup_col = excluded.lookup_col, lookup_display = excluded.lookup_display,
+        lookup_values = excluded.lookup_values
+    `).run(dbName, tableName, colName, baseType, qualifier, scale, kind, lTable, lCol, lDisplay, lValues);
   }
 
   getColumnType(dbName: string, tableName: string, colName: string): ColumnTypeInfo | null {
     const row = this.db.prepare(
-      'SELECT base_type AS baseType, qualifier, scale FROM column_types WHERE db_name = ? AND table_name = ? AND col_name = ?'
-    ).get(dbName, tableName, colName) as ColumnTypeInfo | undefined;
-    return row ?? null;
+      `SELECT ${SELECT_COLS} FROM column_types WHERE db_name = ? AND table_name = ? AND col_name = ?`
+    ).get(dbName, tableName, colName) as Row | undefined;
+    return row ? rowToMeta(row) : null;
   }
 
   listColumnTypes(dbName: string, tableName: string): Record<string, ColumnTypeInfo> {
     const rows = this.db.prepare(
-      'SELECT col_name AS colName, base_type AS baseType, qualifier, scale FROM column_types WHERE db_name = ? AND table_name = ?'
-    ).all(dbName, tableName) as Array<ColumnTypeInfo & { colName: string }>;
+      `SELECT col_name AS colName, ${SELECT_COLS} FROM column_types WHERE db_name = ? AND table_name = ?`
+    ).all(dbName, tableName) as Array<Row & { colName: string }>;
     const out: Record<string, ColumnTypeInfo> = {};
-    for (const r of rows) out[r.colName] = { baseType: r.baseType, qualifier: r.qualifier, scale: r.scale };
+    for (const r of rows) out[r.colName] = rowToMeta(r);
     return out;
   }
 

src/interpreter/Executor.ts

@@ -781,7 +781,7 @@ export class Executor implements IndexCommandsHost {
     // validate edits.
     for (const c of cols) {
       this.columnMetaStore?.setColumnType(
-        this.metaDb, name, c.name, c.colType.toUpperCase(), c.size ?? null, c.scale ?? null,
+        this.metaDb, name, c.name, c.colType.toUpperCase(), c.size ?? null, c.scale ?? null, c.lookup ?? null,
       );
     }
     this.area.table = name;
@@ -917,7 +917,7 @@ export class Executor implements IndexCommandsHost {
       if (has(node.col)) return { output: [{ text: `ALTER TABLE: column already exists: ${node.col}`, cls: 'error' }] };
       await this.db.exec(`ALTER TABLE ${q(name)} ADD COLUMN ${q(node.col)} ${mapType(node.colType)}`);
       // ALTER TABLE drops any (n)/(p,s) qualifier — the parser skips it.
-      this.columnMetaStore?.setColumnType(this.metaDb, name, node.col, node.colType.toUpperCase(), null, null);
+      this.columnMetaStore?.setColumnType(this.metaDb, name, node.col, node.colType.toUpperCase(), null, null, node.lookup);
       await this.refreshIfActive(name);
       return { output: [{ text: `Added column ${node.col} to ${name}.`, cls: 'ok' }] };
     }
@@ -968,7 +968,7 @@ export class Executor implements IndexCommandsHost {
       await this.db.exec(`CREATE TABLE ${q(name)} (${colDefs})`);
       await this.db.exec(`INSERT INTO ${q(name)} SELECT ${colList} FROM ${q(tmp)}`);
       await this.db.exec(`DROP TABLE ${q(tmp)}`);
-      this.columnMetaStore?.setColumnType(this.metaDb, name, node.col, node.colType.toUpperCase(), null, null);
+      this.columnMetaStore?.setColumnType(this.metaDb, name, node.col, node.colType.toUpperCase(), null, null, node.lookup);
       await this.refreshIfActive(name);
       return { output: [
         { text: `Changed type of ${node.col} to ${node.colType} in ${name}.`, cls: 'ok' },

src/interpreter/LookupResolver.ts

@@ -0,0 +1,43 @@
+import type { IDatabaseBridge } from '../shared/types';
+import type { Lookup, LookupOption } from '../shared/cellValidation';
+
+export const LOOKUP_MAX_VALUES = 1000;
+
+function q(name: string): string {
+  return `"${name.replace(/"/g, '""')}"`;
+}
+
+/**
+ * Resolve a lookup to concrete {value,label} options, or null when it cannot
+ * be honoured (missing table/column, empty source, or over the ceiling).
+ * Callers degrade to free text and skip membership enforcement on null —
+ * an unresolvable lookup is a warning, not a lock. Never truncates: a clipped
+ * option list would hide legal values while membership validation rejected them.
+ */
+export async function resolveLookup(db: IDatabaseBridge, lookup: Lookup): Promise<LookupOption[] | null> {
+  if (lookup.kind === 'list') {
+    return lookup.values.map(v => ({ value: v, label: v }));
+  }
+  if (!(await db.tableExists(lookup.table))) return null;
+  const cols = await db.getStructure(lookup.table);
+  const valueCol = cols.find(c => c.name.toUpperCase() === lookup.column.toUpperCase());
+  if (!valueCol) return null;
+  let displayCol;
+  if (lookup.display) {
+    displayCol = cols.find(c => c.name.toUpperCase() === lookup.display!.toUpperCase());
+    if (!displayCol) return null;
+  }
+  const sel = displayCol ? `${q(valueCol.name)}, ${q(displayCol.name)}` : q(valueCol.name);
+  const rows = await db.query(
+    `SELECT DISTINCT ${sel} FROM ${q(lookup.table)} ORDER BY 1 LIMIT ${LOOKUP_MAX_VALUES + 1}`
+  );
+  const options: LookupOption[] = [];
+  for (const r of rows) {
+    const value = String(r[valueCol.name] ?? '');
+    if (value === '') continue;                      // NULL/empty is not a pickable value
+    const label = displayCol ? String(r[displayCol.name] ?? value) : value;
+    options.push({ value, label });
+  }
+  if (options.length === 0 || options.length > LOOKUP_MAX_VALUES) return null;
+  return options;
+}