From 810087dd4216337309533bef0a4aea53bdc1002e Mon Sep 17 00:00:00 2001 From: Nokome Bentley Date: Tue, 24 Mar 2026 12:37:45 -0600 Subject: [PATCH 1/8] feat(atproto): generate atproto lexicon and support conversion to it --- README.md | 18 + docs/atproto-lexicon.md | 231 ++++++++ docs/myst.yml | 1 + lexicon/document/defs.json | 94 ++++ lexicon/document/document.json | 50 ++ lexicon/richtext/facet.json | 57 ++ packages/oxa-core/src/cli.test.ts | 152 +++++- packages/oxa-core/src/cli.ts | 122 ++++- packages/oxa-core/src/convert.test.ts | 739 ++++++++++++++++++++++++++ packages/oxa-core/src/convert.ts | 277 ++++++++++ packages/oxa-core/src/index.test.ts | 11 + packages/oxa-core/src/index.ts | 2 + packages/oxa-core/tsconfig.json | 3 +- scripts/codegen.ts | 2 + scripts/lib/generate-lexicon.ts | 466 ++++++++++++++++ 15 files changed, 2216 insertions(+), 9 deletions(-) create mode 100644 docs/atproto-lexicon.md create mode 100644 lexicon/document/defs.json create mode 100644 lexicon/document/document.json create mode 100644 lexicon/richtext/facet.json create mode 100644 packages/oxa-core/src/convert.test.ts create mode 100644 packages/oxa-core/src/convert.ts create mode 100644 packages/oxa-core/src/index.test.ts create mode 100644 scripts/lib/generate-lexicon.ts diff --git a/README.md b/README.md index aa8a39e..50553e4 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,24 @@ The schema can be downloaded from oxa.dev. https://oxa.dev/schemas/0.0.1.json +## ATProto conversion + +The `@oxa/core` CLI can also convert OXA documents to the ATProto JSON shape used by +the OXA lexicon. + +```bash +oxa convert --to atproto --created-at 2026-03-22T00:00:00.000Z examples/document.yaml +cat examples/document.yaml | oxa convert --to atproto --yaml --created-at 2026-03-22T00:00:00.000Z - +``` + +The conversion API is also exported from the package entrypoint: + +```ts +import { flattenInlines, mapBlock, oxaToAtproto } from "@oxa/core"; +``` + +The generated ATProto records use the `dev.oxa.document.*` lexicon namespace. + ## Licensing and Attribution Each document can include licensing and rights metadata: diff --git a/docs/atproto-lexicon.md b/docs/atproto-lexicon.md new file mode 100644 index 0000000..c755cd5 --- /dev/null +++ b/docs/atproto-lexicon.md @@ -0,0 +1,231 @@ +--- +title: AT Protocol Lexicon +--- + +# AT Protocol Lexicon + +OXA defines an [AT Protocol (atproto)](https://atproto.com) [Lexicon](https://atproto.com/guides/lexicon) for publishing scientific documents to the [Atmosphere](https://atproto.com/guides/understanding-atproto) — the open, decentralized social network behind [Bluesky](https://bsky.app). + +The lexicon lives under the `dev.oxa.*` namespace and enables OXA documents to be stored as records in any AT Protocol [Personal Data Server (PDS)](https://atproto.com/guides/the-at-stack), making scientific content natively available alongside social interactions, feeds, and moderation infrastructure. + +## Why an AT Protocol lexicon? + +Scientific publishing today relies on centralized platforms. Researchers upload papers to a service, and that service controls access, discovery, and permanence. AT Protocol offers a different model: + +- **User-owned data.** Documents live in the author's signed data repository and can be migrated between hosts. +- **Decentralized discovery.** Any indexer can crawl the network's [firehose](https://atproto.com/specs/sync) to discover and aggregate scientific content — no single gatekeeper required. +- **Interoperability by default.** The Lexicon type system gives every consumer the same schema, so tools can read, validate, and render documents without out-of-band agreements. +- **Built-in identity.** Authors are identified by [DIDs](https://atproto.com/specs/did) and [handles](https://atproto.com/specs/handle), providing a ready-made, portable identity layer. + +By defining a lexicon, OXA documents become first-class objects on the AT Protocol network, subject to the same sync, auth, and moderation primitives as any other record type. + +## Lexicon structure + +The OXA lexicon is organised into two namespaces: + +| File | NSID | Purpose | +|------|------|---------| +| `lexicon/document/document.json` | `dev.oxa.document.document` | The `Document` record type — the root object stored in a PDS | +| `lexicon/document/defs.json` | `dev.oxa.document.defs` | Block-level type definitions (`paragraph`, `heading`, `richText`) and the `block` union | +| `lexicon/richtext/facet.json` | `dev.oxa.richtext.facet` | Facet annotations for inline formatting (`emphasis`, `strong`, `byteSlice`) | + +A `Document` record contains an array of `children` (blocks). Each block carries a `text` string and an optional `facets` array that annotates ranges of that text with formatting features. + +## Following the AT Protocol style guide + +The lexicon follows the conventions described in the [Lexicon Style Guide](https://atproto.com/guides/lexicon-style-guide#design-patterns): + +- **Rich text via facets.** Instead of embedding markup in strings, inline formatting is represented as byte-range annotations — the same pattern established by [`app.bsky.richtext.facet`](https://docs.bsky.app/docs/advanced-guides/post-richtext). This keeps text plain and makes it safe to render even if a consumer doesn't understand a particular facet type. See [why `dev.oxa.richtext.facet`?](#why-devoxa-not-appbsky) below for why OXA defines its own facet lexicon rather than reusing Bluesky's. +- **Open unions.** The `block` union and the facet `features` union are both declared with `"closed": false`, allowing future extension without breaking existing consumers. +- **Minimal required fields.** Only fields that are truly necessary for functionality are marked `required` (e.g. `children` and `createdAt` on a document, `level` on a heading). This keeps the schema flexible for evolution. +- **Singular nouns for record schemas.** The record type is named `document` (not `documents`), following the convention for record schemas. +- **Reusable definitions.** Shared types like `richText` and `byteSlice` are defined once and referenced across lexicons, following the pattern of `defs` files. +- **`createdAt` timestamp.** The document record includes a `createdAt` field (datetime string), which is standard practice for ATProto records. +- **`$type` discriminators.** Every block and facet feature carries a `$type` string so consumers can identify types in the open union without ambiguity. + +## Why `dev.oxa.richtext.facet`? {#why-devoxa-not-appbsky} + +OXA defines its own facet lexicon (`dev.oxa.richtext.facet`) rather than reusing Bluesky's `app.bsky.richtext.facet`. This is a deliberate choice driven by the different domains the two lexicons serve. + +Bluesky's facet features are designed for social microblogging — its feature union contains `mention` (account references), `link` (URLs), and `tag` (hashtags). These are not the annotations scientific documents need. OXA documents require typographic and semantic formatting: `emphasis`, `strong`, and in the future inline types like `subscript`, `superscript`, `inlineMath`, `inlineCode`, `link`, `cite`, and others defined in the OXA schema. + +There are also structural differences. The `app.bsky.richtext.facet` features union is closed, meaning third parties cannot extend it without modifying the original lexicon. The `dev.oxa.richtext.facet` features union is declared with `"closed": false`, following the style guide's recommendation for extensibility. This allows the OXA facet feature set to grow as the schema grows — and because the lexicon is generated from the OXA schema, new inline types become facet features automatically. + +Finally, the `app.bsky` namespace is owned by Bluesky PBC. Extending it with document-formatting features would conflate social and scientific concerns in a namespace OXA does not control. + +> **Note:** As the AT Protocol ecosystem matures, it is possible that a shared, domain-neutral rich text facet standard will emerge. If that happens, OXA will revisit this decision and consider adopting or aligning with such a standard to maximise interoperability across applications. + +## Flattening inlines into facets + +The most significant transformation between an OXA document and its lexicon representation is how inline content is handled. + +### OXA's tree model + +In the OXA schema, inline content is a recursive tree. A `Paragraph` has `children` that can be `Text`, `Emphasis`, `Strong`, or other inline types, and formatting nodes themselves contain `children`: + +```yaml +type: Paragraph +children: + - type: Text + value: "This is " + - type: Strong + children: + - type: Text + value: "bold and " + - type: Emphasis + children: + - type: Text + value: "italic" + - type: Text + value: " text." +``` + +### AT Protocol's flat model + +AT Protocol [uses facets instead of a tree](https://www.pfrazee.com/blog/why-facets). The text is stored as a single plain string, and formatting is described by byte-range annotations: + +```json +{ + "$type": "dev.oxa.document.defs#paragraph", + "text": "This is bold and italic text.", + "facets": [ + { + "index": { "byteStart": 8, "byteEnd": 23 }, + "features": [{ "$type": "dev.oxa.richtext.facet#strong" }] + }, + { + "index": { "byteStart": 17, "byteEnd": 23 }, + "features": [{ "$type": "dev.oxa.richtext.facet#emphasis" }] + } + ] +} +``` + +The conversion walks the inline tree depth-first, concatenating all `Text` node values into a single string. Each formatting node records the byte offset before and after its children are processed, producing a `byteStart`/`byteEnd` pair. Indices count bytes of the UTF-8 encoded text (not characters), matching the AT Protocol convention. + +This design has several advantages in a decentralized setting: + +- **Safe rendering.** A consumer that doesn't recognise a facet type can still display the plain text. +- **Simple validation.** Facets are flat — there is no recursive nesting to validate. +- **Extensibility.** New facet feature types (links, mentions, math, etc.) can be added to the open union without changing the text representation. + +## Converting documents with the CLI + +The `oxa` CLI (provided by the `oxa` npm package) can convert an OXA document to its AT Protocol lexicon representation. + +The `--to` option is required and specifies the target format. Currently the only supported format is `atproto`. + +### From a JSON file + +```bash +oxa convert --to atproto examples/document.json +``` + +### From a YAML file via stdin + +```bash +cat examples/document.yaml | oxa convert --to atproto --yaml - +``` + +### Setting `createdAt` + +By default `createdAt` is set to the current time. You can provide a fixed value: + +```bash +oxa convert --to atproto --created-at 2026-03-22T00:00:00.000Z examples/document.json +``` + +### Example + +Given this OXA document (`doc.yaml`): + +```yaml +type: Document +children: + - type: Heading + level: 1 + children: + - type: Text + value: Hello + - type: Paragraph + children: + - type: Text + value: "Some " + - type: Emphasis + children: + - type: Text + value: emphasized + - type: Text + value: " text." +``` + +Running: + +```bash +oxa convert --to atproto --yaml --created-at 2026-01-01T00:00:00.000Z doc.yaml +``` + +Produces: + +```json +{ + "$type": "dev.oxa.document.document", + "children": [ + { + "$type": "dev.oxa.document.defs#heading", + "level": 1, + "text": "Hello", + "facets": [] + }, + { + "$type": "dev.oxa.document.defs#paragraph", + "text": "Some emphasized text.", + "facets": [ + { + "index": { "byteStart": 5, "byteEnd": 15 }, + "features": [{ "$type": "dev.oxa.richtext.facet#emphasis" }] + } + ] + } + ], + "createdAt": "2026-01-01T00:00:00.000Z" +} +``` + +### Programmatic API + +The conversion functions are also exported from the `@oxa/core` package: + +```typescript +import { flattenInlines, mapBlock, oxaToAtproto } from "@oxa/core"; + +const atprotoRecord = oxaToAtproto(oxaDocument, { + createdAt: "2026-01-01T00:00:00.000Z", +}); +``` + +## Generated from the OXA schema + +The lexicon files are generated from the OXA YAML schema definitions by the codegen script (`scripts/lib/generate-lexicon.ts`). The generator: + +1. Loads the merged OXA JSON Schema. +2. Classifies each type as inline or block based on the `Inline` and `Block` union definitions. +3. Maps inline types to facet features in `dev.oxa.richtext.facet` (excluding `Text`, which becomes the plain text string). +4. Maps block types to object definitions in `dev.oxa.document.defs`, replacing their inline `children` arrays with `text` + `facets` pairs. +5. Emits the `Document` record type in `dev.oxa.document.document`. + +To regenerate the lexicon after changing the schema: + +```bash +pnpm --filter scripts codegen lexicon +``` + +This means that when a new type is added to the OXA schema — for example a new inline type like `Subscript` or a new block type like `CodeBlock` — it is **immediately available** in the lexicon after running codegen. New inline types automatically appear as facet features in the open union, and new block types appear in the block union. No manual lexicon authoring is required. + +## Further reading + +- [AT Protocol documentation](https://atproto.com/docs) +- [Lexicon specification](https://atproto.com/specs/lexicon) +- [Lexicon Style Guide](https://atproto.com/guides/lexicon-style-guide) +- ["Why RichText facets in Bluesky"](https://www.pfrazee.com/blog/why-facets) — the design rationale behind facets +- [OXA Schema overview](./schema-overview.md) diff --git a/docs/myst.yml b/docs/myst.yml index a86c0d0..5e5fdcf 100644 --- a/docs/myst.yml +++ b/docs/myst.yml @@ -17,6 +17,7 @@ project: - file: index.md - file: install.md - file: schema-overview.md + - file: atproto-lexicon.md - title: Schema Reference children: - pattern: schema/*.md diff --git a/lexicon/document/defs.json b/lexicon/document/defs.json new file mode 100644 index 0000000..48e2eba --- /dev/null +++ b/lexicon/document/defs.json @@ -0,0 +1,94 @@ +{ + "lexicon": 1, + "id": "dev.oxa.document.defs", + "defs": { + "richText": { + "type": "object", + "description": "Rich text content: a plain text string with facet annotations.", + "properties": { + "text": { + "type": "string" + }, + "facets": { + "type": "array", + "items": { + "type": "ref", + "ref": "dev.oxa.richtext.facet" + } + } + } + }, + "heading": { + "type": "object", + "description": "A heading with a level and inline content.", + "properties": { + "id": { + "type": "string" + }, + "classes": { + "type": "array", + "items": { + "type": "string" + } + }, + "data": { + "type": "unknown" + }, + "level": { + "type": "integer", + "minimum": 1, + "maximum": 6 + }, + "text": { + "type": "string" + }, + "facets": { + "type": "array", + "items": { + "type": "ref", + "ref": "dev.oxa.richtext.facet" + } + } + }, + "required": [ + "level" + ] + }, + "paragraph": { + "type": "object", + "description": "A paragraph of inline content.", + "properties": { + "id": { + "type": "string" + }, + "classes": { + "type": "array", + "items": { + "type": "string" + } + }, + "data": { + "type": "unknown" + }, + "text": { + "type": "string" + }, + "facets": { + "type": "array", + "items": { + "type": "ref", + "ref": "dev.oxa.richtext.facet" + } + } + } + }, + "block": { + "type": "union", + "closed": false, + "refs": [ + "#heading", + "#paragraph" + ] + } + } +} diff --git a/lexicon/document/document.json b/lexicon/document/document.json new file mode 100644 index 0000000..7b4d34c --- /dev/null +++ b/lexicon/document/document.json @@ -0,0 +1,50 @@ +{ + "lexicon": 1, + "id": "dev.oxa.document.document", + "defs": { + "main": { + "type": "record", + "description": "A document with metadata, title, and block content.", + "key": "tid", + "record": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "classes": { + "type": "array", + "items": { + "type": "string" + } + }, + "data": { + "type": "unknown" + }, + "metadata": { + "type": "unknown" + }, + "title": { + "type": "ref", + "ref": "dev.oxa.document.defs#richText" + }, + "children": { + "type": "array", + "items": { + "type": "ref", + "ref": "dev.oxa.document.defs#block" + } + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + }, + "required": [ + "children", + "createdAt" + ] + } + } + } +} diff --git a/lexicon/richtext/facet.json b/lexicon/richtext/facet.json new file mode 100644 index 0000000..5546919 --- /dev/null +++ b/lexicon/richtext/facet.json @@ -0,0 +1,57 @@ +{ + "lexicon": 1, + "id": "dev.oxa.richtext.facet", + "defs": { + "main": { + "type": "object", + "description": "Annotation of a sub-string within rich text.", + "required": [ + "index", + "features" + ], + "properties": { + "index": { + "type": "ref", + "ref": "#byteSlice" + }, + "features": { + "type": "array", + "items": { + "type": "union", + "closed": false, + "refs": [ + "#emphasis", + "#strong" + ] + } + } + } + }, + "byteSlice": { + "type": "object", + "description": "Specifies the sub-string range a facet feature applies to. Start index is inclusive, end index is exclusive. Indices are zero-indexed, counting bytes of the UTF-8 encoded text.", + "required": [ + "byteStart", + "byteEnd" + ], + "properties": { + "byteStart": { + "type": "integer", + "minimum": 0 + }, + "byteEnd": { + "type": "integer", + "minimum": 0 + } + } + }, + "emphasis": { + "type": "object", + "description": "Emphasized content (typically italicized)." + }, + "strong": { + "type": "object", + "description": "Strongly emphasized content (typically bold)." + } + } +} diff --git a/packages/oxa-core/src/cli.test.ts b/packages/oxa-core/src/cli.test.ts index e3aea31..7c8bbce 100644 --- a/packages/oxa-core/src/cli.test.ts +++ b/packages/oxa-core/src/cli.test.ts @@ -18,27 +18,164 @@ const validYaml = `type: Document children: [] `; -describe("oxa validate", () => { +const convertibleDocument = { + type: "Document", + title: [{ type: "Text", value: "CLI Example" }], + metadata: { license: "CC-BY-4.0" }, + children: [ + { + type: "Paragraph", + children: [{ type: "Text", value: "Hello from CLI" }], + }, + ], +}; + +const convertibleYaml = `type: Document +title: + - type: Text + value: CLI Example +metadata: + license: CC-BY-4.0 +children: + - type: Paragraph + children: + - type: Text + value: Hello from CLI +`; + +describe("oxa cli", () => { let tempDir: string; let validJsonFile: string; let validYamlFile: string; let invalidFile: string; + let convertibleJsonFile: string; + let convertibleYamlFile: string; beforeAll(() => { tempDir = mkdtempSync(join(tmpdir(), "oxa-cli-test-")); validJsonFile = join(tempDir, "valid.json"); validYamlFile = join(tempDir, "valid.yaml"); invalidFile = join(tempDir, "invalid.json"); + convertibleJsonFile = join(tempDir, "convert.json"); + convertibleYamlFile = join(tempDir, "convert.yaml"); writeFileSync(validJsonFile, JSON.stringify(validDocument)); writeFileSync(validYamlFile, validYaml); writeFileSync(invalidFile, '{"type": "Document"}'); + writeFileSync(convertibleJsonFile, JSON.stringify(convertibleDocument)); + writeFileSync(convertibleYamlFile, convertibleYaml); }); afterAll(() => { unlinkSync(validJsonFile); unlinkSync(validYamlFile); unlinkSync(invalidFile); + unlinkSync(convertibleJsonFile); + unlinkSync(convertibleYamlFile); + }); + + describe("convert", () => { + const createdAt = "2026-03-22T00:00:00.000Z"; + const expectedConverted = { + $type: "dev.oxa.document.document", + title: { + text: "CLI Example", + facets: [], + }, + metadata: { license: "CC-BY-4.0" }, + children: [ + { + $type: "dev.oxa.document.defs#paragraph", + text: "Hello from CLI", + facets: [], + }, + ], + createdAt, + }; + + it("converts JSON from stdin to ATProto JSON with --created-at", async () => { + const { exitCode, stdout } = await execa( + "node", + [CLI_PATH, "convert", "--to", "atproto", "--created-at", createdAt, "-"], + { + input: JSON.stringify(convertibleDocument), + }, + ); + + expect(exitCode).toBe(0); + expect(JSON.parse(stdout)).toEqual(expectedConverted); + }); + + it("converts YAML from stdin with --yaml", async () => { + const { exitCode, stdout } = await execa( + "node", + [CLI_PATH, "convert", "--to", "atproto", "--yaml", "--created-at", createdAt, "-"], + { + input: convertibleYaml, + }, + ); + + expect(exitCode).toBe(0); + expect(JSON.parse(stdout)).toEqual(expectedConverted); + }); + + it("converts a YAML file and exits 0", async () => { + const { exitCode, stdout } = await execa("node", [ + CLI_PATH, + "convert", + "--to", + "atproto", + "--created-at", + createdAt, + convertibleYamlFile, + ]); + + expect(exitCode).toBe(0); + expect(JSON.parse(stdout)).toEqual(expectedConverted); + }); + + it("converts a JSON file and exits 0", async () => { + const { exitCode, stdout } = await execa("node", [ + CLI_PATH, + "convert", + "--to", + "atproto", + "--created-at", + createdAt, + convertibleJsonFile, + ]); + + expect(exitCode).toBe(0); + expect(JSON.parse(stdout)).toEqual(expectedConverted); + }); + + it("exits with error for missing --to option", async () => { + const result = await execa( + "node", + [CLI_PATH, "convert", "--created-at", createdAt, "-"], + { + input: JSON.stringify(convertibleDocument), + reject: false, + }, + ); + + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("--to"); + }); + + it("exits with error for unknown --to format", async () => { + const result = await execa( + "node", + [CLI_PATH, "convert", "--to", "unknown", "--created-at", createdAt, "-"], + { + input: JSON.stringify(convertibleDocument), + reject: false, + }, + ); + + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("Unknown format"); + }); }); describe("stdin input", () => { @@ -194,11 +331,24 @@ describe("oxa validate", () => { expect(stdout).toContain("CLI for validating OXA documents"); }); + it("--help lists the convert command", async () => { + const { stdout } = await execa("node", [CLI_PATH, "--help"]); + expect(stdout).toContain("convert"); + }); + it("validate --help shows command help", async () => { const { stdout } = await execa("node", [CLI_PATH, "validate", "--help"]); expect(stdout).toContain("Validate JSON or YAML files"); }); + it("convert --help shows command help and ATProto options", async () => { + const { stdout } = await execa("node", [CLI_PATH, "convert", "--help"]); + expect(stdout).toContain("Convert OXA documents"); + expect(stdout).toContain("--to"); + expect(stdout).toContain("--created-at"); + expect(stdout).toContain("--yaml"); + }); + it("--version shows version", async () => { const { stdout } = await execa("node", [CLI_PATH, "--version"]); expect(stdout).toMatch(/\d+\.\d+\.\d+/); diff --git a/packages/oxa-core/src/cli.ts b/packages/oxa-core/src/cli.ts index be93889..c46eefc 100644 --- a/packages/oxa-core/src/cli.ts +++ b/packages/oxa-core/src/cli.ts @@ -5,7 +5,10 @@ */ import { program } from "commander"; +import { readFileSync } from "fs"; +import yaml from "js-yaml"; import version from "./version.js"; +import { oxaToAtproto, type DocumentNode } from "./convert.js"; import { validateFile, validateJson, @@ -13,6 +16,8 @@ import { type ValidationResult, } from "./validate.js"; +const yamlFileExtensions = [".yaml", ".yml"] as const; + // Exit codes const EXIT_SUCCESS = 0; const EXIT_VALIDATION_FAILURE = 1; @@ -80,6 +85,44 @@ async function readStdin(): Promise { return Buffer.concat(chunks).toString("utf-8"); } +function parseDocumentText(content: string, isYaml: boolean): unknown { + return isYaml ? yaml.load(content) : JSON.parse(content); +} + +function isYamlFilePath(filePath: string): boolean { + return yamlFileExtensions.some((extension) => filePath.endsWith(extension)); +} + +function parseFile(filePath: string): unknown { + const content = readFileSync(filePath, "utf-8"); + return parseDocumentText(content, isYamlFilePath(filePath)); +} + +function validateContent( + content: string, + options: { type?: string; yaml?: boolean }, + format: "cli" | "js", +): ValidationResult { + return options.yaml + ? validateYaml(content, { type: options.type, format }) + : validateJson(content, { type: options.type, format }); +} + +function isStdinInput(file: string | undefined): file is undefined | "-" { + return file === undefined || file === "-"; +} + +async function readDocument( + file: string | undefined, + options: { yaml?: boolean }, +): Promise { + if (isStdinInput(file)) { + return parseDocumentText(await readStdin(), options.yaml ?? false); + } + + return parseFile(file); +} + program .name("oxa") .description("CLI for validating OXA documents") @@ -114,13 +157,7 @@ program // Handle stdin if no files or "-" specified if (files.length === 0 || (files.length === 1 && files[0] === "-")) { const content = await readStdin(); - - let result: ValidationResult; - if (options.yaml) { - result = validateYaml(content, { type: options.type, format }); - } else { - result = validateJson(content, { type: options.type, format }); - } + const result = validateContent(content, options, format); if (!result.valid) { hasFailures = true; @@ -153,4 +190,75 @@ program }, ); +const SUPPORTED_FORMATS = ["atproto"] as const; +type ConvertFormat = (typeof SUPPORTED_FORMATS)[number]; + +program + .command("convert") + .description("Convert OXA documents to other formats") + .argument("[file]", "File to convert (use - for stdin)") + .requiredOption( + "--to ", + `Target format (${SUPPORTED_FORMATS.join(", ")})`, + ) + .option("--yaml", "Parse stdin as YAML") + .option("--created-at ", "Set the ATProto createdAt value") + .action( + async ( + file: string | undefined, + options: { + to: string; + yaml?: boolean; + createdAt?: string; + }, + ) => { + try { + if ( + !SUPPORTED_FORMATS.includes(options.to as ConvertFormat) + ) { + throw new Error( + `Unknown format: "${options.to}". Supported formats: ${SUPPORTED_FORMATS.join(", ")}`, + ); + } + + if (options.createdAt !== undefined) { + const date = new Date(options.createdAt); + if (isNaN(date.getTime())) { + throw new Error( + `Invalid --created-at value: "${options.createdAt}" is not a valid datetime`, + ); + } + } + + const document = await readDocument(file, options); + + if ( + !document || + typeof document !== "object" || + (document as Record).type !== "Document" || + !Array.isArray((document as Record).children) + ) { + throw new Error( + "Input is not a valid OXA Document (missing type: Document or children array)", + ); + } + + console.log( + JSON.stringify( + oxaToAtproto(document as DocumentNode, { + createdAt: options.createdAt, + }), + ), + ); + + process.exit(EXIT_SUCCESS); + } catch (error) { + console.error( + `Error: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exit(EXIT_EXECUTION_ERROR); + } + }, + ); + program.parse(); diff --git a/packages/oxa-core/src/convert.test.ts b/packages/oxa-core/src/convert.test.ts new file mode 100644 index 0000000..6911b68 --- /dev/null +++ b/packages/oxa-core/src/convert.test.ts @@ -0,0 +1,739 @@ +import { describe, it, expect, vi } from "vitest"; +import { readFileSync } from "fs"; +import { dirname, relative, resolve } from "path"; +import { fileURLToPath } from "url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(__dirname, "../../.."); + +const lexiconFiles = { + facet: { + id: "dev.oxa.richtext.facet", + path: resolve(REPO_ROOT, "lexicon/richtext/facet.json"), + }, + defs: { + id: "dev.oxa.document.defs", + path: resolve(REPO_ROOT, "lexicon/document/defs.json"), + }, + document: { + id: "dev.oxa.document.document", + path: resolve(REPO_ROOT, "lexicon/document/document.json"), + }, +} as const; + +const facetFragments = ["emphasis", "strong"] as const; + +const documentBlockRefs = ["#heading", "#paragraph"] as const; + +const requiredDocumentDefs = [ + "richText", + "paragraph", + "heading", + "block", +] as const; + +type LexiconDoc = { + lexicon?: number; + id?: string; + defs?: Record; + [key: string]: unknown; +}; + +type TestTextNode = { + type: "Text"; + value: string; +}; + +type TestFormattingNode = { + type: "Strong" | "Emphasis"; + children: TestInlineNode[]; + id?: string; + classes?: string[]; + data?: Record; +}; + +type TestInlineNode = TestTextNode | TestFormattingNode; + +type TestParagraphNode = { + type: "Paragraph"; + children: TestInlineNode[]; + id?: string; + classes?: string[]; + data?: Record; +}; + +type TestHeadingNode = { + type: "Heading"; + level: number; + children: TestInlineNode[]; + id?: string; + classes?: string[]; + data?: Record; +}; + +type TestBlockNode = TestParagraphNode | TestHeadingNode; + +type TestDocumentNode = { + type: "Document"; + children: TestBlockNode[]; + title?: TestInlineNode[]; + metadata?: Record; +}; + +function text(value: string) { + return { type: "Text", value } satisfies TestTextNode; +} + +function strong( + children: readonly TestInlineNode[], + props: Omit, "type" | "children"> = {}, +) { + return { + type: "Strong", + children: [...children], + ...props, + } satisfies TestFormattingNode; +} + +function emphasis( + children: readonly TestInlineNode[], + props: Omit, "type" | "children"> = {}, +) { + return { + type: "Emphasis", + children: [...children], + ...props, + } satisfies TestFormattingNode; +} + +function paragraph( + children: readonly TestInlineNode[], + props: Omit, "type" | "children"> = {}, +) { + return { + type: "Paragraph", + children: [...children], + ...props, + } satisfies TestParagraphNode; +} + +function heading( + level: number, + children: readonly TestInlineNode[], + props: Omit, "type" | "level" | "children"> = {}, +) { + return { + type: "Heading", + level, + children: [...children], + ...props, + } satisfies TestHeadingNode; +} + +function documentNode( + children: readonly TestBlockNode[], + props: Omit, "type" | "children"> = {}, +) { + return { + type: "Document", + children: [...children], + ...props, + } satisfies TestDocumentNode; +} + +async function flatten(inlines: unknown[]) { + const convertModule = await import("./convert.js").catch((error) => { + if (error instanceof Error && error.message.includes("/src/convert.js")) { + return undefined; + } + + throw error; + }); + + const flattenInlines = convertModule?.flattenInlines; + + expect( + flattenInlines, + "Expected packages/oxa-core/src/convert.ts to export flattenInlines", + ).toBeTypeOf("function"); + + return flattenInlines!(inlines as never); +} + +async function map(block: unknown) { + const convertModule = await import("./convert.js"); + const mapBlock = convertModule.mapBlock; + + expect( + mapBlock, + "Expected packages/oxa-core/src/convert.ts to export mapBlock", + ).toBeTypeOf("function"); + + return mapBlock!(block as never); +} + +async function convertDocument(document: unknown, options?: unknown) { + const convertModule = await import("./convert.js"); + const oxaToAtproto = convertModule.oxaToAtproto; + + expect( + oxaToAtproto, + "Expected packages/oxa-core/src/convert.ts to export oxaToAtproto", + ).toBeTypeOf("function"); + + return oxaToAtproto!(document as never, options as never); +} + +function readLexicon(filePath: string): LexiconDoc { + try { + return JSON.parse(readFileSync(filePath, "utf8")) as LexiconDoc; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error( + `Failed to read or parse ${relative(REPO_ROOT, filePath)}: ${message}`, + ); + } +} + +function stderrOutput(writeSpy: ReturnType) { + return writeSpy.mock.calls + .map(([chunk]: [unknown, ...unknown[]]) => String(chunk)) + .join(""); +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function getLexiconDefs(filePath: string): Record { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return readLexicon(filePath).defs as Record; +} + +function expectFragments( + defs: Record, + fragments: readonly string[], + messagePrefix: string, +) { + for (const fragment of fragments) { + expect(defs[fragment], `${messagePrefix} ${fragment}`).toBeDefined(); + } +} + +function collectRefs(value: unknown, refs: string[] = []): string[] { + if (Array.isArray(value)) { + for (const item of value) { + collectRefs(item, refs); + } + + return refs; + } + + if (value && typeof value === "object") { + const record = value as Record; + + if (typeof record.ref === "string") { + refs.push(record.ref); + } + + for (const nested of Object.values(record)) { + collectRefs(nested, refs); + } + } + + return refs; +} + +function resolveRef( + source: LexiconDoc, + ref: string, + docs: Map, +) { + if (!source.id) { + throw new Error("Source lexicon is missing an id"); + } + + const [targetId, fragment] = ref.startsWith("#") + ? [source.id, ref.slice(1)] + : ref.split("#", 2); + + const targetDoc = docs.get(targetId); + expect( + targetDoc, + `Expected ref ${ref} to resolve to a known lexicon id`, + ).toBeDefined(); + + if (fragment) { + expect( + targetDoc?.defs?.[fragment], + `Expected ref ${ref} to resolve to a defs fragment`, + ).toBeDefined(); + } +} + +describe("ATProto lexicon structure", () => { + it("parses all Phase 1 lexicon files and gives them the expected lexicon ids", () => { + for (const { id, path } of Object.values(lexiconFiles)) { + const lexicon = readLexicon(path); + + expect(lexicon.lexicon).toBe(1); + expect(lexicon.id).toBe(id); + expect(lexicon.defs).toBeTypeOf("object"); + } + }); + + it("defines the richtext facet feature union and required feature fragments", () => { + const defs = getLexiconDefs(lexiconFiles.facet.path); + const featureUnion = defs.main.properties.features.items; + + expect(defs.byteSlice.required).toEqual(["byteStart", "byteEnd"]); + expect(featureUnion.type).toBe("union"); + expect(featureUnion.closed).toBe(false); + expect(featureUnion.refs).toEqual( + facetFragments.map((fragment) => `#${fragment}`), + ); + + expectFragments(defs, facetFragments, "Missing facet fragment"); + }); + + it("defines the shared document defs and open block unions", () => { + const defs = getLexiconDefs(lexiconFiles.defs.path); + + expectFragments(defs, requiredDocumentDefs, "Missing defs fragment"); + + expect(defs.richText.properties.facets.items.ref).toBe( + "dev.oxa.richtext.facet", + ); + expect(defs.block.type).toBe("union"); + expect(defs.block.closed).toBe(false); + expect(defs.block.refs).toEqual(documentBlockRefs); + }); + + it("defines the top-level document record against the shared rich text and block defs", () => { + const main = getLexiconDefs(lexiconFiles.document.path).main; + const record = main.record; + + expect(main.type).toBe("record"); + expect(main.key).toBe("tid"); + expect(record.required).toEqual(["children", "createdAt"]); + expect(record.properties.title.ref).toBe("dev.oxa.document.defs#richText"); + expect(record.properties.children.items.ref).toBe( + "dev.oxa.document.defs#block", + ); + expect(record.properties.createdAt).toEqual({ + type: "string", + format: "datetime", + }); + }); + + it("resolves every local and cross-file ref to a known lexicon definition", () => { + const docs = new Map( + Object.values(lexiconFiles).map(({ id, path }) => [ + id, + readLexicon(path), + ]), + ); + + for (const lexicon of docs.values()) { + for (const ref of collectRefs(lexicon)) { + resolveRef(lexicon, ref, docs); + } + } + }); +}); + +describe("mapBlock", () => { + it("maps a Paragraph block to an ATProto paragraph preserving id, classes, and data", async () => { + const block = paragraph( + [text("Hello "), strong([text("bold")]), text(" text")], + { + id: "para-1", + classes: ["lead"], + data: { align: "left" }, + }, + ); + + await expect(map(block)).resolves.toEqual({ + $type: "dev.oxa.document.defs#paragraph", + id: "para-1", + classes: ["lead"], + data: { align: "left" }, + text: "Hello bold text", + facets: [ + { + index: { byteStart: 6, byteEnd: 10 }, + features: [{ $type: "dev.oxa.richtext.facet#strong" }], + }, + ], + }); + }); + + it("maps a Heading block to an ATProto heading with level and flattened rich text", async () => { + const block = heading(2, [text("Read "), emphasis([text("this")])], { + id: "intro", + classes: ["hero"], + data: { section: true }, + }); + + await expect(map(block)).resolves.toEqual({ + $type: "dev.oxa.document.defs#heading", + id: "intro", + classes: ["hero"], + data: { section: true }, + level: 2, + text: "Read this", + facets: [ + { + index: { byteStart: 5, byteEnd: 9 }, + features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + }, + ], + }); + }); + + it("warns and omits unknown block types instead of coercing them into known ATProto blocks", async () => { + const writeSpy = vi + .spyOn(process.stderr, "write") + .mockImplementation(() => true); + + try { + await expect( + map({ + type: "Callout", + children: [text("Should be dropped")], + id: "callout-1", + classes: ["note"], + data: { severity: "warning" }, + }), + ).resolves.toBeUndefined(); + + expect(writeSpy).toHaveBeenCalled(); + + const warning = stderrOutput(writeSpy); + expect(warning).toContain("unknown block type"); + expect(warning).toContain("Callout"); + } finally { + writeSpy.mockRestore(); + } + }); +}); + +describe("oxaToAtproto", () => { + it("builds a complete ATProto document with title, metadata, mapped children, and deterministic createdAt", async () => { + const createdAt = "2026-03-22T00:00:00.000Z"; + const document = documentNode( + [ + heading(1, [text("Introduction")]), + paragraph([ + text("This is "), + strong([text("bold")]), + text(" and "), + emphasis([text("italic")]), + text(" text."), + ]), + ], + { + title: [text("Hello, World")], + metadata: { + license: "CC-BY-4.0", + author: "Jane Doe", + }, + }, + ); + + await expect(convertDocument(document, { createdAt })).resolves.toEqual({ + $type: "dev.oxa.document.document", + title: { + text: "Hello, World", + facets: [], + }, + metadata: { + license: "CC-BY-4.0", + author: "Jane Doe", + }, + children: [ + { + $type: "dev.oxa.document.defs#heading", + level: 1, + text: "Introduction", + facets: [], + }, + { + $type: "dev.oxa.document.defs#paragraph", + text: "This is bold and italic text.", + facets: [ + { + index: { byteStart: 8, byteEnd: 12 }, + features: [{ $type: "dev.oxa.richtext.facet#strong" }], + }, + { + index: { byteStart: 17, byteEnd: 23 }, + features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + }, + ], + }, + ], + createdAt, + }); + }); + + it("builds a minimal ATProto document when title and metadata are absent", async () => { + const createdAt = "2026-01-01T00:00:00.000Z"; + + await expect( + convertDocument(documentNode([]), { createdAt }), + ).resolves.toEqual({ + $type: "dev.oxa.document.document", + children: [], + createdAt, + }); + }); + + it("omits title when absent, preserves metadata unchanged, and drops unknown child blocks with a warning", async () => { + const createdAt = "2026-03-22T00:00:00.000Z"; + const metadata = { + license: "CC-BY-4.0", + nested: { + tags: ["oxa", "atproto"], + }, + }; + const writeSpy = vi + .spyOn(process.stderr, "write") + .mockImplementation(() => true); + + try { + const converted = await convertDocument( + { + type: "Document", + metadata, + children: [ + paragraph([text("Keep this paragraph")]), + { + type: "Callout", + children: [text("Drop this block")], + data: { severity: "warning" }, + }, + ], + }, + { createdAt }, + ); + + expect(converted).toEqual({ + $type: "dev.oxa.document.document", + metadata, + children: [ + { + $type: "dev.oxa.document.defs#paragraph", + text: "Keep this paragraph", + facets: [], + }, + ], + createdAt, + }); + expect(converted.metadata).toBe(metadata); + expect("title" in converted).toBe(false); + + expect(writeSpy).toHaveBeenCalled(); + + const warning = stderrOutput(writeSpy); + expect(warning).toContain("unknown block type"); + expect(warning).toContain("Callout"); + } finally { + writeSpy.mockRestore(); + } + }); +}); + +describe("flattenInlines", () => { + it("concatenates plain text children and emits no facets", async () => { + const richText = await flatten([ + text("Hello"), + text(", "), + text("world"), + text("!"), + ]); + + expect(richText).toEqual({ + text: "Hello, world!", + facets: [], + }); + }); + + it("emits a strong facet for a single strong node", async () => { + const richText = await flatten([text("Hello "), strong([text("world")])]); + + expect(richText).toEqual({ + text: "Hello world", + facets: [ + { + index: { byteStart: 6, byteEnd: 11 }, + features: [{ $type: "dev.oxa.richtext.facet#strong" }], + }, + ], + }); + }); + + it("emits strong and emphasis facets with the expected byte ranges", async () => { + const richText = await flatten([ + text("This is "), + strong([text("bold")]), + text(" and "), + emphasis([text("italic")]), + text(" text."), + ]); + + expect(richText).toEqual({ + text: "This is bold and italic text.", + facets: [ + { + index: { byteStart: 8, byteEnd: 12 }, + features: [{ $type: "dev.oxa.richtext.facet#strong" }], + }, + { + index: { byteStart: 17, byteEnd: 23 }, + features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + }, + ], + }); + }); + + it("uses UTF-8 byte offsets for formatted multibyte text", async () => { + const richText = await flatten([text("Say "), emphasis([text("café")])]); + + expect(richText).toEqual({ + text: "Say café", + facets: [ + { + index: { byteStart: 4, byteEnd: 9 }, + features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + }, + ], + }); + }); + + it("treats multiple text children inside formatting nodes as one continuous facet span", async () => { + const richText = await flatten([ + text("Start "), + strong([text("very"), text(" bold")]), + text(" and "), + emphasis([text("quite"), text(" italic")]), + ]); + + expect(richText).toEqual({ + text: "Start very bold and quite italic", + facets: [ + { + index: { byteStart: 6, byteEnd: 15 }, + features: [{ $type: "dev.oxa.richtext.facet#strong" }], + }, + { + index: { byteStart: 20, byteEnd: 32 }, + features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + }, + ], + }); + }); + + it("returns an empty rich text result for empty input", async () => { + await expect(flatten([])).resolves.toEqual({ + text: "", + facets: [], + }); + }); + + it("emits overlapping facets for nested strong and emphasis nodes", async () => { + const richText = await flatten([ + strong([text("bold and "), emphasis([text("bold-italic")])]), + ]); + + expect(richText.text).toBe("bold and bold-italic"); + expect(richText.facets).toHaveLength(2); + expect(richText.facets).toEqual( + expect.arrayContaining([ + { + index: { byteStart: 0, byteEnd: 20 }, + features: [{ $type: "dev.oxa.richtext.facet#strong" }], + }, + { + index: { byteStart: 9, byteEnd: 20 }, + features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + }, + ]), + ); + }); + + it("computes UTF-8 byte offsets for 4-byte characters inside formatting spans", async () => { + const richText = await flatten([emphasis([text("🧪test")])]); + + expect(richText).toEqual({ + text: "🧪test", + facets: [ + { + index: { byteStart: 0, byteEnd: 8 }, + features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + }, + ], + }); + }); + + it("handles deeply nested formatting with overlapping facet ranges", async () => { + const richText = await flatten([ + strong([ + text("deep "), + emphasis([text("nest "), strong([text("core")])]), + ]), + ]); + + expect(richText.text).toBe("deep nest core"); + expect(richText.facets).toHaveLength(3); + expect(richText.facets).toEqual( + expect.arrayContaining([ + { + index: { byteStart: 0, byteEnd: 14 }, + features: [{ $type: "dev.oxa.richtext.facet#strong" }], + }, + { + index: { byteStart: 5, byteEnd: 14 }, + features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + }, + { + index: { byteStart: 10, byteEnd: 14 }, + features: [{ $type: "dev.oxa.richtext.facet#strong" }], + }, + ]), + ); + }); + + it("warns to stderr and drops inline id, classes, and data properties on formatting nodes", async () => { + const writeSpy = vi + .spyOn(process.stderr, "write") + .mockImplementation(() => true); + + try { + const richText = await flatten([ + strong([text("styled")], { + id: "inline-id", + classes: ["callout", "accent"], + data: { note: "keep warning only" }, + }), + ]); + + expect(richText).toEqual({ + text: "styled", + facets: [ + { + index: { byteStart: 0, byteEnd: 6 }, + features: [{ $type: "dev.oxa.richtext.facet#strong" }], + }, + ], + }); + + expect(writeSpy).toHaveBeenCalled(); + + const warning = stderrOutput(writeSpy); + expect(warning).toContain("Strong"); + expect(warning).toContain("id"); + expect(warning).toContain("classes"); + expect(warning).toContain("data"); + } finally { + writeSpy.mockRestore(); + } + }); +}); diff --git a/packages/oxa-core/src/convert.ts b/packages/oxa-core/src/convert.ts new file mode 100644 index 0000000..13a4a37 --- /dev/null +++ b/packages/oxa-core/src/convert.ts @@ -0,0 +1,277 @@ +/** + * Minimal inline conversion helpers for OXA rich text. + */ + +type TextNode = { + type: "Text"; + value: string; +}; + +type FormattingNode = { + type: "Strong" | "Emphasis"; + children: InlineNode[]; + id?: string; + classes?: string[]; + data?: Record; +}; + +type InlineNode = TextNode | FormattingNode; + +type BlockNodeBase = { + id?: string; + classes?: string[]; + data?: Record; +}; + +type ParagraphNode = BlockNodeBase & { + type: "Paragraph"; + children: InlineNode[]; +}; + +type HeadingNode = BlockNodeBase & { + type: "Heading"; + level: number; + children: InlineNode[]; +}; + +type UnknownBlockNode = BlockNodeBase & { + type: string; + children?: InlineNode[]; +}; + +type BlockNode = ParagraphNode | HeadingNode | UnknownBlockNode; + +export type DocumentNode = { + type: "Document"; + children: BlockNode[]; + title?: InlineNode[]; + metadata?: Record; +}; + +interface FacetFeature { + $type: "dev.oxa.richtext.facet#strong" | "dev.oxa.richtext.facet#emphasis"; +} + +interface Facet { + index: { + byteStart: number; + byteEnd: number; + }; + features: FacetFeature[]; +} + +interface RichText { + text: string; + facets: Facet[]; +} + +type AtprotoParagraph = RichText & + BlockNodeBase & { + $type: "dev.oxa.document.defs#paragraph"; + }; + +type AtprotoHeading = RichText & + BlockNodeBase & { + $type: "dev.oxa.document.defs#heading"; + level: number; + }; + +type AtprotoBlock = AtprotoParagraph | AtprotoHeading; + +type AtprotoDocument = { + $type: "dev.oxa.document.document"; + title?: RichText; + metadata?: Record; + children: AtprotoBlock[]; + createdAt: string; +}; + +type OxaToAtprotoOptions = { + createdAt?: string; +}; + +type FormattingPropertyName = "id" | "classes" | "data"; +type BlockPropertyName = keyof BlockNodeBase; +type KnownBlockNode = ParagraphNode | HeadingNode; + +const facetFeatureTypes = { + Strong: "dev.oxa.richtext.facet#strong", + Emphasis: "dev.oxa.richtext.facet#emphasis", +} as const; + +const formattingPropertyNames = ["id", "classes", "data"] as const; +const blockPropertyNames = ["id", "classes", "data"] as const; +const paragraphType = "dev.oxa.document.defs#paragraph" as const; +const headingType = "dev.oxa.document.defs#heading" as const; + +const encoder = new TextEncoder(); + +function byteLength(text: string): number { + return encoder.encode(text).byteLength; +} + +function getCurrentByteOffset(richText: RichText): number { + return byteLength(richText.text); +} + +function createFacet( + node: FormattingNode, + byteStart: number, + byteEnd: number, +): Facet { + return { + index: { byteStart, byteEnd }, + features: [{ $type: facetFeatureTypes[node.type] }], + }; +} + +function warn(message: string): void { + process.stderr.write(`Warning: ${message}\n`); +} + +function getDroppedFormattingProperties( + node: FormattingNode, +): FormattingPropertyName[] { + return formattingPropertyNames.filter( + (propertyName) => node[propertyName] !== undefined, + ); +} + +function copyDefinedProperties( + source: T, + propertyNames: readonly K[], +): Pick { + const props = {} as Pick; + + for (const propertyName of propertyNames) { + const value = source[propertyName]; + + if (value !== undefined) { + props[propertyName] = value; + } + } + + return props; +} + +function warnDroppedProperties(node: FormattingNode): void { + const dropped = getDroppedFormattingProperties(node); + + if (dropped.length === 0) { + return; + } + + warn( + `dropping unsupported inline properties on ${node.type}: ${dropped.join(", ")}`, + ); +} + +function flattenNode(node: InlineNode, richText: RichText): void { + if (node.type === "Text") { + richText.text += node.value; + return; + } + + warnDroppedProperties(node); + + const byteStart = getCurrentByteOffset(richText); + + for (const child of node.children) { + flattenNode(child, richText); + } + + const byteEnd = getCurrentByteOffset(richText); + + richText.facets.push(createFacet(node, byteStart, byteEnd)); +} + +export function flattenInlines(inlines: InlineNode[]): RichText { + const richText: RichText = { + text: "", + facets: [], + }; + + for (const inline of inlines) { + flattenNode(inline, richText); + } + + return richText; +} + +function copyBlockProps(block: BlockNodeBase): BlockNodeBase { + return copyDefinedProperties( + block, + blockPropertyNames, + ); +} + +function mapBlockRichText(block: KnownBlockNode): BlockNodeBase & RichText { + return { + ...copyBlockProps(block), + ...flattenInlines(block.children), + }; +} + +function getOptionalDocumentFields( + document: DocumentNode, +): Partial { + return { + ...(document.title !== undefined + ? { title: flattenInlines(document.title) } + : {}), + ...(document.metadata !== undefined ? { metadata: document.metadata } : {}), + }; +} + +function isKnownBlock(block: BlockNode): block is KnownBlockNode { + return block.type === "Paragraph" || block.type === "Heading"; +} + +function warnUnknownBlockType(block: BlockNode): void { + warn(`unknown block type: ${block.type}`); +} + +function mapKnownBlock(block: KnownBlockNode): AtprotoBlock { + const richTextBlock = mapBlockRichText(block); + + if (block.type === "Paragraph") { + return { + $type: paragraphType, + ...richTextBlock, + }; + } + + return { + $type: headingType, + ...richTextBlock, + level: block.level, + }; +} + +function mapKnownBlocks(blocks: BlockNode[]): AtprotoBlock[] { + return blocks.flatMap((block) => { + const mapped = mapBlock(block); + return mapped === undefined ? [] : [mapped]; + }); +} + +export function mapBlock(block: BlockNode): AtprotoBlock | undefined { + if (!isKnownBlock(block)) { + warnUnknownBlockType(block); + return undefined; + } + + return mapKnownBlock(block); +} + +export function oxaToAtproto( + document: DocumentNode, + options: OxaToAtprotoOptions = {}, +): AtprotoDocument { + return { + $type: "dev.oxa.document.document", + ...getOptionalDocumentFields(document), + children: mapKnownBlocks(document.children), + createdAt: options.createdAt ?? new Date().toISOString(), + }; +} diff --git a/packages/oxa-core/src/index.test.ts b/packages/oxa-core/src/index.test.ts new file mode 100644 index 0000000..dffa472 --- /dev/null +++ b/packages/oxa-core/src/index.test.ts @@ -0,0 +1,11 @@ +import { describe, it, expect } from "vitest"; + +describe("package exports", () => { + it("re-exports the ATProto conversion API from the package entrypoint", async () => { + const pkg = await import("./index.js"); + + expect(pkg.flattenInlines).toBeTypeOf("function"); + expect(pkg.mapBlock).toBeTypeOf("function"); + expect(pkg.oxaToAtproto).toBeTypeOf("function"); + }); +}); diff --git a/packages/oxa-core/src/index.ts b/packages/oxa-core/src/index.ts index 8107a6f..bba0f82 100644 --- a/packages/oxa-core/src/index.ts +++ b/packages/oxa-core/src/index.ts @@ -20,6 +20,8 @@ export type * from "oxa-types"; // Export validation functions and types +export { flattenInlines, mapBlock, oxaToAtproto } from "./convert.js"; + export { validate, validateJson, diff --git a/packages/oxa-core/tsconfig.json b/packages/oxa-core/tsconfig.json index a9a3bcf..b052a1c 100644 --- a/packages/oxa-core/tsconfig.json +++ b/packages/oxa-core/tsconfig.json @@ -6,5 +6,6 @@ "rootDir": "./src", "noEmit": false }, - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts"], + "exclude": ["dist", "src/**/*.test.ts"] } diff --git a/scripts/codegen.ts b/scripts/codegen.ts index 9b84534..2a1c0fd 100644 --- a/scripts/codegen.ts +++ b/scripts/codegen.ts @@ -9,6 +9,7 @@ import { program } from "commander"; import { generateConformance } from "./lib/generate-conformance.js"; import { generateDocs } from "./lib/generate-docs.js"; import { generateJson } from "./lib/generate-json.js"; +import { generateLexicon } from "./lib/generate-lexicon.js"; import { generatePy } from "./lib/generate-py.js"; import { generateRs } from "./lib/generate-rs.js"; import { generateTs } from "./lib/generate-ts.js"; @@ -31,6 +32,7 @@ const generators: Generator[] = [ fn: generateConformance, }, { name: "docs", label: "Schema documentation", fn: generateDocs }, + { name: "lexicon", label: "AT Protocol Lexicon", fn: generateLexicon }, ]; async function validate(): Promise { diff --git a/scripts/lib/generate-lexicon.ts b/scripts/lib/generate-lexicon.ts new file mode 100644 index 0000000..51f947d --- /dev/null +++ b/scripts/lib/generate-lexicon.ts @@ -0,0 +1,466 @@ +/** + * Generate AT Protocol Lexicon files from the OXA JSON Schema. + * + * This module translates the OXA tree-structured document schema into + * AT Protocol Lexicon format, following the patterns established by + * Bluesky's own lexicons (e.g., app.bsky.richtext.facet). + * + * The key transformation is flattening OXA's recursive inline content + * tree (Text, Emphasis, Strong, etc.) into AT Proto's flat rich text + * model (a plain text string + facets with byte-slice annotations). + * + * Output files: + * lexicon/richtext/facet.json — facet definition + inline feature types + * lexicon/document/defs.json — block-level type defs and rich text helper + * lexicon/document/document.json — the Document record type + */ + +import { mkdirSync, writeFileSync } from "fs"; +import { join } from "path"; + +import { loadMergedSchema } from "./schema.js"; + +const LEXICON_DIR = join(import.meta.dirname, "../../lexicon"); + +interface SchemaProperty { + type?: string; + const?: string; + enum?: string[]; + description?: string; + items?: { $ref?: string; type?: string }; + $ref?: string; + minimum?: number; + maximum?: number; + additionalProperties?: boolean; +} + +interface SchemaDefinition { + title: string; + description?: string; + type?: string; + anyOf?: Array<{ $ref: string }>; + properties?: Record; + required?: string[]; +} + +interface LexiconFile { + lexicon: number; + id: string; + defs: Record; +} + +export async function generateLexicon(): Promise { + const schema = loadMergedSchema(); + const definitions = schema.definitions as Record; + + // Classify types by union membership + const inlineMembers = getUnionMembers(definitions, "Inline"); + const blockMembers = getUnionMembers(definitions, "Block"); + + // Generate the three lexicon files + const facetFile = generateFacetLexicon(definitions, inlineMembers); + const defsFile = generateDefsLexicon(definitions, blockMembers); + const documentFile = generateDocumentLexicon(definitions); + + // Write files + writeLexiconFile(join(LEXICON_DIR, "richtext", "facet.json"), facetFile); + writeLexiconFile(join(LEXICON_DIR, "document", "defs.json"), defsFile); + writeLexiconFile( + join(LEXICON_DIR, "document", "document.json"), + documentFile, + ); + + console.log(`Generated lexicon files in ${LEXICON_DIR}`); +} + +function getUnionMembers( + definitions: Record, + unionName: string, +): string[] { + const def = definitions[unionName]; + if (!def?.anyOf) return []; + return def.anyOf.map((item) => item.$ref.replace("#/definitions/", "")); +} + +/** + * Generate dev.oxa.richtext.facet lexicon. + * + * Follows the Bluesky pattern: main has `index` (ref to #byteSlice) + * and `features` (array of union refs to feature types). + * Each inline type from OXA becomes a facet feature def. + * Text is excluded — it represents the plain text string itself. + */ +function generateFacetLexicon( + definitions: Record, + inlineMembers: string[], +): LexiconFile { + const defs: Record = {}; + + // Collect feature type names (excluding Text, which becomes the text string) + const featureNames = inlineMembers.filter((name) => name !== "Text"); + const featureRefs = featureNames.map((name) => `#${toCamelCase(name)}`); + + // main: the facet object with index + features, matching Bluesky's pattern + defs["main"] = { + type: "object", + description: "Annotation of a sub-string within rich text.", + required: ["index", "features"], + properties: { + index: { type: "ref", ref: "#byteSlice" }, + features: { + type: "array", + items: { type: "union", closed: false, refs: featureRefs }, + }, + }, + }; + + // byteSlice: byte-range index, matching Bluesky's pattern + defs["byteSlice"] = { + type: "object", + description: + "Specifies the sub-string range a facet feature applies to. Start index is inclusive, end index is exclusive. Indices are zero-indexed, counting bytes of the UTF-8 encoded text.", + required: ["byteStart", "byteEnd"], + properties: { + byteStart: { type: "integer", minimum: 0 }, + byteEnd: { type: "integer", minimum: 0 }, + }, + }; + + // Generate a feature def for each inline type (except Text) + for (const name of featureNames) { + const def = definitions[name]; + if (!def) continue; + + const featureDef: Record = { type: "object" }; + + if (def.description) { + featureDef.description = def.description; + } + + // Extract non-structural properties from the inline type. + // Skip: type (AT Proto uses $type), children (structural — the facet + // byte-slice replaces this), id/classes/data (preserved at block level + // but not meaningful on facet features). + const featureProps = extractNonStructuralProperties(def, true); + if (featureProps) { + const { properties, required } = featureProps; + if (Object.keys(properties).length > 0) { + featureDef.properties = properties; + if (required.length > 0) { + featureDef.required = required; + } + } + } + + defs[toCamelCase(name)] = featureDef; + } + + return { + lexicon: 1, + id: "dev.oxa.richtext.facet", + defs, + }; +} + +/** + * Generate dev.oxa.document.defs lexicon. + * + * Contains: + * - richText: reusable object with text + facets (like Bluesky's pattern) + * - One def per block type, with children: Inline[] flattened to richText + * - block: union of all block types + */ +function generateDefsLexicon( + definitions: Record, + blockMembers: string[], +): LexiconFile { + const defs: Record = {}; + + // richText: reusable def for text + facets + defs["richText"] = { + type: "object", + description: + "Rich text content: a plain text string with facet annotations.", + properties: { + text: { type: "string" }, + facets: { + type: "array", + items: { + type: "ref", + ref: "dev.oxa.richtext.facet", + }, + }, + }, + }; + + // Generate a def for each block type + for (const name of blockMembers) { + const def = definitions[name]; + if (!def) continue; + + const blockDef: Record = { type: "object" }; + + if (def.description) { + blockDef.description = def.description; + } + + const properties: Record = {}; + const required: string[] = []; + const schemaRequired = new Set(def.required || []); + + for (const [propName, prop] of Object.entries(def.properties || {})) { + // Skip 'type' — AT Proto uses $type + if (propName === "type") continue; + + // Flatten children: Inline[] → richText ref + if (propName === "children" && isInlineArray(prop)) { + properties["text"] = { type: "string" }; + properties["facets"] = { + type: "array", + items: { + type: "ref", + ref: "dev.oxa.richtext.facet", + }, + }; + continue; + } + + // Preserve id, classes, data, and other properties + const lexProp = convertPropertyToLexicon(prop); + if (lexProp) { + properties[propName] = lexProp; + if (schemaRequired.has(propName) && propName !== "type") { + required.push(propName); + } + } + } + + if (Object.keys(properties).length > 0) { + blockDef.properties = properties; + } + if (required.length > 0) { + blockDef.required = required; + } + + defs[toCamelCase(name)] = blockDef; + } + + // block: union of all block types + defs["block"] = { + type: "union", + closed: false, + refs: blockMembers.map((name) => `#${toCamelCase(name)}`), + }; + + return { + lexicon: 1, + id: "dev.oxa.document.defs", + defs, + }; +} + +/** + * Generate dev.oxa.document.document lexicon. + * + * The Document type becomes a record (matching Bluesky's app.bsky.feed.post pattern). + */ +function generateDocumentLexicon( + definitions: Record, +): LexiconFile { + const docDef = definitions["Document"]; + if (!docDef) { + throw new Error("Document type not found in schema"); + } + + const recordProperties: Record = {}; + const required: string[] = []; + const schemaRequired = new Set(docDef.required || []); + + for (const [propName, prop] of Object.entries(docDef.properties || {})) { + // Skip 'type' — AT Proto uses $type + if (propName === "type") continue; + + // Title: Inline[] → richText ref + if (propName === "title" && isInlineArray(prop)) { + recordProperties["title"] = { + type: "ref", + ref: "dev.oxa.document.defs#richText", + }; + continue; + } + + // Children: Block[] → array of block union refs + if (propName === "children" && isBlockArray(prop)) { + recordProperties["children"] = { + type: "array", + items: { + type: "ref", + ref: "dev.oxa.document.defs#block", + }, + }; + if (schemaRequired.has(propName)) { + required.push(propName); + } + continue; + } + + // Preserve other properties (id, classes, data, metadata) + const lexProp = convertPropertyToLexicon(prop); + if (lexProp) { + recordProperties[propName] = lexProp; + if (schemaRequired.has(propName) && propName !== "type") { + required.push(propName); + } + } + } + + // Add createdAt following the Bluesky record pattern + recordProperties["createdAt"] = { + type: "string", + format: "datetime", + }; + required.push("createdAt"); + + const recordObject: Record = { + type: "object", + properties: recordProperties, + }; + if (required.length > 0) { + recordObject.required = required; + } + + return { + lexicon: 1, + id: "dev.oxa.document.document", + defs: { + main: { + type: "record", + description: docDef.description, + key: "tid", + record: recordObject, + }, + }, + }; +} + +/** + * Convert an OXA JSON Schema property to a Lexicon property. + */ +function convertPropertyToLexicon( + prop: SchemaProperty, +): Record | null { + // Handle arrays + if (prop.type === "array" && prop.items) { + if (prop.items.type === "string") { + return { type: "array", items: { type: "string" } }; + } + if (prop.items.$ref) { + const refType = prop.items.$ref.replace("#/definitions/", ""); + return { + type: "array", + items: { type: "ref", ref: resolveRefToLexicon(refType) }, + }; + } + return { type: "array", items: { type: "unknown" } }; + } + + // Handle $ref + if (prop.$ref) { + const refType = prop.$ref.replace("#/definitions/", ""); + return { type: "ref", ref: resolveRefToLexicon(refType) }; + } + + // Handle basic types + switch (prop.type) { + case "string": + return { type: "string" }; + case "integer": { + const result: Record = { type: "integer" }; + if (prop.minimum !== undefined) result.minimum = prop.minimum; + if (prop.maximum !== undefined) result.maximum = prop.maximum; + return result; + } + case "number": + // ATProto Lexicon has no float type; encode as string to avoid + // silently truncating fractional values. + return { type: "string", description: "Numeric value (JSON number)" }; + case "boolean": + return { type: "boolean" }; + case "object": + return { type: "unknown" }; + default: + return { type: "unknown" }; + } +} + +/** + * Check if a property is an array of Inline refs. + */ +function isInlineArray(prop: SchemaProperty): boolean { + return prop.type === "array" && prop.items?.$ref === "#/definitions/Inline"; +} + +/** + * Check if a property is an array of Block refs. + */ +function isBlockArray(prop: SchemaProperty): boolean { + return prop.type === "array" && prop.items?.$ref === "#/definitions/Block"; +} + +/** + * Extract non-structural properties from an inline type definition. + * Skips: type, children (recursive inline content), and optionally + * common node properties (id, classes, data) for facet features. + */ +function extractNonStructuralProperties( + def: SchemaDefinition, + isFacetFeature: boolean, +): { properties: Record; required: string[] } | null { + if (!def.properties) return null; + + const skip = new Set(["type", "children"]); + if (isFacetFeature) { + // Facet features don't carry id/classes/data — these are node-level + // concerns that don't translate to byte-range annotations + skip.add("id"); + skip.add("classes"); + skip.add("data"); + } + + const schemaRequired = new Set(def.required || []); + const properties: Record = {}; + const required: string[] = []; + + for (const [propName, prop] of Object.entries(def.properties)) { + if (skip.has(propName)) continue; + + const lexProp = convertPropertyToLexicon(prop); + if (lexProp) { + properties[propName] = lexProp; + if (schemaRequired.has(propName)) { + required.push(propName); + } + } + } + + return { properties, required }; +} + +/** + * Resolve an OXA type name to a Lexicon NSID reference. + * This is a placeholder for when more types are added to the schema. + */ +function resolveRefToLexicon(typeName: string): string { + // For now, types map to document defs + return `dev.oxa.document.defs#${toCamelCase(typeName)}`; +} + +function toCamelCase(s: string): string { + return s.charAt(0).toLowerCase() + s.slice(1); +} + +function writeLexiconFile(path: string, content: LexiconFile): void { + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, JSON.stringify(content, null, 2) + "\n"); + console.log(` ${path}`); +} From ef7432c7530d1892f8f379155e7ee846a1f63d81 Mon Sep 17 00:00:00 2001 From: Nokome Bentley Date: Tue, 24 Mar 2026 12:37:45 -0600 Subject: [PATCH 2/8] chore(*): update formatted/generated files [skip ci] --- packages/oxa-core/src/cli.test.ts | 31 ++++++++++++++++++++++++++++--- packages/oxa-core/src/cli.ts | 4 +--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/packages/oxa-core/src/cli.test.ts b/packages/oxa-core/src/cli.test.ts index 7c8bbce..d0cd8e3 100644 --- a/packages/oxa-core/src/cli.test.ts +++ b/packages/oxa-core/src/cli.test.ts @@ -96,7 +96,15 @@ describe("oxa cli", () => { it("converts JSON from stdin to ATProto JSON with --created-at", async () => { const { exitCode, stdout } = await execa( "node", - [CLI_PATH, "convert", "--to", "atproto", "--created-at", createdAt, "-"], + [ + CLI_PATH, + "convert", + "--to", + "atproto", + "--created-at", + createdAt, + "-", + ], { input: JSON.stringify(convertibleDocument), }, @@ -109,7 +117,16 @@ describe("oxa cli", () => { it("converts YAML from stdin with --yaml", async () => { const { exitCode, stdout } = await execa( "node", - [CLI_PATH, "convert", "--to", "atproto", "--yaml", "--created-at", createdAt, "-"], + [ + CLI_PATH, + "convert", + "--to", + "atproto", + "--yaml", + "--created-at", + createdAt, + "-", + ], { input: convertibleYaml, }, @@ -166,7 +183,15 @@ describe("oxa cli", () => { it("exits with error for unknown --to format", async () => { const result = await execa( "node", - [CLI_PATH, "convert", "--to", "unknown", "--created-at", createdAt, "-"], + [ + CLI_PATH, + "convert", + "--to", + "unknown", + "--created-at", + createdAt, + "-", + ], { input: JSON.stringify(convertibleDocument), reject: false, diff --git a/packages/oxa-core/src/cli.ts b/packages/oxa-core/src/cli.ts index c46eefc..7d336ac 100644 --- a/packages/oxa-core/src/cli.ts +++ b/packages/oxa-core/src/cli.ts @@ -213,9 +213,7 @@ program }, ) => { try { - if ( - !SUPPORTED_FORMATS.includes(options.to as ConvertFormat) - ) { + if (!SUPPORTED_FORMATS.includes(options.to as ConvertFormat)) { throw new Error( `Unknown format: "${options.to}". Supported formats: ${SUPPORTED_FORMATS.join(", ")}`, ); From b46cc9014e06733bd1013e6b2f3040df6c47119b Mon Sep 17 00:00:00 2001 From: Nokome Bentley Date: Tue, 24 Mar 2026 12:37:45 -0600 Subject: [PATCH 3/8] refactor(lexicon): use pub.oxa NSIDs --- README.md | 2 +- docs/atproto-lexicon.md | 40 ++++++++--------- lexicon/document/defs.json | 8 ++-- lexicon/document/document.json | 6 +-- lexicon/richtext/facet.json | 2 +- packages/oxa-core/src/cli.test.ts | 4 +- packages/oxa-core/src/convert.test.ts | 62 +++++++++++++-------------- packages/oxa-core/src/convert.ts | 18 ++++---- scripts/lib/generate-lexicon.ts | 22 +++++----- 9 files changed, 82 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 50553e4..9acd140 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,7 @@ The conversion API is also exported from the package entrypoint: import { flattenInlines, mapBlock, oxaToAtproto } from "@oxa/core"; ``` -The generated ATProto records use the `dev.oxa.document.*` lexicon namespace. +The generated ATProto records use the `pub.oxa.document.*` lexicon namespace. ## Licensing and Attribution diff --git a/docs/atproto-lexicon.md b/docs/atproto-lexicon.md index c755cd5..5ae2540 100644 --- a/docs/atproto-lexicon.md +++ b/docs/atproto-lexicon.md @@ -4,16 +4,16 @@ title: AT Protocol Lexicon # AT Protocol Lexicon -OXA defines an [AT Protocol (atproto)](https://atproto.com) [Lexicon](https://atproto.com/guides/lexicon) for publishing scientific documents to the [Atmosphere](https://atproto.com/guides/understanding-atproto) — the open, decentralized social network behind [Bluesky](https://bsky.app). +OXA defines an [AT Protocol (atproto)](https://atproto.com) [Lexicon](https://atproto.com/guides/lexicon) for publishing scientific documents to the [Atmosphere](https://atproto.com/guides/understanding-atproto). -The lexicon lives under the `dev.oxa.*` namespace and enables OXA documents to be stored as records in any AT Protocol [Personal Data Server (PDS)](https://atproto.com/guides/the-at-stack), making scientific content natively available alongside social interactions, feeds, and moderation infrastructure. +The lexicon lives under the `pub.oxa.*` namespace and enables OXA documents to be stored as records in any AT Protocol [Personal Data Server (PDS)](https://atproto.com/guides/the-at-stack), making scientific content natively available alongside social interactions, feeds, and moderation infrastructure. ## Why an AT Protocol lexicon? Scientific publishing today relies on centralized platforms. Researchers upload papers to a service, and that service controls access, discovery, and permanence. AT Protocol offers a different model: - **User-owned data.** Documents live in the author's signed data repository and can be migrated between hosts. -- **Decentralized discovery.** Any indexer can crawl the network's [firehose](https://atproto.com/specs/sync) to discover and aggregate scientific content — no single gatekeeper required. +- **Decentralized discovery.** Any indexer can crawl the network's [firehose](https://atproto.com/specs/sync) to discover and aggregate scientific content. - **Interoperability by default.** The Lexicon type system gives every consumer the same schema, so tools can read, validate, and render documents without out-of-band agreements. - **Built-in identity.** Authors are identified by [DIDs](https://atproto.com/specs/did) and [handles](https://atproto.com/specs/handle), providing a ready-made, portable identity layer. @@ -25,9 +25,9 @@ The OXA lexicon is organised into two namespaces: | File | NSID | Purpose | |------|------|---------| -| `lexicon/document/document.json` | `dev.oxa.document.document` | The `Document` record type — the root object stored in a PDS | -| `lexicon/document/defs.json` | `dev.oxa.document.defs` | Block-level type definitions (`paragraph`, `heading`, `richText`) and the `block` union | -| `lexicon/richtext/facet.json` | `dev.oxa.richtext.facet` | Facet annotations for inline formatting (`emphasis`, `strong`, `byteSlice`) | +| `lexicon/document/document.json` | `pub.oxa.document.document` | The `Document` record type — the root object stored in a PDS | +| `lexicon/document/defs.json` | `pub.oxa.document.defs` | Block-level type definitions (`paragraph`, `heading`, `richText`) and the `block` union | +| `lexicon/richtext/facet.json` | `pub.oxa.richtext.facet` | Facet annotations for inline formatting (`emphasis`, `strong`, `byteSlice`) | A `Document` record contains an array of `children` (blocks). Each block carries a `text` string and an optional `facets` array that annotates ranges of that text with formatting features. @@ -35,7 +35,7 @@ A `Document` record contains an array of `children` (blocks). Each block carries The lexicon follows the conventions described in the [Lexicon Style Guide](https://atproto.com/guides/lexicon-style-guide#design-patterns): -- **Rich text via facets.** Instead of embedding markup in strings, inline formatting is represented as byte-range annotations — the same pattern established by [`app.bsky.richtext.facet`](https://docs.bsky.app/docs/advanced-guides/post-richtext). This keeps text plain and makes it safe to render even if a consumer doesn't understand a particular facet type. See [why `dev.oxa.richtext.facet`?](#why-devoxa-not-appbsky) below for why OXA defines its own facet lexicon rather than reusing Bluesky's. +- **Rich text via facets.** Instead of embedding markup in strings, inline formatting is represented as byte-range annotations — the same pattern established by [`app.bsky.richtext.facet`](https://docs.bsky.app/docs/advanced-guides/post-richtext). This keeps text plain and makes it safe to render even if a consumer doesn't understand a particular facet type. See [why `pub.oxa.richtext.facet`?](#why-devoxa-not-appbsky) below for why OXA defines its own facet lexicon rather than reusing Bluesky's. - **Open unions.** The `block` union and the facet `features` union are both declared with `"closed": false`, allowing future extension without breaking existing consumers. - **Minimal required fields.** Only fields that are truly necessary for functionality are marked `required` (e.g. `children` and `createdAt` on a document, `level` on a heading). This keeps the schema flexible for evolution. - **Singular nouns for record schemas.** The record type is named `document` (not `documents`), following the convention for record schemas. @@ -43,13 +43,13 @@ The lexicon follows the conventions described in the [Lexicon Style Guide](https - **`createdAt` timestamp.** The document record includes a `createdAt` field (datetime string), which is standard practice for ATProto records. - **`$type` discriminators.** Every block and facet feature carries a `$type` string so consumers can identify types in the open union without ambiguity. -## Why `dev.oxa.richtext.facet`? {#why-devoxa-not-appbsky} +## Why `pub.oxa.richtext.facet`? {#why-devoxa-not-appbsky} -OXA defines its own facet lexicon (`dev.oxa.richtext.facet`) rather than reusing Bluesky's `app.bsky.richtext.facet`. This is a deliberate choice driven by the different domains the two lexicons serve. +OXA defines its own facet lexicon (`pub.oxa.richtext.facet`) rather than reusing Bluesky's `app.bsky.richtext.facet`. This is a deliberate choice driven by the different domains the two lexicons serve. Bluesky's facet features are designed for social microblogging — its feature union contains `mention` (account references), `link` (URLs), and `tag` (hashtags). These are not the annotations scientific documents need. OXA documents require typographic and semantic formatting: `emphasis`, `strong`, and in the future inline types like `subscript`, `superscript`, `inlineMath`, `inlineCode`, `link`, `cite`, and others defined in the OXA schema. -There are also structural differences. The `app.bsky.richtext.facet` features union is closed, meaning third parties cannot extend it without modifying the original lexicon. The `dev.oxa.richtext.facet` features union is declared with `"closed": false`, following the style guide's recommendation for extensibility. This allows the OXA facet feature set to grow as the schema grows — and because the lexicon is generated from the OXA schema, new inline types become facet features automatically. +There are also structural differences. The `app.bsky.richtext.facet` features union is closed, meaning third parties cannot extend it without modifying the original lexicon. The `pub.oxa.richtext.facet` features union is declared with `"closed": false`, following the style guide's recommendation for extensibility. This allows the OXA facet feature set to grow as the schema grows. Because the lexicon is generated from the OXA schema, new inline types become facet features automatically. Finally, the `app.bsky` namespace is owned by Bluesky PBC. Extending it with document-formatting features would conflate social and scientific concerns in a namespace OXA does not control. @@ -86,16 +86,16 @@ AT Protocol [uses facets instead of a tree](https://www.pfrazee.com/blog/why-fac ```json { - "$type": "dev.oxa.document.defs#paragraph", + "$type": "pub.oxa.document.defs#paragraph", "text": "This is bold and italic text.", "facets": [ { "index": { "byteStart": 8, "byteEnd": 23 }, - "features": [{ "$type": "dev.oxa.richtext.facet#strong" }] + "features": [{ "$type": "pub.oxa.richtext.facet#strong" }] }, { "index": { "byteStart": 17, "byteEnd": 23 }, - "features": [{ "$type": "dev.oxa.richtext.facet#emphasis" }] + "features": [{ "$type": "pub.oxa.richtext.facet#emphasis" }] } ] } @@ -169,21 +169,21 @@ Produces: ```json { - "$type": "dev.oxa.document.document", + "$type": "pub.oxa.document.document", "children": [ { - "$type": "dev.oxa.document.defs#heading", + "$type": "pub.oxa.document.defs#heading", "level": 1, "text": "Hello", "facets": [] }, { - "$type": "dev.oxa.document.defs#paragraph", + "$type": "pub.oxa.document.defs#paragraph", "text": "Some emphasized text.", "facets": [ { "index": { "byteStart": 5, "byteEnd": 15 }, - "features": [{ "$type": "dev.oxa.richtext.facet#emphasis" }] + "features": [{ "$type": "pub.oxa.richtext.facet#emphasis" }] } ] } @@ -210,9 +210,9 @@ The lexicon files are generated from the OXA YAML schema definitions by the code 1. Loads the merged OXA JSON Schema. 2. Classifies each type as inline or block based on the `Inline` and `Block` union definitions. -3. Maps inline types to facet features in `dev.oxa.richtext.facet` (excluding `Text`, which becomes the plain text string). -4. Maps block types to object definitions in `dev.oxa.document.defs`, replacing their inline `children` arrays with `text` + `facets` pairs. -5. Emits the `Document` record type in `dev.oxa.document.document`. +3. Maps inline types to facet features in `pub.oxa.richtext.facet` (excluding `Text`, which becomes the plain text string). +4. Maps block types to object definitions in `pub.oxa.document.defs`, replacing their inline `children` arrays with `text` + `facets` pairs. +5. Emits the `Document` record type in `pub.oxa.document.document`. To regenerate the lexicon after changing the schema: diff --git a/lexicon/document/defs.json b/lexicon/document/defs.json index 48e2eba..fc7253e 100644 --- a/lexicon/document/defs.json +++ b/lexicon/document/defs.json @@ -1,6 +1,6 @@ { "lexicon": 1, - "id": "dev.oxa.document.defs", + "id": "pub.oxa.document.defs", "defs": { "richText": { "type": "object", @@ -13,7 +13,7 @@ "type": "array", "items": { "type": "ref", - "ref": "dev.oxa.richtext.facet" + "ref": "pub.oxa.richtext.facet" } } } @@ -46,7 +46,7 @@ "type": "array", "items": { "type": "ref", - "ref": "dev.oxa.richtext.facet" + "ref": "pub.oxa.richtext.facet" } } }, @@ -77,7 +77,7 @@ "type": "array", "items": { "type": "ref", - "ref": "dev.oxa.richtext.facet" + "ref": "pub.oxa.richtext.facet" } } } diff --git a/lexicon/document/document.json b/lexicon/document/document.json index 7b4d34c..b2bcac3 100644 --- a/lexicon/document/document.json +++ b/lexicon/document/document.json @@ -1,6 +1,6 @@ { "lexicon": 1, - "id": "dev.oxa.document.document", + "id": "pub.oxa.document.document", "defs": { "main": { "type": "record", @@ -26,13 +26,13 @@ }, "title": { "type": "ref", - "ref": "dev.oxa.document.defs#richText" + "ref": "pub.oxa.document.defs#richText" }, "children": { "type": "array", "items": { "type": "ref", - "ref": "dev.oxa.document.defs#block" + "ref": "pub.oxa.document.defs#block" } }, "createdAt": { diff --git a/lexicon/richtext/facet.json b/lexicon/richtext/facet.json index 5546919..b9d236a 100644 --- a/lexicon/richtext/facet.json +++ b/lexicon/richtext/facet.json @@ -1,6 +1,6 @@ { "lexicon": 1, - "id": "dev.oxa.richtext.facet", + "id": "pub.oxa.richtext.facet", "defs": { "main": { "type": "object", diff --git a/packages/oxa-core/src/cli.test.ts b/packages/oxa-core/src/cli.test.ts index d0cd8e3..b1b5f58 100644 --- a/packages/oxa-core/src/cli.test.ts +++ b/packages/oxa-core/src/cli.test.ts @@ -77,7 +77,7 @@ describe("oxa cli", () => { describe("convert", () => { const createdAt = "2026-03-22T00:00:00.000Z"; const expectedConverted = { - $type: "dev.oxa.document.document", + $type: "pub.oxa.document.document", title: { text: "CLI Example", facets: [], @@ -85,7 +85,7 @@ describe("oxa cli", () => { metadata: { license: "CC-BY-4.0" }, children: [ { - $type: "dev.oxa.document.defs#paragraph", + $type: "pub.oxa.document.defs#paragraph", text: "Hello from CLI", facets: [], }, diff --git a/packages/oxa-core/src/convert.test.ts b/packages/oxa-core/src/convert.test.ts index 6911b68..8c0c651 100644 --- a/packages/oxa-core/src/convert.test.ts +++ b/packages/oxa-core/src/convert.test.ts @@ -8,15 +8,15 @@ const REPO_ROOT = resolve(__dirname, "../../.."); const lexiconFiles = { facet: { - id: "dev.oxa.richtext.facet", + id: "pub.oxa.richtext.facet", path: resolve(REPO_ROOT, "lexicon/richtext/facet.json"), }, defs: { - id: "dev.oxa.document.defs", + id: "pub.oxa.document.defs", path: resolve(REPO_ROOT, "lexicon/document/defs.json"), }, document: { - id: "dev.oxa.document.document", + id: "pub.oxa.document.document", path: resolve(REPO_ROOT, "lexicon/document/document.json"), }, } as const; @@ -299,7 +299,7 @@ describe("ATProto lexicon structure", () => { expectFragments(defs, requiredDocumentDefs, "Missing defs fragment"); expect(defs.richText.properties.facets.items.ref).toBe( - "dev.oxa.richtext.facet", + "pub.oxa.richtext.facet", ); expect(defs.block.type).toBe("union"); expect(defs.block.closed).toBe(false); @@ -313,9 +313,9 @@ describe("ATProto lexicon structure", () => { expect(main.type).toBe("record"); expect(main.key).toBe("tid"); expect(record.required).toEqual(["children", "createdAt"]); - expect(record.properties.title.ref).toBe("dev.oxa.document.defs#richText"); + expect(record.properties.title.ref).toBe("pub.oxa.document.defs#richText"); expect(record.properties.children.items.ref).toBe( - "dev.oxa.document.defs#block", + "pub.oxa.document.defs#block", ); expect(record.properties.createdAt).toEqual({ type: "string", @@ -351,7 +351,7 @@ describe("mapBlock", () => { ); await expect(map(block)).resolves.toEqual({ - $type: "dev.oxa.document.defs#paragraph", + $type: "pub.oxa.document.defs#paragraph", id: "para-1", classes: ["lead"], data: { align: "left" }, @@ -359,7 +359,7 @@ describe("mapBlock", () => { facets: [ { index: { byteStart: 6, byteEnd: 10 }, - features: [{ $type: "dev.oxa.richtext.facet#strong" }], + features: [{ $type: "pub.oxa.richtext.facet#strong" }], }, ], }); @@ -373,7 +373,7 @@ describe("mapBlock", () => { }); await expect(map(block)).resolves.toEqual({ - $type: "dev.oxa.document.defs#heading", + $type: "pub.oxa.document.defs#heading", id: "intro", classes: ["hero"], data: { section: true }, @@ -382,7 +382,7 @@ describe("mapBlock", () => { facets: [ { index: { byteStart: 5, byteEnd: 9 }, - features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + features: [{ $type: "pub.oxa.richtext.facet#emphasis" }], }, ], }); @@ -439,7 +439,7 @@ describe("oxaToAtproto", () => { ); await expect(convertDocument(document, { createdAt })).resolves.toEqual({ - $type: "dev.oxa.document.document", + $type: "pub.oxa.document.document", title: { text: "Hello, World", facets: [], @@ -450,22 +450,22 @@ describe("oxaToAtproto", () => { }, children: [ { - $type: "dev.oxa.document.defs#heading", + $type: "pub.oxa.document.defs#heading", level: 1, text: "Introduction", facets: [], }, { - $type: "dev.oxa.document.defs#paragraph", + $type: "pub.oxa.document.defs#paragraph", text: "This is bold and italic text.", facets: [ { index: { byteStart: 8, byteEnd: 12 }, - features: [{ $type: "dev.oxa.richtext.facet#strong" }], + features: [{ $type: "pub.oxa.richtext.facet#strong" }], }, { index: { byteStart: 17, byteEnd: 23 }, - features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + features: [{ $type: "pub.oxa.richtext.facet#emphasis" }], }, ], }, @@ -480,7 +480,7 @@ describe("oxaToAtproto", () => { await expect( convertDocument(documentNode([]), { createdAt }), ).resolves.toEqual({ - $type: "dev.oxa.document.document", + $type: "pub.oxa.document.document", children: [], createdAt, }); @@ -516,11 +516,11 @@ describe("oxaToAtproto", () => { ); expect(converted).toEqual({ - $type: "dev.oxa.document.document", + $type: "pub.oxa.document.document", metadata, children: [ { - $type: "dev.oxa.document.defs#paragraph", + $type: "pub.oxa.document.defs#paragraph", text: "Keep this paragraph", facets: [], }, @@ -564,7 +564,7 @@ describe("flattenInlines", () => { facets: [ { index: { byteStart: 6, byteEnd: 11 }, - features: [{ $type: "dev.oxa.richtext.facet#strong" }], + features: [{ $type: "pub.oxa.richtext.facet#strong" }], }, ], }); @@ -584,11 +584,11 @@ describe("flattenInlines", () => { facets: [ { index: { byteStart: 8, byteEnd: 12 }, - features: [{ $type: "dev.oxa.richtext.facet#strong" }], + features: [{ $type: "pub.oxa.richtext.facet#strong" }], }, { index: { byteStart: 17, byteEnd: 23 }, - features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + features: [{ $type: "pub.oxa.richtext.facet#emphasis" }], }, ], }); @@ -602,7 +602,7 @@ describe("flattenInlines", () => { facets: [ { index: { byteStart: 4, byteEnd: 9 }, - features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + features: [{ $type: "pub.oxa.richtext.facet#emphasis" }], }, ], }); @@ -621,11 +621,11 @@ describe("flattenInlines", () => { facets: [ { index: { byteStart: 6, byteEnd: 15 }, - features: [{ $type: "dev.oxa.richtext.facet#strong" }], + features: [{ $type: "pub.oxa.richtext.facet#strong" }], }, { index: { byteStart: 20, byteEnd: 32 }, - features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + features: [{ $type: "pub.oxa.richtext.facet#emphasis" }], }, ], }); @@ -649,11 +649,11 @@ describe("flattenInlines", () => { expect.arrayContaining([ { index: { byteStart: 0, byteEnd: 20 }, - features: [{ $type: "dev.oxa.richtext.facet#strong" }], + features: [{ $type: "pub.oxa.richtext.facet#strong" }], }, { index: { byteStart: 9, byteEnd: 20 }, - features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + features: [{ $type: "pub.oxa.richtext.facet#emphasis" }], }, ]), ); @@ -667,7 +667,7 @@ describe("flattenInlines", () => { facets: [ { index: { byteStart: 0, byteEnd: 8 }, - features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + features: [{ $type: "pub.oxa.richtext.facet#emphasis" }], }, ], }); @@ -687,15 +687,15 @@ describe("flattenInlines", () => { expect.arrayContaining([ { index: { byteStart: 0, byteEnd: 14 }, - features: [{ $type: "dev.oxa.richtext.facet#strong" }], + features: [{ $type: "pub.oxa.richtext.facet#strong" }], }, { index: { byteStart: 5, byteEnd: 14 }, - features: [{ $type: "dev.oxa.richtext.facet#emphasis" }], + features: [{ $type: "pub.oxa.richtext.facet#emphasis" }], }, { index: { byteStart: 10, byteEnd: 14 }, - features: [{ $type: "dev.oxa.richtext.facet#strong" }], + features: [{ $type: "pub.oxa.richtext.facet#strong" }], }, ]), ); @@ -720,7 +720,7 @@ describe("flattenInlines", () => { facets: [ { index: { byteStart: 0, byteEnd: 6 }, - features: [{ $type: "dev.oxa.richtext.facet#strong" }], + features: [{ $type: "pub.oxa.richtext.facet#strong" }], }, ], }); diff --git a/packages/oxa-core/src/convert.ts b/packages/oxa-core/src/convert.ts index 13a4a37..f729e4e 100644 --- a/packages/oxa-core/src/convert.ts +++ b/packages/oxa-core/src/convert.ts @@ -49,7 +49,7 @@ export type DocumentNode = { }; interface FacetFeature { - $type: "dev.oxa.richtext.facet#strong" | "dev.oxa.richtext.facet#emphasis"; + $type: "pub.oxa.richtext.facet#strong" | "pub.oxa.richtext.facet#emphasis"; } interface Facet { @@ -67,19 +67,19 @@ interface RichText { type AtprotoParagraph = RichText & BlockNodeBase & { - $type: "dev.oxa.document.defs#paragraph"; + $type: "pub.oxa.document.defs#paragraph"; }; type AtprotoHeading = RichText & BlockNodeBase & { - $type: "dev.oxa.document.defs#heading"; + $type: "pub.oxa.document.defs#heading"; level: number; }; type AtprotoBlock = AtprotoParagraph | AtprotoHeading; type AtprotoDocument = { - $type: "dev.oxa.document.document"; + $type: "pub.oxa.document.document"; title?: RichText; metadata?: Record; children: AtprotoBlock[]; @@ -95,14 +95,14 @@ type BlockPropertyName = keyof BlockNodeBase; type KnownBlockNode = ParagraphNode | HeadingNode; const facetFeatureTypes = { - Strong: "dev.oxa.richtext.facet#strong", - Emphasis: "dev.oxa.richtext.facet#emphasis", + Strong: "pub.oxa.richtext.facet#strong", + Emphasis: "pub.oxa.richtext.facet#emphasis", } as const; const formattingPropertyNames = ["id", "classes", "data"] as const; const blockPropertyNames = ["id", "classes", "data"] as const; -const paragraphType = "dev.oxa.document.defs#paragraph" as const; -const headingType = "dev.oxa.document.defs#heading" as const; +const paragraphType = "pub.oxa.document.defs#paragraph" as const; +const headingType = "pub.oxa.document.defs#heading" as const; const encoder = new TextEncoder(); @@ -269,7 +269,7 @@ export function oxaToAtproto( options: OxaToAtprotoOptions = {}, ): AtprotoDocument { return { - $type: "dev.oxa.document.document", + $type: "pub.oxa.document.document", ...getOptionalDocumentFields(document), children: mapKnownBlocks(document.children), createdAt: options.createdAt ?? new Date().toISOString(), diff --git a/scripts/lib/generate-lexicon.ts b/scripts/lib/generate-lexicon.ts index 51f947d..0e255d5 100644 --- a/scripts/lib/generate-lexicon.ts +++ b/scripts/lib/generate-lexicon.ts @@ -83,7 +83,7 @@ function getUnionMembers( } /** - * Generate dev.oxa.richtext.facet lexicon. + * Generate pub.oxa.richtext.facet lexicon. * * Follows the Bluesky pattern: main has `index` (ref to #byteSlice) * and `features` (array of union refs to feature types). @@ -157,13 +157,13 @@ function generateFacetLexicon( return { lexicon: 1, - id: "dev.oxa.richtext.facet", + id: "pub.oxa.richtext.facet", defs, }; } /** - * Generate dev.oxa.document.defs lexicon. + * Generate pub.oxa.document.defs lexicon. * * Contains: * - richText: reusable object with text + facets (like Bluesky's pattern) @@ -187,7 +187,7 @@ function generateDefsLexicon( type: "array", items: { type: "ref", - ref: "dev.oxa.richtext.facet", + ref: "pub.oxa.richtext.facet", }, }, }, @@ -219,7 +219,7 @@ function generateDefsLexicon( type: "array", items: { type: "ref", - ref: "dev.oxa.richtext.facet", + ref: "pub.oxa.richtext.facet", }, }; continue; @@ -254,13 +254,13 @@ function generateDefsLexicon( return { lexicon: 1, - id: "dev.oxa.document.defs", + id: "pub.oxa.document.defs", defs, }; } /** - * Generate dev.oxa.document.document lexicon. + * Generate pub.oxa.document.document lexicon. * * The Document type becomes a record (matching Bluesky's app.bsky.feed.post pattern). */ @@ -284,7 +284,7 @@ function generateDocumentLexicon( if (propName === "title" && isInlineArray(prop)) { recordProperties["title"] = { type: "ref", - ref: "dev.oxa.document.defs#richText", + ref: "pub.oxa.document.defs#richText", }; continue; } @@ -295,7 +295,7 @@ function generateDocumentLexicon( type: "array", items: { type: "ref", - ref: "dev.oxa.document.defs#block", + ref: "pub.oxa.document.defs#block", }, }; if (schemaRequired.has(propName)) { @@ -331,7 +331,7 @@ function generateDocumentLexicon( return { lexicon: 1, - id: "dev.oxa.document.document", + id: "pub.oxa.document.document", defs: { main: { type: "record", @@ -452,7 +452,7 @@ function extractNonStructuralProperties( */ function resolveRefToLexicon(typeName: string): string { // For now, types map to document defs - return `dev.oxa.document.defs#${toCamelCase(typeName)}`; + return `pub.oxa.document.defs#${toCamelCase(typeName)}`; } function toCamelCase(s: string): string { From 06e342200304cf0905f927bab503e19ab19ae7fa Mon Sep 17 00:00:00 2001 From: Nokome Bentley Date: Tue, 24 Mar 2026 12:37:45 -0600 Subject: [PATCH 4/8] feat(lexicon): add notion of facet feature compatability --- docs/atproto-lexicon.md | 33 +++++++++++++++- packages/oxa-core/src/convert.test.ts | 55 +++++++++++++++++++++++++++ packages/oxa-core/src/convert.ts | 44 ++++++++++++++++++++- packages/oxa-core/src/index.test.ts | 1 + packages/oxa-core/src/index.ts | 7 +++- 5 files changed, 136 insertions(+), 4 deletions(-) diff --git a/docs/atproto-lexicon.md b/docs/atproto-lexicon.md index 5ae2540..97f9db4 100644 --- a/docs/atproto-lexicon.md +++ b/docs/atproto-lexicon.md @@ -55,6 +55,26 @@ Finally, the `app.bsky` namespace is owned by Bluesky PBC. Extending it with doc > **Note:** As the AT Protocol ecosystem matures, it is possible that a shared, domain-neutral rich text facet standard will emerge. If that happens, OXA will revisit this decision and consider adopting or aligning with such a standard to maximise interoperability across applications. +### Compatible features from other namespaces + +Where an OXA facet feature is semantically equivalent to a feature in another AT Protocol namespace, the converter emits both features in the same facet's `features` array. This gives consumers that understand the other namespace free interoperability without OXA depending on that namespace for its core schema. + +For example, when `Link` is added to the OXA schema, a link facet will carry both the OXA feature and Bluesky's `app.bsky.richtext.facet#link`: + +```json +{ + "index": { "byteStart": 10, "byteEnd": 20 }, + "features": [ + { "$type": "pub.oxa.richtext.facet#link", "uri": "https://example.com" }, + { "$type": "app.bsky.richtext.facet#link", "uri": "https://example.com" } + ] +} +``` + +This works because AT Protocol facets support multiple features per byte range, and consumers ignore feature types they don't recognise. A Bluesky client rendering an OXA document record will make links clickable even though it doesn't understand `pub.oxa.richtext.facet#emphasis`. + +The mapping is maintained in the `compatibleFeatures` export from `@oxa/core`. It is a record keyed by OXA facet feature `$type`, where each value is an array of functions that produce a compatible feature object (or `null` to skip). This design is not Bluesky-specific — any AT Protocol namespace can be added to the map. + ## Flattening inlines into facets The most significant transformation between an OXA document and its lexicon representation is how inline content is handled. @@ -197,13 +217,24 @@ Produces: The conversion functions are also exported from the `@oxa/core` package: ```typescript -import { flattenInlines, mapBlock, oxaToAtproto } from "@oxa/core"; +import { flattenInlines, mapBlock, oxaToAtproto, compatibleFeatures } from "@oxa/core"; const atprotoRecord = oxaToAtproto(oxaDocument, { createdAt: "2026-01-01T00:00:00.000Z", }); ``` +The `compatibleFeatures` export is a mutable record that controls which additional facet features from other AT Protocol namespaces are emitted alongside OXA features (see [compatible features](#compatible-features-from-other-namespaces) above). You can add or remove entries to customise interoperability: + +```typescript +import { compatibleFeatures } from "@oxa/core"; + +// Emit a hypothetical shared-namespace feature alongside OXA links +compatibleFeatures["pub.oxa.richtext.facet#link"] = [ + (node) => ({ $type: "org.example.richtext.facet#link", uri: node.uri as string }), +]; +``` + ## Generated from the OXA schema The lexicon files are generated from the OXA YAML schema definitions by the codegen script (`scripts/lib/generate-lexicon.ts`). The generator: diff --git a/packages/oxa-core/src/convert.test.ts b/packages/oxa-core/src/convert.test.ts index 8c0c651..cabd499 100644 --- a/packages/oxa-core/src/convert.test.ts +++ b/packages/oxa-core/src/convert.test.ts @@ -673,6 +673,61 @@ describe("flattenInlines", () => { }); }); + it("emits compatible features from other namespaces alongside OXA features", async () => { + const { compatibleFeatures } = await import("./convert.js"); + const saved = { ...compatibleFeatures }; + + compatibleFeatures["pub.oxa.richtext.facet#strong"] = [ + () => ({ $type: "com.example.richtext.facet#bold" }), + ]; + + try { + const richText = await flatten([text("a "), strong([text("b")])]); + + expect(richText).toEqual({ + text: "a b", + facets: [ + { + index: { byteStart: 2, byteEnd: 3 }, + features: [ + { $type: "pub.oxa.richtext.facet#strong" }, + { $type: "com.example.richtext.facet#bold" }, + ], + }, + ], + }); + } finally { + for (const key of Object.keys(compatibleFeatures)) { + delete compatibleFeatures[key]; + } + Object.assign(compatibleFeatures, saved); + } + }); + + it("skips compatible features that return null", async () => { + const { compatibleFeatures } = await import("./convert.js"); + const saved = { ...compatibleFeatures }; + + compatibleFeatures["pub.oxa.richtext.facet#emphasis"] = [ + () => null, + () => ({ $type: "com.example.richtext.facet#italic" }), + ]; + + try { + const richText = await flatten([emphasis([text("x")])]); + + expect(richText.facets[0].features).toEqual([ + { $type: "pub.oxa.richtext.facet#emphasis" }, + { $type: "com.example.richtext.facet#italic" }, + ]); + } finally { + for (const key of Object.keys(compatibleFeatures)) { + delete compatibleFeatures[key]; + } + Object.assign(compatibleFeatures, saved); + } + }); + it("handles deeply nested formatting with overlapping facet ranges", async () => { const richText = await flatten([ strong([ diff --git a/packages/oxa-core/src/convert.ts b/packages/oxa-core/src/convert.ts index f729e4e..cbaef1f 100644 --- a/packages/oxa-core/src/convert.ts +++ b/packages/oxa-core/src/convert.ts @@ -49,7 +49,8 @@ export type DocumentNode = { }; interface FacetFeature { - $type: "pub.oxa.richtext.facet#strong" | "pub.oxa.richtext.facet#emphasis"; + $type: string; + [key: string]: unknown; } interface Facet { @@ -99,6 +100,32 @@ const facetFeatureTypes = { Emphasis: "pub.oxa.richtext.facet#emphasis", } as const; +/** + * Compatible facet features from other AT Protocol namespaces. + * + * When an OXA facet feature has a semantically equivalent type in another + * namespace (e.g. Bluesky's `app.bsky.richtext.facet`), the converter emits + * both features in the same facet. This gives consumers that understand the + * other namespace free interoperability without OXA depending on that + * namespace for its core schema. + * + * Each key is an OXA facet feature `$type`. The value is an array of + * functions that receive the OXA inline node and return a compatible + * feature object (or `null` to skip). + * + * Currently empty — the only OXA facet features (`strong`, `emphasis`) have + * no Bluesky equivalents. When `Link` is added to the OXA schema, an entry + * like the following would provide Bluesky link interop: + * + * "pub.oxa.richtext.facet#link": [ + * (node) => ({ $type: "app.bsky.richtext.facet#link", uri: node.uri }), + * ], + */ +export const compatibleFeatures: Record< + string, + Array<(node: Record) => FacetFeature | null> +> = {}; + const formattingPropertyNames = ["id", "classes", "data"] as const; const blockPropertyNames = ["id", "classes", "data"] as const; const paragraphType = "pub.oxa.document.defs#paragraph" as const; @@ -119,9 +146,22 @@ function createFacet( byteStart: number, byteEnd: number, ): Facet { + const oxaType = facetFeatureTypes[node.type]; + const features: FacetFeature[] = [{ $type: oxaType }]; + + const compat = compatibleFeatures[oxaType]; + if (compat) { + for (const toFeature of compat) { + const extra = toFeature(node as unknown as Record); + if (extra) { + features.push(extra); + } + } + } + return { index: { byteStart, byteEnd }, - features: [{ $type: facetFeatureTypes[node.type] }], + features, }; } diff --git a/packages/oxa-core/src/index.test.ts b/packages/oxa-core/src/index.test.ts index dffa472..9d72f27 100644 --- a/packages/oxa-core/src/index.test.ts +++ b/packages/oxa-core/src/index.test.ts @@ -4,6 +4,7 @@ describe("package exports", () => { it("re-exports the ATProto conversion API from the package entrypoint", async () => { const pkg = await import("./index.js"); + expect(pkg.compatibleFeatures).toBeTypeOf("object"); expect(pkg.flattenInlines).toBeTypeOf("function"); expect(pkg.mapBlock).toBeTypeOf("function"); expect(pkg.oxaToAtproto).toBeTypeOf("function"); diff --git a/packages/oxa-core/src/index.ts b/packages/oxa-core/src/index.ts index bba0f82..dec5458 100644 --- a/packages/oxa-core/src/index.ts +++ b/packages/oxa-core/src/index.ts @@ -20,7 +20,12 @@ export type * from "oxa-types"; // Export validation functions and types -export { flattenInlines, mapBlock, oxaToAtproto } from "./convert.js"; +export { + compatibleFeatures, + flattenInlines, + mapBlock, + oxaToAtproto, +} from "./convert.js"; export { validate, From 1fb8f3c2ecf194566f9ee3f1a461dc8e1b0ef49f Mon Sep 17 00:00:00 2001 From: Nokome Bentley Date: Tue, 24 Mar 2026 12:37:45 -0600 Subject: [PATCH 5/8] docs(lexicon): remove note now that have feature compatability awareness --- docs/atproto-lexicon.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/atproto-lexicon.md b/docs/atproto-lexicon.md index 97f9db4..fe55876 100644 --- a/docs/atproto-lexicon.md +++ b/docs/atproto-lexicon.md @@ -53,8 +53,6 @@ There are also structural differences. The `app.bsky.richtext.facet` features un Finally, the `app.bsky` namespace is owned by Bluesky PBC. Extending it with document-formatting features would conflate social and scientific concerns in a namespace OXA does not control. -> **Note:** As the AT Protocol ecosystem matures, it is possible that a shared, domain-neutral rich text facet standard will emerge. If that happens, OXA will revisit this decision and consider adopting or aligning with such a standard to maximise interoperability across applications. - ### Compatible features from other namespaces Where an OXA facet feature is semantically equivalent to a feature in another AT Protocol namespace, the converter emits both features in the same facet's `features` array. This gives consumers that understand the other namespace free interoperability without OXA depending on that namespace for its core schema. From b4eea9361bd1198a21478fba222764ee5d9ef959 Mon Sep 17 00:00:00 2001 From: Rowan Cockett Date: Tue, 24 Mar 2026 12:45:36 -0600 Subject: [PATCH 6/8] Additional facets --- lexicon/document/defs.json | 49 ++++++++++++++++++++++++++- lexicon/richtext/facet.json | 28 ++++++++++++++- packages/oxa-core/src/convert.test.ts | 17 ++++++++-- 3 files changed, 90 insertions(+), 4 deletions(-) diff --git a/lexicon/document/defs.json b/lexicon/document/defs.json index fc7253e..904c586 100644 --- a/lexicon/document/defs.json +++ b/lexicon/document/defs.json @@ -18,6 +18,33 @@ } } }, + "code": { + "type": "object", + "description": "A block of preformatted text, typically source code.", + "properties": { + "id": { + "type": "string" + }, + "classes": { + "type": "array", + "items": { + "type": "string" + } + }, + "data": { + "type": "unknown" + }, + "language": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "value" + ] + }, "heading": { "type": "object", "description": "A heading with a level and inline content.", @@ -82,12 +109,32 @@ } } }, + "thematicBreak": { + "type": "object", + "description": "A thematic or structural division between sections of content.", + "properties": { + "id": { + "type": "string" + }, + "classes": { + "type": "array", + "items": { + "type": "string" + } + }, + "data": { + "type": "unknown" + } + } + }, "block": { "type": "union", "closed": false, "refs": [ + "#code", "#heading", - "#paragraph" + "#paragraph", + "#thematicBreak" ] } } diff --git a/lexicon/richtext/facet.json b/lexicon/richtext/facet.json index b9d236a..5c47fb6 100644 --- a/lexicon/richtext/facet.json +++ b/lexicon/richtext/facet.json @@ -21,7 +21,10 @@ "closed": false, "refs": [ "#emphasis", - "#strong" + "#inlineCode", + "#strong", + "#subscript", + "#superscript" ] } } @@ -49,9 +52,32 @@ "type": "object", "description": "Emphasized content (typically italicized)." }, + "inlineCode": { + "type": "object", + "description": "Short fragments of code appearing within prose.", + "properties": { + "language": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "value" + ] + }, "strong": { "type": "object", "description": "Strongly emphasized content (typically bold)." + }, + "subscript": { + "type": "object", + "description": "Content rendered below the baseline (e.g. chemical formulae, variable indices)." + }, + "superscript": { + "type": "object", + "description": "Content rendered above the baseline (e.g. exponents, ordinal suffixes)." } } } diff --git a/packages/oxa-core/src/convert.test.ts b/packages/oxa-core/src/convert.test.ts index cabd499..a1916d7 100644 --- a/packages/oxa-core/src/convert.test.ts +++ b/packages/oxa-core/src/convert.test.ts @@ -21,14 +21,27 @@ const lexiconFiles = { }, } as const; -const facetFragments = ["emphasis", "strong"] as const; +const facetFragments = [ + "emphasis", + "inlineCode", + "strong", + "subscript", + "superscript", +] as const; -const documentBlockRefs = ["#heading", "#paragraph"] as const; +const documentBlockRefs = [ + "#code", + "#heading", + "#paragraph", + "#thematicBreak", +] as const; const requiredDocumentDefs = [ "richText", + "code", "paragraph", "heading", + "thematicBreak", "block", ] as const; From 2213740c90886ede50029b5756a3562e0d4776e6 Mon Sep 17 00:00:00 2001 From: Rowan Cockett Date: Tue, 24 Mar 2026 13:18:45 -0600 Subject: [PATCH 7/8] Updates to add inline code and nodes from rfc0003. --- docs/atproto-lexicon.md | 36 +++++--- examples/rfc0003.atproto.json | 152 +++++++++++++++++++++++++++++++ examples/rfc0003.json | 135 +++++++++++++++++++++++++++ packages/oxa-core/src/convert.ts | 106 +++++++++++++++++++-- 4 files changed, 407 insertions(+), 22 deletions(-) create mode 100644 examples/rfc0003.atproto.json create mode 100644 examples/rfc0003.json diff --git a/docs/atproto-lexicon.md b/docs/atproto-lexicon.md index fe55876..18af741 100644 --- a/docs/atproto-lexicon.md +++ b/docs/atproto-lexicon.md @@ -2,8 +2,6 @@ title: AT Protocol Lexicon --- -# AT Protocol Lexicon - OXA defines an [AT Protocol (atproto)](https://atproto.com) [Lexicon](https://atproto.com/guides/lexicon) for publishing scientific documents to the [Atmosphere](https://atproto.com/guides/understanding-atproto). The lexicon lives under the `pub.oxa.*` namespace and enables OXA documents to be stored as records in any AT Protocol [Personal Data Server (PDS)](https://atproto.com/guides/the-at-stack), making scientific content natively available alongside social interactions, feeds, and moderation infrastructure. @@ -21,13 +19,13 @@ By defining a lexicon, OXA documents become first-class objects on the AT Protoc ## Lexicon structure -The OXA lexicon is organised into two namespaces: +The OXA lexicon is organized into two namespaces: -| File | NSID | Purpose | -|------|------|---------| -| `lexicon/document/document.json` | `pub.oxa.document.document` | The `Document` record type — the root object stored in a PDS | -| `lexicon/document/defs.json` | `pub.oxa.document.defs` | Block-level type definitions (`paragraph`, `heading`, `richText`) and the `block` union | -| `lexicon/richtext/facet.json` | `pub.oxa.richtext.facet` | Facet annotations for inline formatting (`emphasis`, `strong`, `byteSlice`) | +| File | NSID | Purpose | +| -------------------------------- | --------------------------- | --------------------------------------------------------------------------------------- | +| `lexicon/document/document.json` | `pub.oxa.document.document` | The `Document` record type — the root object stored in a PDS | +| `lexicon/document/defs.json` | `pub.oxa.document.defs` | Block-level type definitions (`paragraph`, `heading`, `richText`) and the `block` union | +| `lexicon/richtext/facet.json` | `pub.oxa.richtext.facet` | Facet annotations for inline formatting (`emphasis`, `strong`, `byteSlice`) | A `Document` record contains an array of `children` (blocks). Each block carries a `text` string and an optional `facets` array that annotates ranges of that text with formatting features. @@ -35,7 +33,7 @@ A `Document` record contains an array of `children` (blocks). Each block carries The lexicon follows the conventions described in the [Lexicon Style Guide](https://atproto.com/guides/lexicon-style-guide#design-patterns): -- **Rich text via facets.** Instead of embedding markup in strings, inline formatting is represented as byte-range annotations — the same pattern established by [`app.bsky.richtext.facet`](https://docs.bsky.app/docs/advanced-guides/post-richtext). This keeps text plain and makes it safe to render even if a consumer doesn't understand a particular facet type. See [why `pub.oxa.richtext.facet`?](#why-devoxa-not-appbsky) below for why OXA defines its own facet lexicon rather than reusing Bluesky's. +- **Rich text via facets.** Instead of embedding markup in strings, inline formatting is represented as byte-range annotations — the same pattern established by [`app.bsky.richtext.facet`](https://docs.bsky.app/docs/advanced-guides/post-richtext). This keeps text plain and makes it safe to render even if a consumer doesn't understand a particular facet type. See [why `pub.oxa.richtext.facet`?](#why-puboxa-not-appbsky) below for why OXA defines its own facet lexicon rather than reusing Bluesky's. - **Open unions.** The `block` union and the facet `features` union are both declared with `"closed": false`, allowing future extension without breaking existing consumers. - **Minimal required fields.** Only fields that are truly necessary for functionality are marked `required` (e.g. `children` and `createdAt` on a document, `level` on a heading). This keeps the schema flexible for evolution. - **Singular nouns for record schemas.** The record type is named `document` (not `documents`), following the convention for record schemas. @@ -43,7 +41,9 @@ The lexicon follows the conventions described in the [Lexicon Style Guide](https - **`createdAt` timestamp.** The document record includes a `createdAt` field (datetime string), which is standard practice for ATProto records. - **`$type` discriminators.** Every block and facet feature carries a `$type` string so consumers can identify types in the open union without ambiguity. -## Why `pub.oxa.richtext.facet`? {#why-devoxa-not-appbsky} +(why-puboxa-not-appbsky)= + +## Why `pub.oxa.richtext.facet`? OXA defines its own facet lexicon (`pub.oxa.richtext.facet`) rather than reusing Bluesky's `app.bsky.richtext.facet`. This is a deliberate choice driven by the different domains the two lexicons serve. @@ -123,7 +123,7 @@ The conversion walks the inline tree depth-first, concatenating all `Text` node This design has several advantages in a decentralized setting: -- **Safe rendering.** A consumer that doesn't recognise a facet type can still display the plain text. +- **Safe rendering.** A consumer that doesn't recognize a facet type can still display the plain text. - **Simple validation.** Facets are flat — there is no recursive nesting to validate. - **Extensibility.** New facet feature types (links, mentions, math, etc.) can be added to the open union without changing the text representation. @@ -215,21 +215,29 @@ Produces: The conversion functions are also exported from the `@oxa/core` package: ```typescript -import { flattenInlines, mapBlock, oxaToAtproto, compatibleFeatures } from "@oxa/core"; +import { + flattenInlines, + mapBlock, + oxaToAtproto, + compatibleFeatures, +} from "@oxa/core"; const atprotoRecord = oxaToAtproto(oxaDocument, { createdAt: "2026-01-01T00:00:00.000Z", }); ``` -The `compatibleFeatures` export is a mutable record that controls which additional facet features from other AT Protocol namespaces are emitted alongside OXA features (see [compatible features](#compatible-features-from-other-namespaces) above). You can add or remove entries to customise interoperability: +The `compatibleFeatures` export is a mutable record that controls which additional facet features from other AT Protocol namespaces are emitted alongside OXA features (see [compatible features](#compatible-features-from-other-namespaces) above). You can add or remove entries to customize interoperability: ```typescript import { compatibleFeatures } from "@oxa/core"; // Emit a hypothetical shared-namespace feature alongside OXA links compatibleFeatures["pub.oxa.richtext.facet#link"] = [ - (node) => ({ $type: "org.example.richtext.facet#link", uri: node.uri as string }), + (node) => ({ + $type: "org.example.richtext.facet#link", + uri: node.uri as string, + }), ]; ``` diff --git a/examples/rfc0003.atproto.json b/examples/rfc0003.atproto.json new file mode 100644 index 0000000..e62d593 --- /dev/null +++ b/examples/rfc0003.atproto.json @@ -0,0 +1,152 @@ +{ + "$type": "pub.oxa.document.document", + "title": { + "text": "Water Dissociation: H2O → H+ + OH−", + "facets": [ + { + "index": { "byteStart": 21, "byteEnd": 22 }, + "features": [{ "$type": "pub.oxa.richtext.facet#subscript" }] + }, + { + "index": { "byteStart": 29, "byteEnd": 30 }, + "features": [{ "$type": "pub.oxa.richtext.facet#superscript" }] + }, + { + "index": { "byteStart": 35, "byteEnd": 38 }, + "features": [{ "$type": "pub.oxa.richtext.facet#superscript" }] + } + ] + }, + "children": [ + { + "$type": "pub.oxa.document.defs#heading", + "text": "Introduction", + "facets": [], + "level": 1 + }, + { + "$type": "pub.oxa.document.defs#paragraph", + "text": "Water (H2O) undergoes autoionization, a process in which a water molecule donates a proton to another. The equilibrium constant for this reaction, Kw, is approximately 10−14 at 25 °C.", + "facets": [ + { + "index": { "byteStart": 8, "byteEnd": 9 }, + "features": [{ "$type": "pub.oxa.richtext.facet#subscript" }] + }, + { + "index": { "byteStart": 22, "byteEnd": 36 }, + "features": [{ "$type": "pub.oxa.richtext.facet#strong" }] + }, + { + "index": { "byteStart": 148, "byteEnd": 149 }, + "features": [{ "$type": "pub.oxa.richtext.facet#subscript" }] + }, + { + "index": { "byteStart": 147, "byteEnd": 149 }, + "features": [{ "$type": "pub.oxa.richtext.facet#emphasis" }] + }, + { + "index": { "byteStart": 170, "byteEnd": 175 }, + "features": [{ "$type": "pub.oxa.richtext.facet#superscript" }] + } + ] + }, + { "$type": "pub.oxa.document.defs#thematicBreak" }, + { + "$type": "pub.oxa.document.defs#heading", + "text": "Computing the Equilibrium", + "facets": [], + "level": 2 + }, + { + "$type": "pub.oxa.document.defs#paragraph", + "text": "The following Python snippet computes Kw from ion concentrations:", + "facets": [ + { + "index": { "byteStart": 38, "byteEnd": 40 }, + "features": [{ "$type": "pub.oxa.richtext.facet#inlineCode" }] + } + ] + }, + { + "$type": "pub.oxa.document.defs#code", + "value": "H_plus = 1e-7 # mol/L\nOH_minus = 1e-7 # mol/L\nKw = H_plus * OH_minus\nprint(f\"Kw = {Kw:.2e}\") # Kw = 1.00e-14", + "language": "python" + }, + { + "$type": "pub.oxa.document.defs#paragraph", + "text": "You can run this with python kw.py. The result confirms the well-known value of Kw.", + "facets": [ + { + "index": { "byteStart": 22, "byteEnd": 34 }, + "features": [{ "$type": "pub.oxa.richtext.facet#inlineCode" }] + }, + { + "index": { "byteStart": 81, "byteEnd": 82 }, + "features": [{ "$type": "pub.oxa.richtext.facet#subscript" }] + }, + { + "index": { "byteStart": 80, "byteEnd": 82 }, + "features": [{ "$type": "pub.oxa.richtext.facet#emphasis" }] + } + ] + }, + { "$type": "pub.oxa.document.defs#thematicBreak" }, + { + "$type": "pub.oxa.document.defs#heading", + "text": "Summary", + "facets": [], + "level": 2 + }, + { + "$type": "pub.oxa.document.defs#paragraph", + "text": "This example demonstrates every node type from RFC0003: Heading, Paragraph, Code, ThematicBreak, Text, Emphasis, Strong, Superscript, Subscript, and InlineCode.", + "facets": [ + { + "index": { "byteStart": 47, "byteEnd": 54 }, + "features": [{ "$type": "pub.oxa.richtext.facet#strong" }] + }, + { + "index": { "byteStart": 56, "byteEnd": 63 }, + "features": [{ "$type": "pub.oxa.richtext.facet#inlineCode" }] + }, + { + "index": { "byteStart": 65, "byteEnd": 74 }, + "features": [{ "$type": "pub.oxa.richtext.facet#inlineCode" }] + }, + { + "index": { "byteStart": 76, "byteEnd": 80 }, + "features": [{ "$type": "pub.oxa.richtext.facet#inlineCode" }] + }, + { + "index": { "byteStart": 82, "byteEnd": 95 }, + "features": [{ "$type": "pub.oxa.richtext.facet#inlineCode" }] + }, + { + "index": { "byteStart": 97, "byteEnd": 101 }, + "features": [{ "$type": "pub.oxa.richtext.facet#inlineCode" }] + }, + { + "index": { "byteStart": 103, "byteEnd": 111 }, + "features": [{ "$type": "pub.oxa.richtext.facet#inlineCode" }] + }, + { + "index": { "byteStart": 113, "byteEnd": 119 }, + "features": [{ "$type": "pub.oxa.richtext.facet#inlineCode" }] + }, + { + "index": { "byteStart": 121, "byteEnd": 132 }, + "features": [{ "$type": "pub.oxa.richtext.facet#inlineCode" }] + }, + { + "index": { "byteStart": 134, "byteEnd": 143 }, + "features": [{ "$type": "pub.oxa.richtext.facet#inlineCode" }] + }, + { + "index": { "byteStart": 149, "byteEnd": 159 }, + "features": [{ "$type": "pub.oxa.richtext.facet#inlineCode" }] + } + ] + } + ], + "createdAt": "2026-03-24T19:14:02.007Z" +} diff --git a/examples/rfc0003.json b/examples/rfc0003.json new file mode 100644 index 0000000..5930007 --- /dev/null +++ b/examples/rfc0003.json @@ -0,0 +1,135 @@ +{ + "$schema": "https://oxa.dev/v0.2.0/schema.json", + "type": "Document", + "title": [ + { "type": "Text", "value": "Water Dissociation: H" }, + { "type": "Subscript", "children": [{ "type": "Text", "value": "2" }] }, + { "type": "Text", "value": "O → H" }, + { "type": "Superscript", "children": [{ "type": "Text", "value": "+" }] }, + { "type": "Text", "value": " + OH" }, + { "type": "Superscript", "children": [{ "type": "Text", "value": "−" }] } + ], + "children": [ + { + "type": "Heading", + "level": 1, + "children": [{ "type": "Text", "value": "Introduction" }] + }, + { + "type": "Paragraph", + "children": [ + { "type": "Text", "value": "Water (H" }, + { + "type": "Subscript", + "children": [{ "type": "Text", "value": "2" }] + }, + { "type": "Text", "value": "O) undergoes " }, + { + "type": "Strong", + "children": [{ "type": "Text", "value": "autoionization" }] + }, + { + "type": "Text", + "value": ", a process in which a water molecule donates a proton to another. The equilibrium constant for this reaction, " + }, + { + "type": "Emphasis", + "children": [ + { "type": "Text", "value": "K" }, + { + "type": "Subscript", + "children": [{ "type": "Text", "value": "w" }] + } + ] + }, + { "type": "Text", "value": ", is approximately 10" }, + { + "type": "Superscript", + "children": [{ "type": "Text", "value": "−14" }] + }, + { "type": "Text", "value": " at 25 °C." } + ] + }, + { "type": "ThematicBreak" }, + { + "type": "Heading", + "level": 2, + "children": [{ "type": "Text", "value": "Computing the Equilibrium" }] + }, + { + "type": "Paragraph", + "children": [ + { "type": "Text", "value": "The following Python snippet computes " }, + { "type": "InlineCode", "value": "Kw" }, + { "type": "Text", "value": " from ion concentrations:" } + ] + }, + { + "type": "Code", + "language": "python", + "value": "H_plus = 1e-7 # mol/L\nOH_minus = 1e-7 # mol/L\nKw = H_plus * OH_minus\nprint(f\"Kw = {Kw:.2e}\") # Kw = 1.00e-14" + }, + { + "type": "Paragraph", + "children": [ + { "type": "Text", "value": "You can run this with " }, + { "type": "InlineCode", "value": "python kw.py" }, + { + "type": "Text", + "value": ". The result confirms the well-known value of " + }, + { + "type": "Emphasis", + "children": [ + { "type": "Text", "value": "K" }, + { + "type": "Subscript", + "children": [{ "type": "Text", "value": "w" }] + } + ] + }, + { "type": "Text", "value": "." } + ] + }, + { "type": "ThematicBreak" }, + { + "type": "Heading", + "level": 2, + "children": [{ "type": "Text", "value": "Summary" }] + }, + { + "type": "Paragraph", + "children": [ + { + "type": "Text", + "value": "This example demonstrates every node type from " + }, + { + "type": "Strong", + "children": [{ "type": "Text", "value": "RFC0003" }] + }, + { "type": "Text", "value": ": " }, + { "type": "InlineCode", "value": "Heading" }, + { "type": "Text", "value": ", " }, + { "type": "InlineCode", "value": "Paragraph" }, + { "type": "Text", "value": ", " }, + { "type": "InlineCode", "value": "Code" }, + { "type": "Text", "value": ", " }, + { "type": "InlineCode", "value": "ThematicBreak" }, + { "type": "Text", "value": ", " }, + { "type": "InlineCode", "value": "Text" }, + { "type": "Text", "value": ", " }, + { "type": "InlineCode", "value": "Emphasis" }, + { "type": "Text", "value": ", " }, + { "type": "InlineCode", "value": "Strong" }, + { "type": "Text", "value": ", " }, + { "type": "InlineCode", "value": "Superscript" }, + { "type": "Text", "value": ", " }, + { "type": "InlineCode", "value": "Subscript" }, + { "type": "Text", "value": ", and " }, + { "type": "InlineCode", "value": "InlineCode" }, + { "type": "Text", "value": "." } + ] + } + ] +} diff --git a/packages/oxa-core/src/convert.ts b/packages/oxa-core/src/convert.ts index cbaef1f..5f6eec8 100644 --- a/packages/oxa-core/src/convert.ts +++ b/packages/oxa-core/src/convert.ts @@ -7,15 +7,24 @@ type TextNode = { value: string; }; +type InlineCodeNode = { + type: "InlineCode"; + value: string; + language?: string; + id?: string; + classes?: string[]; + data?: Record; +}; + type FormattingNode = { - type: "Strong" | "Emphasis"; + type: "Strong" | "Emphasis" | "Superscript" | "Subscript"; children: InlineNode[]; id?: string; classes?: string[]; data?: Record; }; -type InlineNode = TextNode | FormattingNode; +type InlineNode = TextNode | InlineCodeNode | FormattingNode; type BlockNodeBase = { id?: string; @@ -34,12 +43,27 @@ type HeadingNode = BlockNodeBase & { children: InlineNode[]; }; +type CodeNode = BlockNodeBase & { + type: "Code"; + value: string; + language?: string; +}; + +type ThematicBreakNode = BlockNodeBase & { + type: "ThematicBreak"; +}; + type UnknownBlockNode = BlockNodeBase & { type: string; children?: InlineNode[]; }; -type BlockNode = ParagraphNode | HeadingNode | UnknownBlockNode; +type BlockNode = + | ParagraphNode + | HeadingNode + | CodeNode + | ThematicBreakNode + | UnknownBlockNode; export type DocumentNode = { type: "Document"; @@ -77,7 +101,21 @@ type AtprotoHeading = RichText & level: number; }; -type AtprotoBlock = AtprotoParagraph | AtprotoHeading; +type AtprotoCode = BlockNodeBase & { + $type: "pub.oxa.document.defs#code"; + value: string; + language?: string; +}; + +type AtprotoThematicBreak = BlockNodeBase & { + $type: "pub.oxa.document.defs#thematicBreak"; +}; + +type AtprotoBlock = + | AtprotoParagraph + | AtprotoHeading + | AtprotoCode + | AtprotoThematicBreak; type AtprotoDocument = { $type: "pub.oxa.document.document"; @@ -93,11 +131,18 @@ type OxaToAtprotoOptions = { type FormattingPropertyName = "id" | "classes" | "data"; type BlockPropertyName = keyof BlockNodeBase; -type KnownBlockNode = ParagraphNode | HeadingNode; +type KnownBlockNode = + | ParagraphNode + | HeadingNode + | CodeNode + | ThematicBreakNode; const facetFeatureTypes = { Strong: "pub.oxa.richtext.facet#strong", Emphasis: "pub.oxa.richtext.facet#emphasis", + Superscript: "pub.oxa.richtext.facet#superscript", + Subscript: "pub.oxa.richtext.facet#subscript", + InlineCode: "pub.oxa.richtext.facet#inlineCode", } as const; /** @@ -130,6 +175,8 @@ const formattingPropertyNames = ["id", "classes", "data"] as const; const blockPropertyNames = ["id", "classes", "data"] as const; const paragraphType = "pub.oxa.document.defs#paragraph" as const; const headingType = "pub.oxa.document.defs#heading" as const; +const codeType = "pub.oxa.document.defs#code" as const; +const thematicBreakType = "pub.oxa.document.defs#thematicBreak" as const; const encoder = new TextEncoder(); @@ -142,7 +189,7 @@ function getCurrentByteOffset(richText: RichText): number { } function createFacet( - node: FormattingNode, + node: FormattingNode | InlineCodeNode, byteStart: number, byteEnd: number, ): Facet { @@ -212,6 +259,15 @@ function flattenNode(node: InlineNode, richText: RichText): void { return; } + if (node.type === "InlineCode") { + warnDroppedProperties(node as unknown as FormattingNode); + const byteStart = getCurrentByteOffset(richText); + richText.text += node.value; + const byteEnd = getCurrentByteOffset(richText); + richText.facets.push(createFacet(node, byteStart, byteEnd)); + return; + } + warnDroppedProperties(node); const byteStart = getCurrentByteOffset(richText); @@ -245,7 +301,9 @@ function copyBlockProps(block: BlockNodeBase): BlockNodeBase { ); } -function mapBlockRichText(block: KnownBlockNode): BlockNodeBase & RichText { +type RichTextBlockNode = ParagraphNode | HeadingNode; + +function mapBlockRichText(block: RichTextBlockNode): BlockNodeBase & RichText { return { ...copyBlockProps(block), ...flattenInlines(block.children), @@ -264,14 +322,46 @@ function getOptionalDocumentFields( } function isKnownBlock(block: BlockNode): block is KnownBlockNode { - return block.type === "Paragraph" || block.type === "Heading"; + return ( + block.type === "Paragraph" || + block.type === "Heading" || + block.type === "Code" || + block.type === "ThematicBreak" + ); } function warnUnknownBlockType(block: BlockNode): void { warn(`unknown block type: ${block.type}`); } +function mapCodeBlock(block: CodeNode): AtprotoCode { + const result: AtprotoCode = { + $type: codeType, + ...copyBlockProps(block), + value: block.value, + }; + if (block.language !== undefined) { + result.language = block.language; + } + return result; +} + +function mapThematicBreak(block: ThematicBreakNode): AtprotoThematicBreak { + return { + $type: thematicBreakType, + ...copyBlockProps(block), + }; +} + function mapKnownBlock(block: KnownBlockNode): AtprotoBlock { + if (block.type === "Code") { + return mapCodeBlock(block); + } + + if (block.type === "ThematicBreak") { + return mapThematicBreak(block); + } + const richTextBlock = mapBlockRichText(block); if (block.type === "Paragraph") { From d47a60410304e42892c31ef38fbda1f4c8bc7704 Mon Sep 17 00:00:00 2001 From: Rowan Cockett Date: Tue, 24 Mar 2026 13:19:34 -0600 Subject: [PATCH 8/8] changeset --- .changeset/silly-pears-joke.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/silly-pears-joke.md diff --git a/.changeset/silly-pears-joke.md b/.changeset/silly-pears-joke.md new file mode 100644 index 0000000..a0a39a7 --- /dev/null +++ b/.changeset/silly-pears-joke.md @@ -0,0 +1,5 @@ +--- +"@oxa/core": patch +--- + +Add atproto conversion scripts