From a83389d6711c202b70ee6a2d83deaff802994699 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Thu, 22 Oct 2026 18:21:31 -0700 Subject: [PATCH 1/5] feat(agent): serve a create-legacy-rule recipe per tool with --tool Adds Ruff, ESLint, Biome, pylint, and flake8 recipes, makes route name the --tool command for each detected linter, and pins the recipes' claims in an opt-in vendor contract test with its own workflow. --- .changeset/legacy-linter-recipes.md | 2 + .github/workflows/legacy-linter-contract.yml | 53 +++++ .../changes/legacy-linter-recipes/design.md | 2 +- .../changes/legacy-linter-recipes/tasks.md | 20 +- .../cli/src/agent/create-legacy-rule.biome.md | 71 +++++++ .../src/agent/create-legacy-rule.eslint.md | 109 ++++++++++ .../src/agent/create-legacy-rule.flake8.md | 67 +++++++ .../src/agent/create-legacy-rule.pylint.md | 68 +++++++ .../cli/src/agent/create-legacy-rule.ruff.md | 87 ++++++++ packages/cli/src/commands/agent.ts | 41 +++- packages/cli/src/prompts/recipes.ts | 59 ++++-- packages/cli/test/agent-extensions.test.ts | 72 +++++++ packages/cli/test/legacy-ladder.test.ts | 78 ++++++++ .../legacy-linter-vendor-contract.test.ts | 188 ++++++++++++++++++ packages/cli/test/prompts.test.ts | 17 +- .../cli/test/recipe-cross-references.test.ts | 15 +- 16 files changed, 909 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/legacy-linter-contract.yml create mode 100644 packages/cli/src/agent/create-legacy-rule.biome.md create mode 100644 packages/cli/src/agent/create-legacy-rule.eslint.md create mode 100644 packages/cli/src/agent/create-legacy-rule.flake8.md create mode 100644 packages/cli/src/agent/create-legacy-rule.pylint.md create mode 100644 packages/cli/src/agent/create-legacy-rule.ruff.md create mode 100644 packages/cli/test/legacy-linter-vendor-contract.test.ts diff --git a/.changeset/legacy-linter-recipes.md b/.changeset/legacy-linter-recipes.md index 8020d930..5f05ba20 100644 --- a/.changeset/legacy-linter-recipes.md +++ b/.changeset/legacy-linter-recipes.md @@ -5,3 +5,5 @@ Rules for a linter the repository already runs now follow a fixed order. The agent first looks for a rule the linter already ships, checking each tool's catch-all rules by name (`no-restricted-syntax`, `banned-api`, `noRestrictedImports`, and others). It writes a new custom rule in that linter only when the repository already has five or more of its own to model it on. Otherwise the request becomes a Taskless rule. `route` (topic v8) no longer asks the user to choose between a linter rule and a Taskless rule, and it describes each linter `detect` found in the repository. During onboarding (topic v11), the agent only enables rules the repository's installed packages already ship. `detect --json` reports `customRules` for ESLint, Biome, pylint, flake8, and Ruff: how many rules the repository wrote itself for that linter, and where they are. A count that cannot be read without running a JavaScript config is reported as `"unknown"`. + +`taskless agent create-legacy-rule --tool=` serves a recipe for one linter, named as `detect` reports it: `ruff`, `eslint` (including typescript-eslint), `biome`, `pylint`, or `flake8`. Each recipe names the tool's catch-all rules, says how to confirm a rule exists in the installed version (`ruff rule`, `biome explain`, `pylint --help-msg`), and covers the traps that make a rule silently not fire, such as Ruff preview rules and ESLint 10's removal of `.eslintrc`. The ESLint recipe also lists eight well-known plugins, each with a docs link. Other tool names fall back to the generic recipe. `ast-grep`, `vale`, `prettier`, and `black` are refused with a pointer to the right recipe. `route` now names the `--tool` command for each linter it describes. diff --git a/.github/workflows/legacy-linter-contract.yml b/.github/workflows/legacy-linter-contract.yml new file mode 100644 index 00000000..f6b78123 --- /dev/null +++ b/.github/workflows/legacy-linter-contract.yml @@ -0,0 +1,53 @@ +# Pins what the `create-legacy-rule` recipes say about Ruff, pylint, flake8, +# and Biome, by asking each linter at a pinned version. See +# `packages/cli/test/legacy-linter-vendor-contract.test.ts`. +# +# Separate from `Validate` on purpose: it downloads linters from PyPI and npm, +# and a registry outage must not block every merge. A red run here means a +# recipe sentence stopped being true, or a pinned version moved. +name: Legacy linter contract + +on: + # No `branches:` filter, for the reason `validate.yml` gives. + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths: + - "packages/cli/src/agent/create-legacy-rule*.md" + - "packages/cli/test/legacy-linter-vendor-contract.test.ts" + - ".github/workflows/legacy-linter-contract.yml" + workflow_dispatch: + +permissions: + contents: read + +env: + TASKLESS_TELEMETRY_DISABLED: "1" + +jobs: + contract: + name: Legacy linter contract + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - name: Setup Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .nvmrc + + - name: Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Test + working-directory: packages/cli + env: + TASKLESS_LEGACY_LINTER_CONTRACT: "1" + run: pnpm vitest run test/legacy-linter-vendor-contract.test.ts diff --git a/openspec/changes/legacy-linter-recipes/design.md b/openspec/changes/legacy-linter-recipes/design.md index 76389aee..80108d31 100644 --- a/openspec/changes/legacy-linter-recipes/design.md +++ b/openspec/changes/legacy-linter-recipes/design.md @@ -40,7 +40,7 @@ Constraints that shape the approach: **The ESLint plugin list is curated and linked, and the recipe never lists individual rules.** Eight plugins, each with a summary and a docs link. This narrows the standing "no catalog" line instead of removing it, because a catalog of rules goes stale with every plugin release, while a list of plugins changes rarely. -**Vendor tests pin recipe claims against real installs.** Each claim a recipe makes about tool behavior gets a test that runs the tool, in the same spirit as the existing Vale and ast-grep vendor-contract tests: preview status in the `ruff rule` JSON, `--help-msg` failing without `--load-plugins`, `biome explain` output, and `banned-modules` in `flake8-tidy-imports --help`. They run through `uvx` and `pnpm dlx` at pinned versions, so a version bump that breaks a claim fails a test instead of shipping a wrong recipe. +**Vendor tests pin recipe claims against real installs.** Each claim a recipe makes about tool behavior gets a test that runs the tool, in the same spirit as the existing Vale and ast-grep vendor-contract tests: preview status in the `ruff rule` JSON, `--help-msg` failing without `--load-plugins`, `biome explain` output, and `banned-modules` in `flake8-tidy-imports --help`. They run through `uvx` and `pnpm dlx` at pinned versions, so a version bump that breaks a claim fails a test instead of shipping a wrong recipe. Because they download packages, they are opt-in (`TASKLESS_LEGACY_LINTER_CONTRACT=1`) and run in their own `legacy-linter-contract` workflow on changes to the legacy recipes, not in `Validate`: a registry outage must not block every merge. ## Research record diff --git a/openspec/changes/legacy-linter-recipes/tasks.md b/openspec/changes/legacy-linter-recipes/tasks.md index 7cda3a14..68d6c8c0 100644 --- a/openspec/changes/legacy-linter-recipes/tasks.md +++ b/openspec/changes/legacy-linter-recipes/tasks.md @@ -15,19 +15,19 @@ ## 3. The `--tool` flag (PR 2) -- [ ] 3.1 Collect `create-legacy-rule..md` variants at build time with the same glob mechanism as `.anonymous`, and add `--tool` to the agent command's value flags. Resolve `--tool` before `--anonymous`. Verify tests: a known tool serves its variant, an unknown tool falls back to the generic recipe, `--tool` as `--tool x` and `--tool=x` both parse, and the positional parser does not read the value as a topic. -- [ ] 3.2 Refuse `ast-grep`, `vale`, `prettier`, `black`, and `anonymous` with a non-zero exit that names the right recipe. Verify a test per refused name. -- [ ] 3.3 Make the `route` paragraphs name `create-legacy-rule --tool=`. Verify the "Shipped recipes name only commands that exist" check passes for every rendered paragraph. +- [x] 3.1 Collect `create-legacy-rule..md` variants at build time with the same glob mechanism as `.anonymous`, and add `--tool` to the agent command's value flags. Resolve `--tool` before `--anonymous`. Verify tests: a known tool serves its variant, an unknown tool falls back to the generic recipe, `--tool` as `--tool x` and `--tool=x` both parse, and the positional parser does not read the value as a topic. +- [x] 3.2 Refuse `ast-grep`, `vale`, `prettier`, `black`, and `anonymous` with a non-zero exit that names the right recipe. Verify a test per refused name. +- [x] 3.3 Make the `route` paragraphs name `create-legacy-rule --tool=`. Verify the "Shipped recipes name only commands that exist" check passes for every rendered paragraph. ## 4. Per-tool recipes (PR 2) -- [ ] 4.1 Write `create-legacy-rule.ruff.md`: `banned-api` first, web discovery confirmed with `ruff rule`, preview status checked, the 0.16 default-set change, `extend` and nested configs, and how to run Ruff. Verify recipe tests for the Ruff scenarios. -- [ ] 4.2 Write `create-legacy-rule.eslint.md`: core rules and the `no-restricted-*` catch-alls first (including `@typescript-eslint/`), the eight curated plugins each with a summary and docs link, installed plugins preferred, the ESLint 10 peer gap and `import-x`, flat config against `.eslintrc`, and how to run ESLint. Verify recipe tests that every plugin has a summary and URL and that no individual plugin rules are listed. -- [ ] 4.3 Write `create-legacy-rule.biome.md`: an existing rule first, `noRestrictedImports`/`noRestrictedGlobals` as the catch-alls, web discovery confirmed with `biome explain`, nested configs and `extends`, and how to run Biome. Verify the recipe tests. -- [ ] 4.4 Write `create-legacy-rule.pylint.md`: `--list-msgs`/`--help-msg` with the `--load-plugins` trap, `deprecated-modules`/`preferred-modules`/`bad-functions`, config search order, and how to run pylint. Verify the recipe tests. -- [ ] 4.5 Write `create-legacy-rule.flake8.md`: no discovery step, `banned-modules` from `flake8-tidy-imports` (a dependency that needs consent), the three config files, and how to run flake8. Verify the recipe tests. -- [ ] 4.6 Add vendor tests that run each tool at a pinned version and assert every behavioral claim the recipes make: the `preview` field in `ruff rule --output-format json`, `ruff rule TID251`, `biome explain` output, `pylint --help-msg` failing without and succeeding with `--load-plugins`, and `--banned-modules` in `flake8 --help` with the plugin. Verify they pass locally and in CI, and fail when a claim is edited to be false. -- [ ] 4.7 Extend the changeset with `--tool` and the five recipes. Run `pnpm typecheck` and `pnpm lint`; both pass. +- [x] 4.1 Write `create-legacy-rule.ruff.md`: `banned-api` first, web discovery confirmed with `ruff rule`, preview status checked, the 0.16 default-set change, `extend` and nested configs, and how to run Ruff. Verify recipe tests for the Ruff scenarios. +- [x] 4.2 Write `create-legacy-rule.eslint.md`: core rules and the `no-restricted-*` catch-alls first (including `@typescript-eslint/`), the eight curated plugins each with a summary and docs link, installed plugins preferred, the ESLint 10 peer gap and `import-x`, flat config against `.eslintrc`, and how to run ESLint. Verify recipe tests that every plugin has a summary and URL and that no individual plugin rules are listed. +- [x] 4.3 Write `create-legacy-rule.biome.md`: an existing rule first, `noRestrictedImports`/`noRestrictedGlobals` as the catch-alls, web discovery confirmed with `biome explain`, nested configs and `extends`, and how to run Biome. Verify the recipe tests. +- [x] 4.4 Write `create-legacy-rule.pylint.md`: `--list-msgs`/`--help-msg` with the `--load-plugins` trap, `deprecated-modules`/`preferred-modules`/`bad-functions`, config search order, and how to run pylint. Verify the recipe tests. +- [x] 4.5 Write `create-legacy-rule.flake8.md`: no discovery step, `banned-modules` from `flake8-tidy-imports` (a dependency that needs consent), the three config files, and how to run flake8. Verify the recipe tests. +- [x] 4.6 Add vendor tests that run each tool at a pinned version and assert every behavioral claim the recipes make: the `preview` field in `ruff rule --output-format json`, `ruff rule TID251`, `biome explain` output, `pylint --help-msg` failing without and succeeding with `--load-plugins`, and `--banned-modules` in `flake8 --help` with the plugin. Verify they pass locally and in CI, and fail when a claim is edited to be false. +- [x] 4.7 Extend the changeset with `--tool` and the five recipes. Run `pnpm typecheck` and `pnpm lint`; both pass. ## 5. Archive (PR 2, tip) diff --git a/packages/cli/src/agent/create-legacy-rule.biome.md b/packages/cli/src/agent/create-legacy-rule.biome.md new file mode 100644 index 00000000..a8a67414 --- /dev/null +++ b/packages/cli/src/agent/create-legacy-rule.biome.md @@ -0,0 +1,71 @@ +# Topic: create-legacy-rule (biome) (CLI v%(CLI_VERSION)s / topic v1) + +%(ONE_COMMAND_PER_CALL)s + +## You are here +This is `create-legacy-rule` for Biome. It solves a request by enabling +or configuring a rule Biome already ships. When nothing here fits, +re-run `%(TASKLESS_CLI)s agent route` and continue from step 3 of its legacy +ladder. + +## Goal +Find the Biome rule that makes the problem go away, confirm it in the +installed version, and enable it in the repository's Biome configuration. + +## Preconditions +- `%(TASKLESS_CLI)s detect --json` reports `biome`. +- No auth required. + +## Steps + +1. **Always check the catch-all rules first.** They take your own list + and message, and a search for the request's subject will not surface + them: + - `noRestrictedImports` (group `style`): bans modules or named + imports, with a message. + - `noRestrictedGlobals` (group `style`): bans global names, such as + `event` or `location`. + Both do nothing until they are given a list, so a repo that has not + configured them is not using them. + +2. **Otherwise, find a shipped rule.** Biome ships hundreds of rules, + too many to match a request against from the command line. Search + the rules page, https://biomejs.dev/linter/rules/, for the behavior + the request describes. Many rules port a well-known ESLint rule, and + the page names the source. + +3. **Confirm the rule against the installed version.** Run + `biome explain ` (for example `biome explain noRestrictedImports`). + It prints the rule's documentation, including its group and options, + or fails when the installed version does not have it. + +4. **Enable it where the repo configures Biome.** Biome reads + `biome.json`, `biome.jsonc`, `.biome.json`, or `.biome.jsonc`, from + the current directory up. A nested config marks itself with + `"root": false` or `"extends": "//"`, and `extends` can pull rules + and plugins in from a package, so read what a config extends before + assuming a rule is off. Rules go under `linter.rules..`: + ```json + "linter": { "rules": { "style": { "noRestrictedImports": { + "level": "error", + "options": { "paths": { "lodash": "Use the helpers in src/util." } } + } } } } + ``` + Use `overrides` when the rule should apply to some paths only. + +5. **Report, and say who runs it.** Show the file you changed and the + command that shows the rule firing, e.g. `biome lint `. + `%(TASKLESS_CLI)s check` does NOT run Biome. + +## Important Notes + +- Do not write a Biome plugin. GritQL and JavaScript plugins exist, but + Taskless does not teach authoring them. A request no shipped rule + solves is a Taskless rule. +- Biome 2 changed the configuration format. On Biome 1, check the + installed version's docs before copying a snippet. + +## See Also + +- `%(TASKLESS_CLI)s agent route`: continue down the legacy ladder when no Biome rule fits +- `%(TASKLESS_CLI)s agent create-sg-rule`: ast-grep parses JavaScript and TypeScript diff --git a/packages/cli/src/agent/create-legacy-rule.eslint.md b/packages/cli/src/agent/create-legacy-rule.eslint.md new file mode 100644 index 00000000..486709f4 --- /dev/null +++ b/packages/cli/src/agent/create-legacy-rule.eslint.md @@ -0,0 +1,109 @@ +# Topic: create-legacy-rule (eslint) (CLI v%(CLI_VERSION)s / topic v1) + +%(ONE_COMMAND_PER_CALL)s + +## You are here +This is `create-legacy-rule` for ESLint, including typescript-eslint. It +solves a request with a rule ESLint or an installed plugin already +ships, and writes a new custom rule only from the repository's own +examples. When neither fits, re-run `%(TASKLESS_CLI)s agent route` and +continue from step 3 of its legacy ladder. + +## Goal +Enable or configure the ESLint rule that solves the request, in the +configuration the repository already uses. + +## Preconditions +- `%(TASKLESS_CLI)s detect --json` reports `eslint`. +- No auth required. + +## Steps + +1. **Find which config is live.** ESLint 9 made flat config + (`eslint.config.{js,mjs,cjs,ts,mts,cts}`) the default, and ESLint 10 + removed `.eslintrc*` and `package.json#eslintConfig` entirely. Check + the installed major version in the lockfile: + - On 10, an `.eslintrc*` file is dead. Edit the flat config. + - On 9, `.eslintrc*` is read only when `ESLINT_USE_FLAT_CONFIG=false`. + - In a monorepo, ESLint 10 looks for the config from each linted + file's directory upward, so a package can have its own. + +2. **Always try a core rule first.** Two core rules take your own + pattern and message, and cover most "never do X" requests: + - **`no-restricted-syntax`**: an AST selector and a message. It bans + any syntax shape a selector can describe, such as a call to a + particular member or a particular literal. + ```js + "no-restricted-syntax": ["error", { + selector: "CallExpression[callee.object.name='console'][callee.property.name='log']", + message: "Use the logger in src/log.ts.", + }] + ``` + - **`no-restricted-imports`**: bans modules or named imports, with a + message. + - In a TypeScript repo, prefer the `@typescript-eslint/` counterparts + where they exist: `@typescript-eslint/no-restricted-imports` (it + also understands `import type`) and + `@typescript-eslint/no-restricted-types`. + If one of these rules is already configured, add an entry to it + rather than a second copy: a rule set twice keeps only the last. + Then check the rest of the core rules at + https://eslint.org/docs/latest/rules/. + +3. **Then a plugin, preferring one already installed.** These are the + well-known plugins worth checking. Read the docs of any the repo + already depends on before anything else: + - `eslint-plugin-import`: import and export problems, such as + unresolved paths, cycles, ordering, and duplicates. + https://github.com/import-js/eslint-plugin-import + - `eslint-plugin-regexp`: regular expression mistakes and style. + https://ota-meshi.github.io/eslint-plugin-regexp/ + - `eslint-plugin-promise`: promise practice, such as missing returns + or catches, and nesting. + https://github.com/eslint-community/eslint-plugin-promise + - `eslint-plugin-n`: Node.js, such as deprecated or unsupported APIs, + module resolution, and `process` use. + https://github.com/eslint-community/eslint-plugin-n + - `eslint-plugin-react`: React component and JSX correctness and + conventions. https://github.com/jsx-eslint/eslint-plugin-react + - `eslint-plugin-react-hooks`: the Rules of Hooks and effect + dependencies. https://react.dev/reference/eslint-plugin-react-hooks + - `eslint-plugin-jsx-a11y`: accessibility in JSX elements. + https://github.com/jsx-eslint/eslint-plugin-jsx-a11y + - `eslint-plugin-sonarjs`: bugs and code smells, such as duplicate + branches, unreachable code, and complexity. + https://github.com/SonarSource/SonarJS/blob/master/packages/analysis/src/jsts/rules/README.md + + **On ESLint 10, check the plugin's `eslint` peer range first.** + `eslint-plugin-import`, `eslint-plugin-react`, and + `eslint-plugin-jsx-a11y` have declared support only up to ESLint 9. + For imports on ESLint 10, `eslint-plugin-import-x` declares support, + https://github.com/un-ts/eslint-plugin-import-x. + + **A plugin the repo does not have is a new dependency.** Name the + package and ask the user before adding it. + +4. **No shipped rule fits: check the repo's own rules.** Read + `customRules` for `eslint` in `detect --json`. + - **A `count` of 5 or more:** you may write a new rule. Read the + rules in `evidence` first and model yours on them: their file + layout, how they are registered in the config, and their tests. + - **Fewer than 5, `"unknown"`, or none:** do not write a custom + rule. Return to `%(TASKLESS_CLI)s agent route`. + +5. **Report, and say who runs it.** Show the file you changed and the + command that shows the rule firing, e.g. `eslint ` or the + repo's lint script. `%(TASKLESS_CLI)s check` does NOT run ESLint. + +## Important Notes + +- Do not list or recommend individual plugin rules from memory. Read + the plugin's docs for the version installed. +- A preset (`js.configs.recommended`, `tseslint.configs.recommended`) + is a set of rules someone else chose. Adding one to solve one request + turns on many others; enable the single rule instead. + +## See Also + +- `%(TASKLESS_CLI)s agent route`: continue down the legacy ladder when no ESLint rule fits +- `%(TASKLESS_CLI)s agent create-sg-rule`: ast-grep parses JavaScript and TypeScript diff --git a/packages/cli/src/agent/create-legacy-rule.flake8.md b/packages/cli/src/agent/create-legacy-rule.flake8.md new file mode 100644 index 00000000..23ca52eb --- /dev/null +++ b/packages/cli/src/agent/create-legacy-rule.flake8.md @@ -0,0 +1,67 @@ +# Topic: create-legacy-rule (flake8) (CLI v%(CLI_VERSION)s / topic v1) + +%(ONE_COMMAND_PER_CALL)s + +## You are here +This is `create-legacy-rule` for flake8. flake8's rules come from its +plugins and it has no command that lists them, so this recipe goes +straight to the one check that covers most requests. When it does not +fit, re-run `%(TASKLESS_CLI)s agent route` and continue from step 3 of its +legacy ladder. + +## Goal +Ban a module or import with a message, through `banned-modules`, in the +configuration the repository already uses. + +## Preconditions +- `%(TASKLESS_CLI)s detect --json` reports `flake8`. +- No auth required. + +## Steps + +1. **Use `banned-modules`.** It comes from the `flake8-tidy-imports` + plugin and maps a module to the message shown when it is imported. + It covers most "never import X" and "use Y instead of X" requests. + ```ini + [flake8] + banned-modules = + requests = Use the shared HTTP client in app.http. + mock = Use unittest.mock. + ``` + Its error code is `I251`. If the repo restricts codes with `select`, + add `I25` there; `extend-select` (flake8 4.0 and later) adds it + without replacing the existing selection. + +2. **Check whether the plugin is installed.** `flake8 --bug-report` + prints the installed plugins as JSON. If `flake8-tidy-imports` is + missing, it is a new dependency: name it and ask the user before + adding it. + +3. **Edit the config flake8 actually reads.** flake8 reads the + `[flake8]` section of `.flake8`, `setup.cfg`, or `tox.ini`. It does + not read `pyproject.toml` on its own, so a `[tool.flake8]` table + there only works through a third-party shim the repo would already + have. User-level config is ignored since flake8 4.0. + +4. **No catch-all fits: check the repo's own plugins.** Read + `customRules` for `flake8` in `detect --json`. + - **A `count` of 5 or more:** you may write a new checker. Read the + `[flake8:local-plugins]` entries in `evidence` and the modules they + point to, and model yours on them. + - **Fewer than 5, or none:** do not write a checker. Return to + `%(TASKLESS_CLI)s agent route`. + +5. **Report, and say who runs it.** Show the file you changed and the + command that shows the check firing, e.g. `flake8 `. + `%(TASKLESS_CLI)s check` does NOT run flake8. + +## Important Notes + +- Many `[flake8]` sections only set `max-line-length`, `exclude`, and + `extend-ignore = E203` for Black. That is formatter accommodation, + not a place the repo writes rules. + +## See Also + +- `%(TASKLESS_CLI)s agent route`: continue down the legacy ladder when `banned-modules` does not fit +- `%(TASKLESS_CLI)s agent create-sg-rule`: ast-grep parses Python, for the Taskless rule diff --git a/packages/cli/src/agent/create-legacy-rule.pylint.md b/packages/cli/src/agent/create-legacy-rule.pylint.md new file mode 100644 index 00000000..d47b53ec --- /dev/null +++ b/packages/cli/src/agent/create-legacy-rule.pylint.md @@ -0,0 +1,68 @@ +# Topic: create-legacy-rule (pylint) (CLI v%(CLI_VERSION)s / topic v1) + +%(ONE_COMMAND_PER_CALL)s + +## You are here +This is `create-legacy-rule` for pylint. It solves a request with a +message pylint already ships, and writes a new checker only from the +repository's own examples. When neither fits, re-run +`%(TASKLESS_CLI)s agent route` and continue from step 3 of its legacy ladder. + +## Goal +Enable or configure the pylint message that solves the request, in the +configuration the repository already uses. + +## Preconditions +- `%(TASKLESS_CLI)s detect --json` reports `pylint`. +- pylint is installed, or can be run through the repo's own tooling. +- No auth required. + +## Steps + +1. **Always check the catch-all options first.** They take your own + list, and a search for the request's subject will not surface them: + - `deprecated-modules`: modules that must not be imported. + - `preferred-modules`: `old:new` pairs, reporting the old module + with the one to use instead. + - `bad-functions`: builtin functions that must not be called. It + belongs to the optional `pylint.extensions.bad_builtin` checker, + which must be listed in `load-plugins` to run. + +2. **Otherwise, find a shipped message.** Run `pylint --list-msgs` for + every message the installed version knows, and + `pylint --help-msg=` for one. Messages from an optional + extension (`pylint.extensions.*`) are missing from both unless the + extension is loaded: `--help-msg=bad-builtin` reports "No such + message id or symbol" until you add + `--load-plugins=pylint.extensions.bad_builtin`. Load the extension + before concluding a message does not exist. + https://pylint.readthedocs.io/en/stable/user_guide/checkers/extensions.html + +3. **Enable it where the repo configures pylint.** pylint uses the first + config it finds: `pylintrc`, `pylintrc.toml`, `.pylintrc`, + `.pylintrc.toml`, then `pyproject.toml` (with a `[tool.pylint.*]` + section), `setup.cfg`, and `tox.ini`. Edit that one. pylint enables + most messages by default, so check the message is not just listed in + `disable` before adding anything. + +4. **No shipped message fits: check the repo's own checkers.** Read + `customRules` for `pylint` in `detect --json`. + - **A `count` of 5 or more:** you may write a new checker. Read the + modules in `evidence` first and model yours on them, including how + `load-plugins` reaches them. + - **Fewer than 5, or none:** do not write a checker. Return to + `%(TASKLESS_CLI)s agent route`. + +5. **Report, and say who runs it.** Show the file you changed and the + command that shows the message firing, e.g. `pylint `. + `%(TASKLESS_CLI)s check` does NOT run pylint. + +## Important Notes + +- A long `disable` list weakens pylint rather than expressing a + convention. Removing an entry from it can be the whole fix. + +## See Also + +- `%(TASKLESS_CLI)s agent route`: continue down the legacy ladder when no pylint message fits +- `%(TASKLESS_CLI)s agent create-sg-rule`: ast-grep parses Python, for the Taskless rule diff --git a/packages/cli/src/agent/create-legacy-rule.ruff.md b/packages/cli/src/agent/create-legacy-rule.ruff.md new file mode 100644 index 00000000..61882ec9 --- /dev/null +++ b/packages/cli/src/agent/create-legacy-rule.ruff.md @@ -0,0 +1,87 @@ +# Topic: create-legacy-rule (ruff) (CLI v%(CLI_VERSION)s / topic v1) + +%(ONE_COMMAND_PER_CALL)s + +## You are here +This is `create-legacy-rule` for Ruff. It solves a request by enabling +or configuring a rule Ruff already ships. Ruff has no way to write a +custom rule, so when nothing here fits, the request is a Taskless rule: +re-run `%(TASKLESS_CLI)s agent route` and continue from step 3 of its legacy +ladder. + +## Goal +Find the Ruff rule that solves the request, confirm it exists in the +installed version, and enable it in the repository's Ruff configuration. + +## Preconditions +- `%(TASKLESS_CLI)s detect --json` reports `ruff`. +- Ruff is installed, or can be run through the repo's own tooling + (`uv run ruff`, `poetry run ruff`, or the `ruff` on PATH). +- No auth required. + +## Steps + +1. **Always check `banned-api` first.** It is rule `TID251`, configured + under `flake8-tidy-imports`, and it bans an import or an attribute + with a message you write. It covers most "never call X" and "use Y + instead of X" requests, and a search for the request's subject will + not surface it: + ```toml + [tool.ruff.lint.flake8-tidy-imports.banned-api] + "requests.get".msg = "Use the shared HTTP client in app.http." + ``` + Enable it with `TID251` in `extend-select` if the repo has not + already. `banned-module-level-imports` (`TID253`) is the companion + for "import X lazily, inside the function". + +2. **Otherwise, find a shipped rule.** Ruff ships hundreds of rules, + too many to match a request against from the command line. Search + the rules page, https://docs.astral.sh/ruff/rules/, for the behavior + the request describes. Every rule has a code (`B006`) and a name + (`mutable-argument-default`). + +3. **Confirm the rule against the installed version.** Run + `ruff rule `. It prints the rule's documentation, or + an error when the installed version does not have it. Read the top + of the output: + - **"This rule is in preview and is not stable."** The rule does not + fire unless preview mode is on (`preview = true` under `[lint]`). + Turning preview on enables every other preview rule in the + selected set too, so tell the user and let them decide, or choose + a stable rule instead. Never enable a preview rule in a + configuration where it cannot fire. + - **"Derived from the X linter."** The rule's settings live under + `[lint.X]`, as `banned-api` does under `flake8-tidy-imports`. + +4. **Enable it where the repo configures Ruff.** Ruff reads + `.ruff.toml`, then `ruff.toml`, then `pyproject.toml` (only when it + has a `[tool.ruff]` section), and uses the closest one to each file. + It does not merge them; a file inherits another only through + `extend = "..."`. Edit the file that actually governs the code the + request is about. In `pyproject.toml` the keys are under + `[tool.ruff.lint]`; in `ruff.toml` under `[lint]`. A top-level + `select` is the deprecated spelling, so follow whichever the repo + uses. + - Add the code to `extend-select` rather than `select`. `select` + replaces the default set, so adding to it can silently turn off + the rules the repo relies on today. + - Since Ruff 0.16 the default set is about 400 rules, up from 59, so + check the rule is not already on before adding it. + +5. **Report, and say who runs it.** Show the file you changed and the + command that shows the rule firing, e.g. `ruff check `. + `%(TASKLESS_CLI)s check` does NOT run Ruff; the repo's own lint step + does. + +## Important Notes + +- Do not write a Ruff plugin or a custom rule. Ruff has no mechanism + for one, and the `customRules` count `detect` reports for Ruff is + always 0. +- `ignore` and `per-file-ignores` turn rules off. They never solve a + request to enforce something. + +## See Also + +- `%(TASKLESS_CLI)s agent route`: continue down the legacy ladder when no Ruff rule fits +- `%(TASKLESS_CLI)s agent create-sg-rule`: ast-grep parses Python, for the Taskless rule diff --git a/packages/cli/src/commands/agent.ts b/packages/cli/src/commands/agent.ts index 859e2d8b..34edba6c 100644 --- a/packages/cli/src/commands/agent.ts +++ b/packages/cli/src/commands/agent.ts @@ -105,6 +105,26 @@ async function resolveDescription( */ const UNLISTED_COMMANDS = new Set(["agent", "feedback"]); +/** + * Names `--tool` refuses rather than falling back on. Each one is a linter + * `detect` reports, so an agent could copy it into the flag, but none of them + * hosts a legacy rule, and the generic recipe would tell it to try. + * `anonymous` is here because it names the other variant file. + */ +const REFUSED_TOOLS = new Map([ + ["ast-grep", "author an ast-grep rule with `agent create-sg-rule`."], + ["vale", "author a Vale rule with `agent create-vale-rule`."], + [ + "prettier", + "it is a formatter and holds no rules; decide again with `agent route`.", + ], + [ + "black", + "it is a formatter and holds no rules; decide again with `agent route`.", + ], + ["anonymous", "pass `--anonymous` for the local-only variant."], +]); + export function createAgentCommand(subCommands: SubCommandsDef) { return defineCommand({ meta: { @@ -124,11 +144,16 @@ export function createAgentCommand(subCommands: SubCommandsDef) { "Return the local-only recipe variant when the topic has one", default: false, }, + tool: { + type: "string", + description: + "Return the topic's variant for one tool, named as `detect` reports it", + }, }, async run({ args, rawArgs }) { // Extract positional args from rawArgs, skipping flags and their values. - // --dir/-d take a value; --json/--anonymous are boolean and do not. - const valueFlagSet = new Set(["--dir", "-d"]); + // --dir/-d/--tool take a value; --json/--anonymous are boolean and do not. + const valueFlagSet = new Set(["--dir", "-d", "--tool"]); const positionals: string[] = []; for (let index = 0; index < rawArgs.length; index++) { const argument = rawArgs[index]!; @@ -241,6 +266,15 @@ export function createAgentCommand(subCommands: SubCommandsDef) { return; } + const tool = args.tool as string | undefined; + const refusal = tool === undefined ? undefined : REFUSED_TOOLS.get(tool); + if (refusal !== undefined) { + telemetry.capture("cli_agent", { topic: key }); + console.error(`--tool=${tool} has no legacy recipe: ${refusal}`); + process.exitCode = 1; + return; + } + // Anonymous variant lookup: prefer .anonymous.md when // --anonymous is set, fall back to the canonical recipe. The lookup and // the render both live in the shared prompts module, so `agent` and the @@ -258,7 +292,7 @@ export function createAgentCommand(subCommands: SubCommandsDef) { // template is asked (`variables` comes from sprintf's own parse) rather // than a topic name being hardcoded, so the next recipe to use the // mechanism needs no edit here. - const template = getRawRecipe(key, { anonymous: args.anonymous }); + const template = getRawRecipe(key, { anonymous: args.anonymous, tool }); const hostTools = template?.variables.some((name) => HOST_TOOL_VARIABLES.has(name) ) @@ -274,6 +308,7 @@ export function createAgentCommand(subCommands: SubCommandsDef) { const linters = scan?.linters.map((linter) => linter.name); const recipe = getRecipe(key, { anonymous: args.anonymous, + tool, invocation, hostTools, linters, diff --git a/packages/cli/src/prompts/recipes.ts b/packages/cli/src/prompts/recipes.ts index 15737eb1..00484b10 100644 --- a/packages/cli/src/prompts/recipes.ts +++ b/packages/cli/src/prompts/recipes.ts @@ -49,30 +49,45 @@ const recipeFiles: Record = import.meta.glob("../agent/*.md", { eager: true, }); -// Build two lookup maps: +// Build three lookup maps: // - recipeMap: "rule-create" → canonical recipe text // - anonymousMap: "rule-create" → anonymous variant text (if exists) +// - toolMap: "create-legacy-rule" → tool name → that tool's variant +// +// A topic name never contains a dot, so everything after the first one in a +// filename is the variant: `anonymous`, or a tool name as `detect` reports it. function buildRecipeMaps(): { recipeMap: Map; anonymousMap: Map; + toolMap: Map>; } { const recipeMap = new Map(); const anonymousMap = new Map(); + const toolMap = new Map>(); for (const [path, rawContent] of Object.entries(recipeFiles)) { const content = stripValeDirectives(rawContent); const filename = path.split("/").pop()?.replace(/\.md$/, ""); if (!filename) continue; - if (filename.endsWith(".anonymous")) { - const topic = filename.slice(0, -".anonymous".length); + const [topic, variant] = filename.split(".", 2) as [string, string?]; + if (variant === undefined) { + recipeMap.set(topic, content); + } else if (variant === "anonymous") { anonymousMap.set(topic, content); } else { - recipeMap.set(filename, content); + const tools = toolMap.get(topic) ?? new Map(); + tools.set(variant, content); + toolMap.set(topic, tools); } } - return { recipeMap, anonymousMap }; + return { recipeMap, anonymousMap, toolMap }; } -const { recipeMap, anonymousMap } = buildRecipeMaps(); +const { recipeMap, anonymousMap, toolMap } = buildRecipeMaps(); + +/** The tools that have their own variant of a topic, e.g. `create-legacy-rule`. */ +export function toolVariants(topic: string): string[] { + return [...(toolMap.get(topic)?.keys() ?? [])]; +} /** The canonical `.md` recipe names present in the build. */ export function canonicalRecipeTopics(): string[] { @@ -169,6 +184,13 @@ export interface RecipeOptions { * @default false */ anonymous?: boolean; + /** + * Select the topic's variant for one tool, named as `detect` reports it + * (`create-legacy-rule` with `ruff` serves `create-legacy-rule.ruff.md`). + * Takes precedence over {@link anonymous}, and falls back the same way + * when the topic has no variant for that tool. + */ + tool?: string; /** * Value substituted for the `%(PACKAGE_MANAGER_DLX)s` placeholder. The * default is an agent-fill marker, which is the right answer whenever @@ -491,7 +513,7 @@ It is a plugin package, so the repo may need to add it.`, */ function legacyLintersBlock( linters: string[] | undefined, - fetchPrefix: string + fetchPrefix: string | undefined ): string { const described = LEGACY_LINTER_GUIDE.filter( ([name]) => linters === undefined || linters.includes(name) @@ -504,14 +526,16 @@ function legacyLintersBlock( linters === undefined ? " Linters with a legacy recipe, and what step 1 always checks:" : " The linters `detect` found here, and what step 1 always checks:"; - const items = described.map( - ([, paragraph]) => ` - ${indentBlock(paragraph, " ")}` - ); + const items = described.map(([name, paragraph]) => { + const fetch = + fetchPrefix === undefined + ? `Fetch \`create-legacy-rule\` with tool \`${name}\`.` + : `Fetch \`${fetchPrefix}create-legacy-rule --tool=${name}\`.`; + return ` - ${indentBlock(`${paragraph}\n${fetch}`, " ")}`; + }); return `${heading} -${items.join("\n")} - - For any of them, fetch \`${fetchPrefix}create-legacy-rule\`.`; +${items.join("\n")}`; } /** The state a passage conditioned on one tool renders from. */ @@ -770,7 +794,9 @@ export function buildVariables( HOST_TOOLS: indentBlock(hostToolsStep(options.hostTools), " "), LEGACY_LINTERS: legacyLintersBlock( options.linters, - options.mechanics === false ? "" : `${resolveInvocation(options)} agent ` + options.mechanics === false + ? undefined + : `${resolveInvocation(options)} agent ` ), }; if (content.includes("%(INPUT_SCHEMA)s")) { @@ -923,6 +949,11 @@ function lookupRecipe( topic: string, options: RecipeOptions ): string | undefined { + const toolVariant = + options.tool === undefined + ? undefined + : toolMap.get(topic)?.get(options.tool); + if (toolVariant !== undefined) return toolVariant; return options.anonymous ? (anonymousMap.get(topic) ?? recipeMap.get(topic)) : recipeMap.get(topic); diff --git a/packages/cli/test/agent-extensions.test.ts b/packages/cli/test/agent-extensions.test.ts index e9b9b6bd..e66e3a8f 100644 --- a/packages/cli/test/agent-extensions.test.ts +++ b/packages/cli/test/agent-extensions.test.ts @@ -551,3 +551,75 @@ describe("create-sg-rule names the tree-sitter traps under Quirks", () => { expect(result.stdout).toContain("kind: await_expression"); }); }); + +describe("taskless agent --tool (variant lookup)", () => { + let cwd: string; + + beforeEach(async () => { + cwd = await mkdtemp(join(tmpdir(), "taskless-agent-tool-")); + }); + + afterEach(async () => { + await rm(cwd, { recursive: true, force: true }); + }); + + it.each(["ruff", "eslint", "biome", "pylint", "flake8"])( + "serves the %s variant of create-legacy-rule", + async (tool) => { + const result = await runCli([ + "agent", + "create-legacy-rule", + `--tool=${tool}`, + "-d", + cwd, + ]); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain(`# Topic: create-legacy-rule (${tool})`); + } + ); + + it("takes the value as a separate argument, not as a second topic", async () => { + const result = await runCli([ + "agent", + "create-legacy-rule", + "--tool", + "ruff", + "-d", + cwd, + ]); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("# Topic: create-legacy-rule (ruff)"); + }); + + it("falls back to the generic recipe for a tool with no variant", async () => { + const generic = await runCli(["agent", "create-legacy-rule", "-d", cwd]); + const rubocop = await runCli([ + "agent", + "create-legacy-rule", + "--tool=rubocop", + "-d", + cwd, + ]); + expect(rubocop.exitCode).toBe(0); + expect(rubocop.stdout).toBe(generic.stdout); + }); + + it.each([ + ["vale", "agent create-vale-rule"], + ["ast-grep", "agent create-sg-rule"], + ["prettier", "agent route"], + ["black", "agent route"], + ["anonymous", "--anonymous"], + ])("refuses --tool=%s and names what to use", async (tool, instead) => { + const result = await runCli([ + "agent", + "create-legacy-rule", + `--tool=${tool}`, + "-d", + cwd, + ]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain(instead); + expect(result.stdout).toBe(""); + }); +}); diff --git a/packages/cli/test/legacy-ladder.test.ts b/packages/cli/test/legacy-ladder.test.ts index 62e2e698..9e59df17 100644 --- a/packages/cli/test/legacy-ladder.test.ts +++ b/packages/cli/test/legacy-ladder.test.ts @@ -8,6 +8,12 @@ function recipe(topic: string): string { return text; } +function tool(name: string): string { + const text = getRecipe("create-legacy-rule", { tool: name }); + if (text === undefined) throw new Error(`no recipe for ${name}`); + return text; +} + describe("route's legacy ladder", () => { const route = recipe("route"); @@ -99,3 +105,75 @@ describe("onboarding's legacy limits", () => { ); }); }); + +describe("the per-tool legacy recipes", () => { + it("has Ruff try banned-api before anything else", () => { + const ruff = tool("ruff"); + const banned = ruff.indexOf("**Always check `banned-api` first.**"); + expect(banned).toBeGreaterThan(-1); + expect(banned).toBeLessThan( + ruff.indexOf("**Otherwise, find a shipped rule.**") + ); + expect(ruff).toContain("`ruff rule `"); + expect(ruff).toContain( + "Never enable a preview rule in a\n configuration where it cannot fire." + ); + }); + + it("has ESLint try a core rule before a plugin", () => { + const eslint = tool("eslint"); + const core = eslint.indexOf("**Always try a core rule first.**"); + expect(core).toBeGreaterThan(-1); + expect(core).toBeLessThan( + eslint.indexOf("**Then a plugin, preferring one already installed.**") + ); + }); + + it("gives every curated ESLint plugin a summary and a docs link", () => { + const eslint = tool("eslint"); + for (const plugin of [ + "eslint-plugin-import", + "eslint-plugin-regexp", + "eslint-plugin-promise", + "eslint-plugin-n", + "eslint-plugin-react", + "eslint-plugin-react-hooks", + "eslint-plugin-jsx-a11y", + "eslint-plugin-sonarjs", + ]) { + expect(eslint).toMatch( + new RegExp( + String.raw`- \`${plugin}\`: [^\n]+(?:\n {5}[^\n]+)*\n? *https://` + ) + ); + } + expect(eslint).toContain("`eslint-plugin-import-x`"); + }); + + it("has Biome check its catch-alls and confirm with biome explain", () => { + const biome = tool("biome"); + expect(biome).toContain("`noRestrictedImports`"); + expect(biome).toContain("`noRestrictedGlobals`"); + expect(biome).toContain("`biome explain `"); + expect(biome).toContain("Do not write a Biome plugin."); + }); + + it("has pylint load an extension before ruling its message out", () => { + expect(tool("pylint")).toContain( + "Load the extension\n before concluding a message does not exist." + ); + }); + + it("sends flake8 straight to banned-modules with no discovery step", () => { + const flake8 = tool("flake8"); + expect(flake8).toContain("**Use `banned-modules`.**"); + expect(flake8).not.toContain("--list"); + }); + + it.each(["ruff", "eslint", "biome", "pylint", "flake8"])( + "has the %s recipe say Taskless does not run the linter", + (name) => { + expect(tool(name)).toMatch(/check` does NOT run /); + } + ); +}); diff --git a/packages/cli/test/legacy-linter-vendor-contract.test.ts b/packages/cli/test/legacy-linter-vendor-contract.test.ts new file mode 100644 index 00000000..4eafbb51 --- /dev/null +++ b/packages/cli/test/legacy-linter-vendor-contract.test.ts @@ -0,0 +1,188 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { getRecipe } from "../src/prompts/recipes"; + +/** + * The linter behaviour the `create-legacy-rule` recipes describe, pinned. + * + * Unlike ast-grep and Vale, Taskless does not ship these linters, so each + * one is fetched at a pinned version (`uvx` for Python, `pnpm dlx` for + * Biome) and asked directly. A bump that changes one of these fails here, + * naming the recipe sentence that rests on it, instead of an agent acting + * on a recipe that is no longer true. + * + * OPT-IN, because it downloads packages. `TASKLESS_LEGACY_LINTER_CONTRACT=1` + * runs it; the `legacy-linter-contract` workflow sets it. + */ +const ENABLED = process.env.TASKLESS_LEGACY_LINTER_CONTRACT === "1"; + +const RUFF = "ruff==0.17.2"; +const PYLINT = "pylint==4.1.3"; +const FLAKE8 = "flake8==7.4.1"; +const FLAKE8_TIDY_IMPORTS = "flake8-tidy-imports"; +const BIOME = "@biomejs/biome@2.6.2"; + +let workdir: string; + +function run(command: string, args: string[]) { + const result = spawnSync(command, args, { + cwd: workdir, + encoding: "utf8", + timeout: 120_000, + // `ruff rule --all` prints every rule's full documentation. + maxBuffer: 64 * 1024 * 1024, + }); + return { + status: result.status, + output: `${result.stdout ?? ""}${result.stderr ?? ""}`, + }; +} + +function recipe(tool: string): string { + return getRecipe("create-legacy-rule", { tool }) ?? ""; +} + +describe.skipIf(!ENABLED)("legacy linter vendor contract", () => { + beforeAll(() => { + workdir = mkdtempSync(join(tmpdir(), "taskless-legacy-contract-")); + }); + afterAll(() => { + rmSync(workdir, { recursive: true, force: true }); + }); + + describe("ruff", () => { + it("reports preview status in `ruff rule` JSON, which the recipe tells the agent to check", () => { + const { status, output } = run("uvx", [ + RUFF, + "rule", + "--all", + "--output-format", + "json", + ]); + expect(status).toBe(0); + const rules = JSON.parse(output) as Array>; + expect(rules.length).toBeGreaterThan(400); + for (const key of ["code", "name", "preview"]) { + expect(rules[0]).toHaveProperty(key); + } + expect(recipe("ruff")).toContain( + '**"This rule is in preview and is not stable."**' + ); + }); + + it("says a preview rule is preview at the top of `ruff rule`", () => { + const { output } = run("uvx", [ + RUFF, + "rule", + "unnecessary-regular-expression-compile", + ]); + expect(output).toContain("This rule is in preview and is not stable."); + }); + + it("has banned-api as TID251, derived from flake8-tidy-imports", () => { + const { status, output } = run("uvx", [RUFF, "rule", "TID251"]); + expect(status).toBe(0); + expect(output).toContain("# banned-api (TID251)"); + expect(output).toContain( + "Derived from the **flake8-tidy-imports** linter." + ); + }); + + it("fails on a rule the installed version does not have", () => { + expect(run("uvx", [RUFF, "rule", "no-such-rule"]).status).not.toBe(0); + }); + }); + + describe("pylint", () => { + it("does not know an extension's message until the extension is loaded", () => { + const without = run("uvx", [ + "--from", + PYLINT, + "pylint", + "--help-msg=bad-builtin", + ]); + expect(without.output).toContain("No such message id or symbol"); + const loaded = run("uvx", [ + "--from", + PYLINT, + "pylint", + "--load-plugins=pylint.extensions.bad_builtin", + "--help-msg=bad-builtin", + ]); + expect(loaded.output).toContain(":bad-builtin (W0141)"); + expect(recipe("pylint")).toContain( + "`--load-plugins=pylint.extensions.bad_builtin`" + ); + }); + + it("has the catch-all options the recipe names", () => { + const { output } = run("uvx", [ + "--from", + PYLINT, + "pylint", + "--load-plugins=pylint.extensions.bad_builtin", + "--help", + ]); + for (const option of [ + "--deprecated-modules", + "--preferred-modules", + "--bad-functions", + ]) { + expect(output).toContain(option); + } + }); + }); + + describe("flake8", () => { + it("reports a banned module as I251 through flake8-tidy-imports", () => { + writeFileSync(join(workdir, "banned.py"), "import requests\n"); + writeFileSync( + join(workdir, ".flake8"), + "[flake8]\nbanned-modules =\n requests = Use the client.\n" + ); + const { output } = run("uvx", [ + "--from", + FLAKE8, + "--with", + FLAKE8_TIDY_IMPORTS, + "flake8", + "banned.py", + ]); + expect(output).toContain("I251 Banned import 'requests' used"); + expect(recipe("flake8")).toContain("Its error code is `I251`."); + }); + + it("lists installed plugins as JSON with --bug-report", () => { + const { status, output } = run("uvx", [ + "--from", + FLAKE8, + "flake8", + "--bug-report", + ]); + expect(status).toBe(0); + expect(JSON.parse(output)).toHaveProperty("plugins"); + }); + }); + + describe("biome", () => { + it("explains both catch-all rules, in the style group", () => { + for (const rule of ["noRestrictedImports", "noRestrictedGlobals"]) { + const { status, output } = run("pnpm", ["dlx", BIOME, "explain", rule]); + expect(status).toBe(0); + expect(output).toContain(`Diagnostic category: lint/style/${rule}`); + expect(recipe("biome")).toContain(`\`${rule}\` (group \`style\`)`); + } + }); + + it("fails to explain a rule it does not have", () => { + expect( + run("pnpm", ["dlx", BIOME, "explain", "noSuchRule"]).status + ).not.toBe(0); + }); + }); +}); diff --git a/packages/cli/test/prompts.test.ts b/packages/cli/test/prompts.test.ts index e10c34b6..14b46ac9 100644 --- a/packages/cli/test/prompts.test.ts +++ b/packages/cli/test/prompts.test.ts @@ -57,11 +57,14 @@ async function importBuiltPrompts(): Promise<{ /** Canonical `.md` names on disk, excluding `.anonymous` variants. */ async function canonicalTopicsOnDisk(): Promise { const entries = await readdir(recipeDirectory); - return entries - .filter((name) => name.endsWith(".md")) - .map((name) => name.slice(0, -".md".length)) - .filter((stem) => !stem.endsWith(".anonymous")) - .toSorted(); + return ( + entries + .filter((name) => name.endsWith(".md")) + .map((name) => name.slice(0, -".md".length)) + // A variant (`.anonymous`, or a tool such as `.ruff`) is not a topic. + .filter((stem) => !stem.includes(".")) + .toSorted() + ); } describe("prompt rendering", () => { @@ -653,7 +656,8 @@ describe("route's per-linter passage", () => { it("names the topic rather than a command when mechanics are off", () => { const route = getPrompt("route", { mechanics: false }); - expect(route).toContain("For any of them, fetch `create-legacy-rule`."); + expect(route).toContain("Fetch `create-legacy-rule` with tool `ruff`."); + expect(route).not.toContain("--tool="); }); it("is rendered from the repository the agent command scans", async () => { @@ -669,6 +673,7 @@ describe("route's per-linter passage", () => { ]); expect(stdout).toContain("The linters `detect` found here"); expect(stdout).toContain("**Ruff**"); + expect(stdout).toContain("agent create-legacy-rule --tool=ruff`."); expect(stdout).not.toContain("**ESLint**"); } finally { await rm(directory, { recursive: true, force: true }); diff --git a/packages/cli/test/recipe-cross-references.test.ts b/packages/cli/test/recipe-cross-references.test.ts index f107ee44..967450b9 100644 --- a/packages/cli/test/recipe-cross-references.test.ts +++ b/packages/cli/test/recipe-cross-references.test.ts @@ -143,7 +143,7 @@ async function embeddedTopics(): Promise> { return new Set( files .map((file) => file.replace(/\.md$/, "")) - .map((name) => name.replace(/\.anonymous$/, "")) + .map((name) => name.split(".")[0]!) ); } @@ -151,10 +151,13 @@ async function embeddedTopics(): Promise> { async function renderedRecipes(): Promise> { const files = await recipeFiles(); return files.map((file) => { - const stem = file.replace(/\.md$/, ""); - const anonymous = stem.endsWith(".anonymous"); - const topic = stem.replace(/\.anonymous$/, ""); - return [file, getRecipe(topic, { anonymous }) ?? ""]; + const [topic, variant] = file.replace(/\.md$/, "").split(".") as [ + string, + string?, + ]; + const anonymous = variant === "anonymous"; + const tool = variant === undefined || anonymous ? undefined : variant; + return [file, getRecipe(topic, { anonymous, tool }) ?? ""]; }); } @@ -212,7 +215,7 @@ describe("shipped recipes name only commands that exist", () => { const mismatched: string[] = []; for (const file of await recipeFiles()) { const source = await readFile(join(recipeDirectory, file), "utf8"); - const topic = file.replace(/(\.anonymous)?\.md$/, ""); + const topic = file.replace(/\.md$/, "").split(".")[0]!; const firstLine = source.split("\n")[0] ?? ""; if (!firstLine.startsWith(`# Topic: ${topic}`)) { mismatched.push(`${file}: ${firstLine}`); From a72123fe1d2de048abf64b2575884341f7b4accd Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Thu, 22 Oct 2026 18:21:57 -0700 Subject: [PATCH 2/5] docs(openspec): archive legacy-linter-recipes --- .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/cli-agent/spec.md | 0 .../specs/cli-detect/spec.md | 0 .../specs/cli-legacy-rule-recipes/spec.md | 0 .../specs/cli-onboard/spec.md | 0 .../specs/cli-rule-routing/spec.md | 0 .../tasks.md | 4 +- openspec/specs/cli-agent/spec.md | 28 ++++- openspec/specs/cli-detect/spec.md | 26 +++++ .../specs/cli-legacy-rule-recipes/spec.md | 109 ++++++++++++++++++ openspec/specs/cli-onboard/spec.md | 17 +++ openspec/specs/cli-rule-routing/spec.md | 90 +++++++++++---- 14 files changed, 244 insertions(+), 30 deletions(-) rename openspec/changes/{legacy-linter-recipes => archive/2026-10-22-legacy-linter-recipes}/.openspec.yaml (100%) rename openspec/changes/{legacy-linter-recipes => archive/2026-10-22-legacy-linter-recipes}/design.md (100%) rename openspec/changes/{legacy-linter-recipes => archive/2026-10-22-legacy-linter-recipes}/proposal.md (100%) rename openspec/changes/{legacy-linter-recipes => archive/2026-10-22-legacy-linter-recipes}/specs/cli-agent/spec.md (100%) rename openspec/changes/{legacy-linter-recipes => archive/2026-10-22-legacy-linter-recipes}/specs/cli-detect/spec.md (100%) rename openspec/changes/{legacy-linter-recipes => archive/2026-10-22-legacy-linter-recipes}/specs/cli-legacy-rule-recipes/spec.md (100%) rename openspec/changes/{legacy-linter-recipes => archive/2026-10-22-legacy-linter-recipes}/specs/cli-onboard/spec.md (100%) rename openspec/changes/{legacy-linter-recipes => archive/2026-10-22-legacy-linter-recipes}/specs/cli-rule-routing/spec.md (100%) rename openspec/changes/{legacy-linter-recipes => archive/2026-10-22-legacy-linter-recipes}/tasks.md (97%) create mode 100644 openspec/specs/cli-legacy-rule-recipes/spec.md diff --git a/openspec/changes/legacy-linter-recipes/.openspec.yaml b/openspec/changes/archive/2026-10-22-legacy-linter-recipes/.openspec.yaml similarity index 100% rename from openspec/changes/legacy-linter-recipes/.openspec.yaml rename to openspec/changes/archive/2026-10-22-legacy-linter-recipes/.openspec.yaml diff --git a/openspec/changes/legacy-linter-recipes/design.md b/openspec/changes/archive/2026-10-22-legacy-linter-recipes/design.md similarity index 100% rename from openspec/changes/legacy-linter-recipes/design.md rename to openspec/changes/archive/2026-10-22-legacy-linter-recipes/design.md diff --git a/openspec/changes/legacy-linter-recipes/proposal.md b/openspec/changes/archive/2026-10-22-legacy-linter-recipes/proposal.md similarity index 100% rename from openspec/changes/legacy-linter-recipes/proposal.md rename to openspec/changes/archive/2026-10-22-legacy-linter-recipes/proposal.md diff --git a/openspec/changes/legacy-linter-recipes/specs/cli-agent/spec.md b/openspec/changes/archive/2026-10-22-legacy-linter-recipes/specs/cli-agent/spec.md similarity index 100% rename from openspec/changes/legacy-linter-recipes/specs/cli-agent/spec.md rename to openspec/changes/archive/2026-10-22-legacy-linter-recipes/specs/cli-agent/spec.md diff --git a/openspec/changes/legacy-linter-recipes/specs/cli-detect/spec.md b/openspec/changes/archive/2026-10-22-legacy-linter-recipes/specs/cli-detect/spec.md similarity index 100% rename from openspec/changes/legacy-linter-recipes/specs/cli-detect/spec.md rename to openspec/changes/archive/2026-10-22-legacy-linter-recipes/specs/cli-detect/spec.md diff --git a/openspec/changes/legacy-linter-recipes/specs/cli-legacy-rule-recipes/spec.md b/openspec/changes/archive/2026-10-22-legacy-linter-recipes/specs/cli-legacy-rule-recipes/spec.md similarity index 100% rename from openspec/changes/legacy-linter-recipes/specs/cli-legacy-rule-recipes/spec.md rename to openspec/changes/archive/2026-10-22-legacy-linter-recipes/specs/cli-legacy-rule-recipes/spec.md diff --git a/openspec/changes/legacy-linter-recipes/specs/cli-onboard/spec.md b/openspec/changes/archive/2026-10-22-legacy-linter-recipes/specs/cli-onboard/spec.md similarity index 100% rename from openspec/changes/legacy-linter-recipes/specs/cli-onboard/spec.md rename to openspec/changes/archive/2026-10-22-legacy-linter-recipes/specs/cli-onboard/spec.md diff --git a/openspec/changes/legacy-linter-recipes/specs/cli-rule-routing/spec.md b/openspec/changes/archive/2026-10-22-legacy-linter-recipes/specs/cli-rule-routing/spec.md similarity index 100% rename from openspec/changes/legacy-linter-recipes/specs/cli-rule-routing/spec.md rename to openspec/changes/archive/2026-10-22-legacy-linter-recipes/specs/cli-rule-routing/spec.md diff --git a/openspec/changes/legacy-linter-recipes/tasks.md b/openspec/changes/archive/2026-10-22-legacy-linter-recipes/tasks.md similarity index 97% rename from openspec/changes/legacy-linter-recipes/tasks.md rename to openspec/changes/archive/2026-10-22-legacy-linter-recipes/tasks.md index 68d6c8c0..aabcc2f0 100644 --- a/openspec/changes/legacy-linter-recipes/tasks.md +++ b/openspec/changes/archive/2026-10-22-legacy-linter-recipes/tasks.md @@ -31,5 +31,5 @@ ## 5. Archive (PR 2, tip) -- [ ] 5.1 Run the pre-archive check from CLAUDE.md (commit, keep the SHA, `pnpm openspec archive legacy-linter-recipes -y`, confirm every prior scenario of `cli-rule-routing`, `cli-agent`, `cli-detect`, and `cli-onboard` survives, then reset to the SHA). -- [ ] 5.2 Archive for real. Verify `pnpm openspec validate --strict` passes on the result. +- [x] 5.1 Run the pre-archive check from CLAUDE.md (commit, keep the SHA, `pnpm openspec archive legacy-linter-recipes -y`, confirm every prior scenario of `cli-rule-routing`, `cli-agent`, `cli-detect`, and `cli-onboard` survives, then reset to the SHA). +- [x] 5.2 Archive for real. Verify `pnpm openspec validate --strict` passes on the result. diff --git a/openspec/specs/cli-agent/spec.md b/openspec/specs/cli-agent/spec.md index 6315c967..9c959e91 100644 --- a/openspec/specs/cli-agent/spec.md +++ b/openspec/specs/cli-agent/spec.md @@ -8,11 +8,14 @@ Defines the `agent` subcommand for the `@taskless/cli` package, including recipe ### Requirement: Agent subcommand serves recipes for commands -The CLI SHALL support an `agent` subcommand that accepts at most one positional argument identifying a topic AND an optional `--anonymous` boolean flag. Topics SHALL be addressed by a single token; the subcommand SHALL NOT join multiple positionals into a topic key. When a topic is provided, the subcommand SHALL look up a matching recipe file embedded at build time using the following resolution order: +The CLI SHALL support an `agent` subcommand that accepts at most one positional argument identifying a topic, an optional `--anonymous` boolean flag, AND an optional `--tool ` value flag. Topics SHALL be addressed by a single token; the subcommand SHALL NOT join multiple positionals into a topic key. When a topic is provided, the subcommand SHALL look up a matching recipe file embedded at build time using the following resolution order: -1. If `--anonymous` is set AND `.anonymous.md` exists in the embedded map, return that file. -2. Otherwise, return `.md`. -3. If neither exists, exit with code 1 and an error message suggesting `taskless agent` for the topic index. +1. If `--tool` is set AND `..md` exists in the embedded map, return that file. +2. If `--anonymous` is set AND `.anonymous.md` exists in the embedded map, return that file. +3. Otherwise, return `.md`. +4. If none of these exists, exit with code 1 and an error message suggesting `taskless agent` for the topic index. + +`--tool` SHALL refuse, with a non-zero exit and a message naming the recipe to use instead, a tool that has its own authoring recipe (`ast-grep`, `vale`) or that cannot hold a rule (`prettier`, `black`). An unrecognized tool name SHALL fall back to `.md`. When no positional argument is provided, the subcommand SHALL print a topic index containing a one-paragraph human slug followed by a topic disambiguation table mapping topic names to their summaries. @@ -39,6 +42,23 @@ The subcommand is named for its reader. It serves agents fetching a procedure, n - **WHEN** a user runs `taskless help check` - **THEN** the CLI SHALL NOT print recipe text for `check` +#### Scenario: A tool variant is served + +- **WHEN** a user runs `taskless agent create-legacy-rule --tool=ruff` +- **AND** `create-legacy-rule.ruff.md` exists +- **THEN** the CLI SHALL print that file's contents + +#### Scenario: An unknown tool falls back to the generic recipe + +- **WHEN** a user runs `taskless agent create-legacy-rule --tool=rubocop` +- **AND** no `create-legacy-rule.rubocop.md` exists +- **THEN** the CLI SHALL print `create-legacy-rule.md` + +#### Scenario: A tool with its own recipe is refused + +- **WHEN** a user runs `taskless agent create-legacy-rule --tool=vale` +- **THEN** the CLI SHALL exit non-zero and name `create-vale-rule` as the recipe to use + ### Requirement: Recipe files are embedded at build time Recipe files SHALL be located at `packages/cli/src/agent/` as `.md` files. The Vite build SHALL embed these files into the CLI bundle via `import.meta.glob` with raw imports. A recipe file SHALL exist for every registered command and subcommand. diff --git a/openspec/specs/cli-detect/spec.md b/openspec/specs/cli-detect/spec.md index 341d80fc..b7aabbcb 100644 --- a/openspec/specs/cli-detect/spec.md +++ b/openspec/specs/cli-detect/spec.md @@ -270,3 +270,29 @@ A linter that ships with a language toolchain rather than being declared as a de - **WHEN** a root CI or hook config names a linter only in a comment, a step's `name`, or another field that is not a command - **THEN** `detect --json` SHALL NOT report that linter from it + +### Requirement: Detect counts the repository's own rules per linter + +`detect --json` SHALL report, for each detected linter that can host custom rules, how many custom rules the repository itself authors for it, and the evidence for that count. A custom rule is one whose logic the repository wrote: + +- ESLint: rules in a local plugin or rule directory, including plugins defined inline in the config or imported from a file in the repository. +- Biome: entries in the configuration's `plugins` list. +- pylint: modules named by `load-plugins` that live in the repository. +- flake8: checkers registered under `[flake8:local-plugins]`, or a `flake8.extension` entry point the repository publishes. + +Optional extensions shipped with the tool, third-party plugin packages, presets, and rule selection SHALL NOT count. Ruff has no custom-rule mechanism and SHALL report a count of zero. The count SHALL be derived from files on disk without executing anything, consistent with detect's other signals. + +#### Scenario: Local ESLint rules are counted + +- **WHEN** the repository has a local ESLint plugin defining six rules +- **THEN** `detect --json` SHALL report a custom-rule count of six for ESLint with the plugin's path as evidence + +#### Scenario: A built-in pylint extension does not count + +- **WHEN** `load-plugins` names only `pylint.extensions.*` modules +- **THEN** `detect --json` SHALL report a custom-rule count of zero for pylint + +#### Scenario: Ruff reports no custom rules + +- **WHEN** the repository configures Ruff, however extensively +- **THEN** `detect --json` SHALL report a custom-rule count of zero for Ruff diff --git a/openspec/specs/cli-legacy-rule-recipes/spec.md b/openspec/specs/cli-legacy-rule-recipes/spec.md new file mode 100644 index 00000000..cf5192fb --- /dev/null +++ b/openspec/specs/cli-legacy-rule-recipes/spec.md @@ -0,0 +1,109 @@ +# cli-legacy-rule-recipes Specification + +## Purpose + +Per-tool recipes that teach an agent to solve a request with the rules a linter the repository already runs ships, and to write a new rule in that linter only when the repository's own rules show how. + +## Requirements + +### Requirement: Legacy recipes use what a tool ships before anything is written + +Every `create-legacy-rule` recipe, generic or per-tool, SHALL direct the agent to first establish whether a rule the tool already ships solves the request, including the tool's configurable catch-all rules. A recipe SHALL name the catch-all rules for its tool explicitly, because a search for the request's subject does not surface a catch-all rule. + +A per-tool recipe SHALL state the check its tool always performs first: + +- Ruff: whether `banned-api` (TID251) covers the request. +- ESLint: whether a core rule covers it, with `no-restricted-syntax` and `no-restricted-imports` (and their `@typescript-eslint/` counterparts) named as the catch-alls, before any plugin. +- Biome: whether an existing rule makes the problem go away, with `noRestrictedImports` and `noRestrictedGlobals` named as the catch-alls. +- pylint: whether a shipped message or the `deprecated-modules`, `preferred-modules`, or `bad-functions` options cover it. +- flake8: whether `banned-modules` from `flake8-tidy-imports` covers it. The flake8 recipe SHALL NOT include a discovery step. + +#### Scenario: Ruff tries banned-api first + +- **WHEN** the agent follows the Ruff recipe for a request to forbid an API +- **THEN** the recipe SHALL direct it to check `banned-api` before any other step + +#### Scenario: ESLint tries a core rule before a plugin + +- **WHEN** the agent follows the ESLint recipe +- **THEN** it SHALL check core rules and the named catch-alls before considering any plugin from the curated list + +#### Scenario: flake8 goes straight to its catch-all + +- **WHEN** the agent follows the flake8 recipe +- **THEN** it SHALL be directed to `banned-modules` without a rule-discovery step + +### Requirement: Rule discovery is confirmed against the installed version + +Where a tool's rule set is too large to match a request against locally, the recipe SHALL direct the agent to find candidate rules on the tool's rules page by web search or fetch, then confirm each candidate against the installed version with the tool's own command before enabling it. The Ruff recipe SHALL use `ruff rule ` and the Biome recipe `biome explain `. + +The Ruff recipe SHALL state that a preview-only rule does not fire unless preview mode is enabled, and SHALL direct the agent to check the rule's preview status before enabling it. The pylint recipe SHALL state that `--help-msg` reports a message from an optional extension as nonexistent unless that extension is passed with `--load-plugins`. + +#### Scenario: A Ruff rule is confirmed locally + +- **WHEN** the agent finds a candidate Ruff rule on the web +- **THEN** it SHALL confirm it with `ruff rule` against the installed version before enabling it + +#### Scenario: A preview-only Ruff rule is not enabled silently + +- **WHEN** the confirmed Ruff rule is preview-only +- **THEN** the agent SHALL either enable preview mode for it with the user's knowledge or choose another rule, and SHALL NOT enable it in a configuration where it cannot fire + +#### Scenario: A pylint extension message is looked up with its extension loaded + +- **WHEN** the agent looks up a message that belongs to a pylint optional extension +- **THEN** it SHALL pass that extension with `--load-plugins` before concluding the message does not exist + +### Requirement: The ESLint recipe carries a curated plugin list + +The ESLint recipe SHALL carry a curated list of plugins, each with a one-line summary of what it covers and a link to its documentation, so the agent can choose whether to investigate further. At launch the list SHALL be `eslint-plugin-import`, `eslint-plugin-regexp`, `eslint-plugin-promise`, `eslint-plugin-n`, `eslint-plugin-react`, `eslint-plugin-react-hooks`, `eslint-plugin-jsx-a11y`, and `eslint-plugin-sonarjs`. + +The recipe SHALL direct the agent to prefer a plugin the repository already has installed. Where a listed plugin does not declare support for the repository's installed ESLint major version, the recipe SHALL say so, and for imports SHALL name `eslint-plugin-import-x` as the alternative. + +The list names rule sources, not individual rules. The recipe SHALL NOT embed a catalog of individual plugin rules. + +#### Scenario: An installed plugin is preferred + +- **WHEN** a request is covered by a plugin the repository already depends on +- **THEN** the agent SHALL use that plugin's rule rather than recommending another + +#### Scenario: Each listed plugin links to its docs + +- **WHEN** the ESLint recipe is rendered +- **THEN** every curated plugin SHALL appear with a one-line summary and a documentation URL + +### Requirement: A new legacy rule is written only from the repository's own examples + +A legacy recipe SHALL permit the agent to write a new custom rule in a linter only when the repository already has at least five custom rules for that linter, as counted by `detect --json`. The agent SHALL use those rules as its reference for structure and conventions. + +The recipes SHALL NOT teach custom-rule authoring for any tool. They SHALL NOT carry instructions for writing an ESLint rule, a pylint checker, a flake8 plugin, or a Biome plugin of any kind. Below the threshold, a request no shipped rule solves SHALL return to `route`, which sends it to a Taskless rule. + +#### Scenario: Five examples permit a new rule + +- **WHEN** no shipped rule solves the request AND `detect` counts five or more custom rules for the linter +- **THEN** the agent MAY write a new custom rule modeled on those rules + +#### Scenario: Fewer examples send the request back to Taskless + +- **WHEN** no shipped rule solves the request AND `detect` counts fewer than five custom rules for the linter +- **THEN** the agent SHALL NOT write a custom rule in that linter +- **AND** it SHALL continue down the `route` ladder to a Taskless rule + +### Requirement: A plugin the repository lacks is added only with consent + +A legacy recipe SHALL treat recommending a plugin the repository does not have installed as adding a dependency. The agent SHALL ask the user before adding it, and SHALL state the package it would add. + +#### Scenario: A missing plugin is proposed, not installed + +- **WHEN** the best existing rule belongs to a plugin the repository does not depend on +- **THEN** the agent SHALL name the package and ask before adding it + +### Requirement: A legacy rule hand-off says who runs it + +Every legacy recipe SHALL end by naming the files it changed and telling the user how to run the linter to see the rule fire. It SHALL state that `taskless check` does not run external linters. + +#### Scenario: The user is told how to see the rule fire + +- **WHEN** the agent finishes a legacy rule +- **THEN** it SHALL name the changed files and the command that runs the linter +- **AND** it SHALL state that `taskless check` does not run that linter diff --git a/openspec/specs/cli-onboard/spec.md b/openspec/specs/cli-onboard/spec.md index 5c6def68..d68a0915 100644 --- a/openspec/specs/cli-onboard/spec.md +++ b/openspec/specs/cli-onboard/spec.md @@ -317,3 +317,20 @@ The step SHALL test whether `.taskless/` is tracked, not whether it exists, beca - **WHEN** the agent is in a linked worktree, `.taskless/taskless.json` is not tracked, and the main checkout has no `.taskless/` - **THEN** the recipe SHALL have the agent go on with onboarding + +### Requirement: Onboarding uses only shipped rules from installed packages + +When onboarding sends a candidate to a legacy destination, the recipe SHALL limit the agent to enabling or configuring a rule that the linter, or a plugin package the repository already has installed, already ships. Onboarding SHALL NOT write a custom rule in any linter, regardless of how many custom rules the repository has, and SHALL NOT add a dependency. + +A candidate that no installed rule solves SHALL continue down the `route` ladder to a Taskless rule, or to the report's `cloud` tier when only a runtime rule could solve it. + +#### Scenario: A plugin that is not installed is not added + +- **WHEN** the best existing rule for a candidate belongs to a plugin the repository does not depend on +- **THEN** onboarding SHALL NOT add that plugin +- **AND** the candidate SHALL continue to a Taskless destination + +#### Scenario: Custom-rule examples do not unlock authoring during onboarding + +- **WHEN** the repository has five or more custom ESLint rules AND no shipped rule solves a candidate +- **THEN** onboarding SHALL NOT write a custom ESLint rule for it diff --git a/openspec/specs/cli-rule-routing/spec.md b/openspec/specs/cli-rule-routing/spec.md index 7ab219b8..2862d17d 100644 --- a/openspec/specs/cli-rule-routing/spec.md +++ b/openspec/specs/cli-rule-routing/spec.md @@ -139,39 +139,22 @@ failed local attempt to a service call. - **THEN** the recipe SHALL NOT invoke `remote` / the service without an explicit user confirmation step -### Requirement: Route asks the user when multiple paths fit - -The `route` recipe SHALL present the viable options to the user with their -trade-offs, rather than silently selecting one, whenever more than one destination -genuinely fits a request (most commonly both `existing` and `static`). The -trade-off framing SHALL note that `remote` consumes a generation and requires -login, so it is appropriate when a request cannot be solved locally rather than as -a default. - -#### Scenario: Both local paths viable surfaces a choice - -- **WHEN** the repository has a detected linter that fits AND the pattern is a - clean local static ast-grep rule -- **THEN** `route` SHALL present both `existing` and `static` with their - trade-offs and let the user choose - -#### Scenario: Trade-off framing names the generation cost of remote - -- **WHEN** `route` presents options that include `remote` -- **THEN** it SHALL state that `remote` consumes a generation and requires login -- **AND** SHALL frame `remote` as the path for what cannot be solved locally - ### Requirement: Existing recipe authors in the detected linter's dialect -The CLI SHALL provide a `create-legacy-rule` help recipe that instructs the agent to author a rule in a linter already detected in the repository, expressed in that tool's own dialect. The recipe SHALL direct the agent to source authoring knowledge first from the repository's own existing rules and only then from the agent's own web research. The recipe SHALL NOT embed or rely on a Taskless-maintained catalog of linter rules. +The CLI SHALL provide a `create-legacy-rule` help recipe that instructs the agent to solve a request with a linter already detected in the repository, in that tool's own dialect. The recipe SHALL direct the agent to look first for a rule the tool already ships, and to write a new custom rule only from the repository's own existing rules for that linter. The recipe MAY carry a short, curated, linked list of rule sources, such as a tool's catch-all rules or a set of well-known plugins. It SHALL NOT embed a catalog of individual linter rules. The recipe is named for the artifact it produces. "Existing" described the repository's state rather than the rule being written, which is not something an agent can address by name. #### Scenario: Repo-first knowledge sourcing -- **WHEN** the agent follows `create-legacy-rule` +- **WHEN** the agent follows `create-legacy-rule` to write a new custom rule - **THEN** it SHALL read the repository's own rules for that linter before consulting any external source +#### Scenario: A shipped rule is looked for first + +- **WHEN** the agent follows `create-legacy-rule` +- **THEN** it SHALL establish whether a rule the tool already ships solves the request before writing anything new + ### Requirement: Static recipe authors a verified local ast-grep rule The CLI SHALL provide a `create-sg-rule` help recipe that instructs the agent to author a @@ -332,3 +315,62 @@ unusable where the engine's platform binary did not resolve. - **WHEN** `route` determines whether remote generation is available - **THEN** it SHALL rely on the `ghOwner` reported by `info` rather than re-deriving the remote independently + +### Requirement: Route applies the legacy ladder in order + +For a request in a repository where `detect --json` reports a linter, the `route` recipe SHALL apply these steps in order and stop at the first that solves the request: + +1. A rule the detected linter already ships, including its catch-all rules, solves it. +2. The repository has five or more of its own custom rules for that linter, and a new one modeled on them is maintainable. +3. A Taskless static rule. +4. A Taskless runtime rule. + +Step 1 SHALL be taken whenever a detected linter could host the rule, regardless of how much custom configuration the repository has. Step 2 SHALL NOT be taken below the five-rule threshold. + +#### Scenario: A shipped rule wins over a Taskless rule + +- **WHEN** a detected linter ships a rule that solves the request +- **THEN** `route` SHALL name `create-legacy-rule` for that linter + +#### Scenario: No shipped rule and no examples falls to Taskless + +- **WHEN** no shipped rule solves the request AND the repository has fewer than five custom rules for any linter that could host it +- **THEN** `route` SHALL continue to a Taskless destination + +### Requirement: Route describes each detected linter + +The `route` recipe SHALL include, for each linter `detect` reports that can host a legacy rule, one paragraph saying what the tool is for, what the agent always checks first in it, and the recipe command to fetch for it. The paragraphs SHALL be rendered when the recipe is served, and SHALL appear only for linters detected in the repository. + +#### Scenario: Only detected linters are described + +- **WHEN** `detect` reports Ruff and no other linter +- **THEN** the served `route` recipe SHALL contain the Ruff paragraph +- **AND** SHALL NOT contain a paragraph for any other linter + +#### Scenario: A tool with no legacy recipe gets no paragraph + +- **WHEN** `detect` reports only `prettier` +- **THEN** the served `route` recipe SHALL contain no linter paragraph + +### Requirement: Route asks the user only when the ladder does not decide + +The `route` recipe SHALL present the viable options to the user with their +trade-offs, rather than silently selecting one, whenever more than one destination +genuinely fits a request. The order of the legacy ladder SHALL decide between a +legacy rule and a Taskless rule, so that pair SHALL NOT be posed as a choice. The +trade-off framing SHALL note that `remote` consumes a generation and requires +login, so it is appropriate when a request cannot be solved locally rather than as +a default. + +#### Scenario: A legacy rule and a static rule are not posed as a choice + +- **WHEN** the repository has a detected linter that ships a rule solving the + request AND the pattern is also a clean local static ast-grep rule +- **THEN** `route` SHALL name the legacy destination without asking the user to + choose between it and `create-sg-rule` + +#### Scenario: Trade-off framing names the generation cost of remote + +- **WHEN** `route` presents options that include `remote` +- **THEN** it SHALL state that `remote` consumes a generation and requires login +- **AND** SHALL frame `remote` as the path for what cannot be solved locally From 8f7795e7945431aebe8cbf6e06e124f575a6ac11 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Thu, 22 Oct 2026 18:23:52 -0700 Subject: [PATCH 3/5] ci(legacy-linter-contract): build the CLI before the suite's global setup --- .github/workflows/legacy-linter-contract.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/legacy-linter-contract.yml b/.github/workflows/legacy-linter-contract.yml index f6b78123..d560af5e 100644 --- a/.github/workflows/legacy-linter-contract.yml +++ b/.github/workflows/legacy-linter-contract.yml @@ -46,6 +46,11 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + # The suite's global setup snapshots the built CLI and refuses to start + # without one, even for a test file that never spawns it. + - name: Build + run: pnpm build + - name: Test working-directory: packages/cli env: From e4706ba7c7f28b5f297f5e76c48186867a8f58d2 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Thu, 22 Oct 2026 18:25:11 -0700 Subject: [PATCH 4/5] test(legacy-linter-contract): parse stdout only, since uvx reports installs on stderr --- .../cli/test/legacy-linter-vendor-contract.test.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/cli/test/legacy-linter-vendor-contract.test.ts b/packages/cli/test/legacy-linter-vendor-contract.test.ts index 4eafbb51..530c56ee 100644 --- a/packages/cli/test/legacy-linter-vendor-contract.test.ts +++ b/packages/cli/test/legacy-linter-vendor-contract.test.ts @@ -39,6 +39,9 @@ function run(command: string, args: string[]) { }); return { status: result.status, + // stdout alone, for output that is parsed: on a cold cache `uvx` reports + // what it installed on stderr. + stdout: result.stdout ?? "", output: `${result.stdout ?? ""}${result.stderr ?? ""}`, }; } @@ -57,7 +60,7 @@ describe.skipIf(!ENABLED)("legacy linter vendor contract", () => { describe("ruff", () => { it("reports preview status in `ruff rule` JSON, which the recipe tells the agent to check", () => { - const { status, output } = run("uvx", [ + const { status, stdout } = run("uvx", [ RUFF, "rule", "--all", @@ -65,7 +68,7 @@ describe.skipIf(!ENABLED)("legacy linter vendor contract", () => { "json", ]); expect(status).toBe(0); - const rules = JSON.parse(output) as Array>; + const rules = JSON.parse(stdout) as Array>; expect(rules.length).toBeGreaterThan(400); for (const key of ["code", "name", "preview"]) { expect(rules[0]).toHaveProperty(key); @@ -158,14 +161,14 @@ describe.skipIf(!ENABLED)("legacy linter vendor contract", () => { }); it("lists installed plugins as JSON with --bug-report", () => { - const { status, output } = run("uvx", [ + const { status, stdout } = run("uvx", [ "--from", FLAKE8, "flake8", "--bug-report", ]); expect(status).toBe(0); - expect(JSON.parse(output)).toHaveProperty("plugins"); + expect(JSON.parse(stdout)).toHaveProperty("plugins"); }); }); From 1444b0aa3be89911ee1597e6eb4c17ddfd007077 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Fri, 23 Oct 2026 01:23:46 -0700 Subject: [PATCH 5/5] fix(agent): refuse --tool only on a topic that has tool variants An unknown topic now reports itself as unknown instead of the legacy-recipe refusal, and a topic without variants ignores --tool as the resolution order already says. The check uses toolVariants(), which had no caller. --- openspec/specs/cli-agent/spec.md | 12 +++++++++++- packages/cli/src/commands/agent.ts | 10 +++++++++- packages/cli/test/agent-extensions.test.ts | 20 ++++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/openspec/specs/cli-agent/spec.md b/openspec/specs/cli-agent/spec.md index 9c959e91..7e34ee6b 100644 --- a/openspec/specs/cli-agent/spec.md +++ b/openspec/specs/cli-agent/spec.md @@ -15,7 +15,7 @@ The CLI SHALL support an `agent` subcommand that accepts at most one positional 3. Otherwise, return `.md`. 4. If none of these exists, exit with code 1 and an error message suggesting `taskless agent` for the topic index. -`--tool` SHALL refuse, with a non-zero exit and a message naming the recipe to use instead, a tool that has its own authoring recipe (`ast-grep`, `vale`) or that cannot hold a rule (`prettier`, `black`). An unrecognized tool name SHALL fall back to `.md`. +On a topic that has tool variants, `--tool` SHALL refuse, with a non-zero exit and a message naming the recipe to use instead, a tool that has its own authoring recipe (`ast-grep`, `vale`) or that cannot hold a rule (`prettier`, `black`). An unrecognized tool name SHALL fall back to `.md`. When no positional argument is provided, the subcommand SHALL print a topic index containing a one-paragraph human slug followed by a topic disambiguation table mapping topic names to their summaries. @@ -59,6 +59,16 @@ The subcommand is named for its reader. It serves agents fetching a procedure, n - **WHEN** a user runs `taskless agent create-legacy-rule --tool=vale` - **THEN** the CLI SHALL exit non-zero and name `create-vale-rule` as the recipe to use +#### Scenario: A refused tool name is ignored on a topic without tool variants + +- **WHEN** a user runs `taskless agent route --tool=vale` +- **THEN** the CLI SHALL print `route.md` and exit zero + +#### Scenario: An unknown topic is not masked by a refused tool + +- **WHEN** a user runs `taskless agent no-such-topic --tool=vale` +- **THEN** the CLI SHALL exit non-zero with the unknown-topic error, not the tool refusal + ### Requirement: Recipe files are embedded at build time Recipe files SHALL be located at `packages/cli/src/agent/` as `.md` files. The Vite build SHALL embed these files into the CLI bundle via `import.meta.glob` with raw imports. A recipe file SHALL exist for every registered command and subcommand. diff --git a/packages/cli/src/commands/agent.ts b/packages/cli/src/commands/agent.ts index 34edba6c..fd4664d8 100644 --- a/packages/cli/src/commands/agent.ts +++ b/packages/cli/src/commands/agent.ts @@ -16,6 +16,7 @@ import { LINTER_VARIABLES, getRawRecipe, getRecipe, + toolVariants, } from "../prompts/recipes"; import { withSurveyInvite } from "../survey/invite"; import { applyCliInvocation } from "../util/invocation"; @@ -267,7 +268,14 @@ export function createAgentCommand(subCommands: SubCommandsDef) { } const tool = args.tool as string | undefined; - const refusal = tool === undefined ? undefined : REFUSED_TOOLS.get(tool); + // Refusal is a statement about a topic's tool variants, so it applies + // only to a topic that has them. Elsewhere `--tool` selects nothing and + // is ignored, and an unknown topic still reports itself as unknown + // rather than hiding behind a "no legacy recipe" message. + const refusal = + tool === undefined || toolVariants(key).length === 0 + ? undefined + : REFUSED_TOOLS.get(tool); if (refusal !== undefined) { telemetry.capture("cli_agent", { topic: key }); console.error(`--tool=${tool} has no legacy recipe: ${refusal}`); diff --git a/packages/cli/test/agent-extensions.test.ts b/packages/cli/test/agent-extensions.test.ts index e66e3a8f..cf745d03 100644 --- a/packages/cli/test/agent-extensions.test.ts +++ b/packages/cli/test/agent-extensions.test.ts @@ -622,4 +622,24 @@ describe("taskless agent --tool (variant lookup)", () => { expect(result.stderr).toContain(instead); expect(result.stdout).toBe(""); }); + + it("refuses only on a topic that has tool variants", async () => { + const plain = await runCli(["agent", "route", "-d", cwd]); + const vale = await runCli(["agent", "route", "--tool=vale", "-d", cwd]); + expect(vale.exitCode).toBe(0); + expect(vale.stdout).toBe(plain.stdout); + }); + + it("reports an unknown topic as unknown, not as a refused tool", async () => { + const result = await runCli([ + "agent", + "no-such-topic", + "--tool=vale", + "-d", + cwd, + ]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("Unknown command: no-such-topic"); + expect(result.stderr).not.toContain("has no legacy recipe"); + }); });