diff --git a/.claude/skills/dart-add-unit-test/SKILL.md b/.claude/skills/dart-add-unit-test/SKILL.md new file mode 100644 index 0000000..dc27083 --- /dev/null +++ b/.claude/skills/dart-add-unit-test/SKILL.md @@ -0,0 +1,122 @@ +--- +name: dart-add-unit-test +description: Write and organize unit tests for functions, methods, and classes using `package:test`. Use when creating new logic or fixing bugs to ensure code remains correct and regression-free. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Fri, 24 Apr 2026 15:07:58 GMT +--- +# Testing Dart and Flutter Applications + +## Contents +- [Structuring Test Files](#structuring-test-files) +- [Writing Tests](#writing-tests) +- [Executing Tests](#executing-tests) +- [Test Implementation Workflow](#test-implementation-workflow) +- [Examples](#examples) + +## Structuring Test Files +Organize test files to mirror the `lib` directory structure to maintain predictability. + +* Place all test code within the `test` directory at the root of the package. +* Append `_test.dart` to the end of all test file names (e.g., `lib/src/utils.dart` should be tested in `test/src/utils_test.dart`). +* If writing integration tests, place them in an `integration_test` directory at the root of the package. + +## Writing Tests +Utilize `package:test` as the standard testing library for Dart applications. + +* Import `package:test/test.dart` (or `package:flutter_test/flutter_test.dart` for Flutter). +* Group related tests using the `group()` function to provide shared context. +* Define individual test cases using the `test()` function. +* Validate outcomes using the `expect()` function alongside matchers (e.g., `equals()`, `isTrue`, `throwsA()`). +* Write asynchronous tests using standard `async`/`await` syntax. The test runner automatically waits for the `Future` to complete. +* Manage test setup and teardown using `setUp()` and `tearDown()` callbacks. +* If testing code that relies on dependency injection, use `package:mockito` alongside `package:test` to generate mock objects, configure fixed scenarios, and verify interactions. + +## Executing Tests +Select the appropriate test runner based on the project type and test location. + +* If working on a pure Dart project, execute tests using the `dart test` command. +* If working on a Flutter project, execute tests using the `flutter test` command. +* If running integration tests, explicitly specify the directory path, as the default runner ignores it: `dart test integration_test` or `flutter test integration_test`. + +## Test Implementation Workflow + +Follow this sequential workflow when implementing new test suites. Copy the checklist to track your progress. + +### Task Progress +- [ ] 1. Create the test file in the `test/` directory, ensuring the `_test.dart` suffix. +- [ ] 2. Import `package:test/test.dart` and the target library. +- [ ] 3. Define a `main()` function. +- [ ] 4. Initialize shared resources or mocks using `setUp()`. +- [ ] 5. Write `test()` cases grouped by functionality using `group()`. +- [ ] 6. Execute the test suite using the appropriate CLI command. +- [ ] 7. **Feedback Loop**: Run test -> Review stack trace for failures -> Fix implementation or assertions -> Re-run until passing. + +## Examples + +### Standard Unit Test Suite +Demonstrates grouping, setup, synchronous, and asynchronous testing. + +```dart +import 'package:test/test.dart'; +import 'package:my_package/calculator.dart'; + +void main() { + group('Calculator', () { + late Calculator calc; + + setUp(() { + calc = Calculator(); + }); + + test('adds two numbers correctly', () { + expect(calc.add(2, 3), equals(5)); + }); + + test('handles asynchronous operations', () async { + final result = await calc.fetchRemoteValue(); + expect(result, isNotNull); + expect(result, greaterThan(0)); + }); + }); +} +``` + +### Mocking with Mockito +Demonstrates configuring a mock object for dependency injection testing. + +```dart +import 'package:test/test.dart'; +import 'package:mockito/mockito.dart'; +import 'package:mockito/annotations.dart'; +import 'package:my_package/api_client.dart'; +import 'package:my_package/data_service.dart'; + +// Generate the mock using build_runner: dart run build_runner build +@GenerateNiceMocks([MockSpec()]) +import 'data_service_test.mocks.dart'; + +void main() { + group('DataService', () { + late MockApiClient mockApiClient; + late DataService dataService; + + setUp(() { + mockApiClient = MockApiClient(); + dataService = DataService(apiClient: mockApiClient); + }); + + test('returns parsed data on successful API call', () async { + // Configure the mock + when(mockApiClient.get('/data')).thenAnswer((_) async => '{"id": 1}'); + + // Execute the system under test + final result = await dataService.fetchData(); + + // Verify outcomes and interactions + expect(result.id, equals(1)); + verify(mockApiClient.get('/data')).called(1); + }); + }); +} +``` diff --git a/.claude/skills/dart-build-cli-app/SKILL.md b/.claude/skills/dart-build-cli-app/SKILL.md new file mode 100644 index 0000000..239a892 --- /dev/null +++ b/.claude/skills/dart-build-cli-app/SKILL.md @@ -0,0 +1,185 @@ +--- +name: dart-build-cli-app +description: Entrypoint structure, exit codes, cross-platform scripts. Use when building command line utilities, scripts, or applications. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Fri, 04 May 2026 17:41:00 GMT +--- +# Building Dart CLI Applications + +## Contents +- [Project Setup & Architecture](#project-setup--architecture) +- [Argument Parsing & Command Routing](#argument-parsing--command-routing) +- [Execution & Error Handling](#execution--error-handling) +- [Testing CLI Applications](#testing-cli-applications) +- [Compilation & Distribution](#compilation--distribution) +- [Workflows](#workflows) +- [Examples](#examples) + +## Project Setup & Architecture + +Initialize new CLI projects using the official Dart template to ensure standard directory structures. + +* Run `dart create -t cli ` to scaffold a console application with basic argument parsing. +* Place executable entry points (files containing `main()`) exclusively in the `bin/` directory. +* Place internal implementation logic in `lib/src/` and expose public APIs via `lib/.dart`. +* Enforce formatting in CI environments by running `dart format . --set-exit-if-changed`. This returns exit code 1 if formatting violations exist. + +## Argument Parsing & Command Routing + +Import the `args` package to manage command-line arguments, flags, and subcommands. + +* If building a simple script: Use `ArgParser` directly to define flags (`addFlag`) and options (`addOption`). +* If building a complex, multi-command CLI (like `git`): Implement `CommandRunner` and extend `Command` for each subcommand. +* Define global arguments on the `CommandRunner.argParser` and command-specific arguments on the individual `Command.argParser`. +* Catch `UsageException` to gracefully handle invalid arguments and display the automatically generated help text. +* **Validate Help Text Accuracy**: Ensure the help text provides all necessary information to run the tool. If the help text references a compiled executable name, and the user needs to add it to their PATH to run it that way, provide clear instructions on how to do so in the help text or description. + +## Execution & Error Handling + +Leverage the `io` and `stack_trace` packages to build robust, production-ready CLI tools. + +* Use the `io` package's `ExitCode` enum to return standard POSIX exit codes (e.g., `ExitCode.success.code`, `ExitCode.usage.code`). +* Use `sharedStdIn` from the `io` package if multiple asynchronous listeners need sequential access to standard input. +* Wrap the application execution in `Chain.capture()` from the `stack_trace` package to track asynchronous stack chains. +* Format output stack traces using `Trace.terse` or `Chain.terse` to strip noisy core library frames and present readable errors to the user. +* **Do not swallow exceptions** in lower-level logic or storage classes unless recovery is possible. Let them bubble up or rethrow them so higher-level commands know operations failed. +* **Fail fast and with non-zero exit codes**: Ensure operation failures result in descriptive error messages to `stderr` and appropriate non-zero exit codes (e.g., using `exit(1)` or triggering a 64 exit code after a caught `UsageException`). + +## Testing CLI Applications + +> [!IMPORTANT] +> **All new commands and significant features must be covered by automated tests.** Manual verification is not sufficient for testing logic. However, manual verification of help text and user experience (UX) is still required to ensure the interface is intuitive and correct. + +Use `test_process` and `test_descriptor` to write high-fidelity integration tests for your CLI. + +* Define expected filesystem states using `test_descriptor` (`d.dir`, `d.file`). +* Create the mock filesystem before execution using `await d.Descriptor.create()`. +* Spawn the CLI process using `TestProcess.start('dart', ['run', 'bin/cli.dart', ...args])`. +* Validate standard output and error streams using `StreamQueue` matchers (e.g., `emitsThrough`, `emits`). +* Assert the final exit code using `await process.shouldExit(0)`. +* Validate resulting filesystem mutations using `await d.Descriptor.validate()`. + +## Compilation & Distribution + +Select the appropriate compilation target based on your distribution requirements. + +* **If testing locally during development:** Use `dart run bin/cli.dart`. This uses the JIT compiler for rapid iteration. +* **If bundling code assets and dynamic libraries:** Use `dart build cli`. This runs build hooks and outputs to `build/cli/_/bundle/`. +* **If distributing a standalone native executable:** Use `dart compile exe bin/cli.dart -o `. This bundles the Dart runtime and machine code into a single file. +* **If distributing multiple apps with strict disk space limits:** Use `dart compile aot-snapshot bin/cli.dart`. Run the resulting `.aot` file using `dartaotruntime`. + +
+Cross-Compilation Targets (Linux Only) + +Dart supports cross-compiling to Linux from macOS, Windows, or Linux hosts. +Use the `--target-os` and `--target-arch` flags with `dart compile exe` or `dart compile aot-snapshot`. + +* `--target-os=linux` (Only Linux is currently supported as a cross-compilation target) +* `--target-arch=arm64` (64-bit ARM) +* `--target-arch=x64` (x86-64) +* `--target-arch=arm` (32-bit ARM) +* `--target-arch=riscv64` (64-bit RISC-V) + +Example: `dart compile exe --target-os=linux --target-arch=arm64 bin/cli.dart` +
+ +## Workflows + +### Task Progress: Implement a New CLI Command +- [ ] Create a new class extending `Command` in `lib/src/commands/`. +- [ ] Define the `name` and `description` properties. +- [ ] Register command-specific flags in the constructor using `argParser.addFlag()` or `argParser.addOption()`. +- [ ] Implement the `run()` method with the core logic. +- [ ] Register the new command in the `CommandRunner` instance in `bin/cli.dart` using `addCommand()`. +- [ ] Create tests for the new command in the `test/` directory using `test_process` or standard tests. +- [ ] Run validator -> Execute `dart run bin/cli.dart help ` to verify help text generation. +- [ ] Verify final UX: Compile the application using `dart compile exe` and run the resulting executable to verify the target user experience (e.g., `./bin/cli `). + +### Task Progress: Compile and Release Native Executable +- [ ] Run validator -> Execute `dart format . --set-exit-if-changed` to ensure code formatting. +- [ ] Run validator -> Execute `dart analyze` to ensure no static analysis errors. +- [ ] Run validator -> Execute `dart test` to pass all integration tests. +- [ ] Compile for host OS: `dart compile exe bin/cli.dart -o build/cli-host` +- [ ] Compile for Linux (if host is macOS/Windows): `dart compile exe --target-os=linux --target-arch=x64 bin/cli.dart -o build/cli-linux-x64` + +## Examples + +### Example: CommandRunner Implementation + +```dart +import 'dart:io'; +import 'package:args/command_runner.dart'; +import 'package:stack_trace/stack_trace.dart'; + +class CommitCommand extends Command { + @override + final String name = 'commit'; + @override + final String description = 'Record changes to the repository.'; + + CommitCommand() { + argParser.addFlag('all', abbr: 'a', help: 'Commit all changed files.'); + } + + @override + Future run() async { + final commitAll = argResults?['all'] as bool? ?? false; + print('Committing... (All: $commitAll)'); + } +} + +void main(List args) { + Chain.capture(() async { + final runner = CommandRunner('dgit', 'Distributed version control.') + ..addCommand(CommitCommand()); + + await runner.run(args); + }, onError: (error, chain) { + if (error is UsageException) { + stderr.writeln(error.message); + stderr.writeln(error.usage); + exit(64); // ExitCode.usage.code + } else { + stderr.writeln('Fatal error: $error'); + stderr.writeln(chain.terse); + exit(1); + } + }); +} +``` + +### Example: Integration Testing with Subprocesses + +```dart +import 'package:test/test.dart'; +import 'package:test_process/test_process.dart'; +import 'package:test_descriptor/test_descriptor.dart' as d; + +void main() { + test('CLI formats output correctly and modifies filesystem', () async { + // 1. Setup mock filesystem + await d.dir('project', [ + d.file('config.json', '{"key": "value"}') + ]).create(); + + // 2. Spawn the CLI process + final process = await TestProcess.start( + 'dart', + ['run', 'bin/cli.dart', 'process', '--path', '${d.sandbox}/project'] + ); + + // 3. Validate stdout stream + await expectLater(process.stdout, emitsThrough('Processing complete.')); + + // 4. Validate exit code + await process.shouldExit(0); + + // 5. Validate filesystem mutations + await d.dir('project', [ + d.file('config.json', '{"key": "value"}'), + d.file('output.log', 'Success') + ]).validate(); + }); +} +``` diff --git a/.claude/skills/dart-collect-coverage/SKILL.md b/.claude/skills/dart-collect-coverage/SKILL.md new file mode 100644 index 0000000..60dad77 --- /dev/null +++ b/.claude/skills/dart-collect-coverage/SKILL.md @@ -0,0 +1,141 @@ +--- +name: dart-collect-coverage +description: Collect coverage using the coverage packge and create an LCOV report +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Fri, 24 Apr 2026 15:14:32 GMT +--- +# Implementing Dart and Flutter Test Coverage + +## Contents +- [Testing Fundamentals](#testing-fundamentals) +- [Coverage Directives](#coverage-directives) +- [Workflow: Configuring and Generating Coverage Reports](#workflow-configuring-and-generating-coverage-reports) +- [Workflow: Advanced Manual Coverage Collection](#workflow-advanced-manual-coverage-collection) +- [Examples](#examples) + +## Testing Fundamentals + +Structure your test suites using the standard Dart testing paradigms. Use `package:test` for Dart projects and `flutter_test` for Flutter projects. + +- **Unit Tests:** Verify individual functions, methods, or classes. +- **Component/Widget Tests:** Verify component behavior, layout, and interaction using mock objects (`package:mockito`). +- **Integration Tests:** Verify entire app flows on simulated or real devices. + +## Coverage Directives + +Exclude specific lines, blocks, or entire files from coverage metrics using inline comments. Pass the `--check-ignore` flag during formatting to enforce these directives. + +- Ignore a single line: `// coverage:ignore-line` +- Ignore a block of code: `// coverage:ignore-start` and `// coverage:ignore-end` +- Ignore an entire file: `// coverage:ignore-file` + +## Workflow: Configuring and Generating Coverage Reports + +Follow this sequential workflow to add the coverage package, execute tests, and generate an LCOV report. + +**Task Progress Checklist:** +- [ ] 1. Add `coverage` as a `dev_dependency`. +- [ ] 2. Execute the automated coverage script. +- [ ] 3. Validate the LCOV output. + +### 1. Add Dependencies +Add the `coverage` package as a `dev_dependency` to your project. Do not add it to standard dependencies. + +If working in a standard Dart project: +```bash +dart pub add dev:coverage +``` + +If working in a Flutter project: +```bash +flutter pub add dev:coverage +``` + +### 2. Collect Coverage and Generate LCOV +Use the bundled `test_with_coverage` script. This script automatically runs all tests, collects the JSON coverage data from the Dart VM, and formats it into an LCOV report. + +```bash +dart run coverage:test_with_coverage +``` +*Note: If working within a Dart workspace (monorepo), specify the test directories explicitly (e.g., `dart run coverage:test_with_coverage -- pkgs/foo/test pkgs/bar/test`).* + +### 3. Feedback Loop: Validate Output +**Run validator -> review errors -> fix:** +1. Verify that the `coverage/` directory was created in the project root. +2. Ensure `coverage/coverage.json` (raw data) and `coverage/lcov.info` (formatted report) exist. +3. If coverage is missing for specific files, ensure they are imported and executed by your test files, or add `// coverage:ignore-file` if they are intentionally excluded. + +## Workflow: Advanced Manual Coverage Collection + +If you require granular control over the VM service, isolate pausing, or need branch/function-level coverage, use the manual collection workflow. + +**Task Progress Checklist:** +- [ ] 1. Run tests with VM service enabled. +- [ ] 2. Collect raw JSON coverage. +- [ ] 3. Format JSON to LCOV. + +### 1. Run Tests with VM Service +Execute tests while pausing isolates on exit and exposing the VM service on a specific port (e.g., 8181). + +```bash +dart run --pause-isolates-on-exit --disable-service-auth-codes --enable-vm-service=8181 test & +``` + +### 2. Collect Raw Coverage +Extract the coverage data from the running VM service and output it to a JSON file. + +```bash +dart run coverage:collect_coverage --wait-paused --uri=http://127.0.0.1:8181/ -o coverage/coverage.json --resume-isolates +``` +*Optional: Append `--function-coverage` and `--branch-coverage` to gather deeper metrics (requires Dart VM 2.17.0+).* + +### 3. Format to LCOV +Convert the raw JSON data into the standard LCOV format. + +```bash +dart run coverage:format_coverage --packages=.dart_tool/package_config.json --lcov -i coverage/coverage.json -o coverage/lcov.info --check-ignore +``` + +## Examples + +### Example: `pubspec.yaml` Configuration +Ensure your `pubspec.yaml` reflects the `coverage` package strictly under `dev_dependencies`. + +```yaml +name: my_dart_app +environment: + sdk: ^3.0.0 + +dependencies: + path: ^1.8.0 + +dev_dependencies: + test: ^1.24.0 + coverage: ^1.15.0 +``` + +### Example: Applying Ignore Directives +Use ignore directives to prevent generated code or untestable edge cases from lowering coverage scores. + +```dart +// coverage:ignore-file +import 'package:meta/meta.dart'; + +class SystemConfig { + final String env; + + SystemConfig(this.env); + + // coverage:ignore-start + void legacyInit() { + print('Deprecated initialization'); + } + // coverage:ignore-end + + bool isProduction() { + if (env == 'prod') return true; + return false; // coverage:ignore-line + } +} +``` diff --git a/.claude/skills/dart-fix-runtime-errors/SKILL.md b/.claude/skills/dart-fix-runtime-errors/SKILL.md new file mode 100644 index 0000000..1a7db85 --- /dev/null +++ b/.claude/skills/dart-fix-runtime-errors/SKILL.md @@ -0,0 +1,166 @@ +--- +name: dart-fix-runtime-errors +description: Uses get_runtime_errors and lsp to fetch an active stack trace, locate the failing line, apply a fix, and verify resolution via hot_reload. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Fri, 24 Apr 2026 15:13:22 GMT +--- +# Resolving Dart Static Analysis Errors + +## Contents +- [Core Concepts & Guidelines](#core-concepts--guidelines) + - [Type System & Soundness](#type-system--soundness) + - [Null Safety](#null-safety) + - [Error Handling](#error-handling) +- [Workflows](#workflows) + - [Workflow: Static Analysis Resolution](#workflow-static-analysis-resolution) +- [Examples](#examples) + +## Core Concepts & Guidelines + +### Type System & Soundness +Enforce Dart's sound type system to prevent runtime invalid states. + +* **Method Overrides:** Maintain sound return types (covariant) and parameter types (contravariant). Never tighten a parameter type in a subclass unless explicitly marked with the `covariant` keyword. +* **Generics & Collections:** Add explicit type annotations to generic classes (e.g., `List`, `Map`). Never assign a `List` to a typed list (e.g., `List`). +* **Downcasting:** Avoid implicit downcasts from `dynamic`. Use explicit casts (e.g., `as List`) when necessary, but ensure the underlying runtime type matches to prevent `TypeError` exceptions. +* **Strict Casts:** Enable `strict-casts: true` in `analysis_options.yaml` under `analyzer: language:` to force explicit casting and catch implicit downcast errors at compile time. + +### Null Safety +Eliminate static errors related to null safety by correctly managing variable initialization and nullability. + +* **Modifiers:** Apply `?` for nullable types, `!` for null assertions, and `required` for named parameters that cannot be null. +* **Late Initialization:** Use the `late` keyword for non-nullable variables guaranteed to be initialized before use. Apply this specifically to top-level or instance variables where Dart's control flow analysis cannot definitively prove initialization. +* **Wildcards:** Use the `_` wildcard variable (Dart 3.7+) for non-binding local variables or parameters to avoid unused variable warnings. + +### Error Handling +Distinguish between recoverable exceptions and unrecoverable errors. + +* **Catching:** Catch `Exception` subtypes for recoverable failures. +* **Errors:** Never explicitly catch `Error` or its subtypes (e.g., `TypeError`, `ArgumentError`). Errors indicate programming bugs that must be fixed, not caught. Enforce this by enabling the `avoid_catching_errors` linter rule. +* **Rethrowing:** Use `rethrow` inside a `catch` block to propagate an exception while preserving its original stack trace. + +## Workflows + +### Workflow: Static Analysis Resolution + +Use this sequential workflow to identify, fix, and verify static analysis errors in a Dart project. Copy the checklist to track your progress. + +**Task Progress:** +- [ ] 1. Run static analyzer. +- [ ] 2. Apply automated fixes. +- [ ] 3. Resolve remaining errors manually. +- [ ] 4. Verify fixes (Feedback Loop). + +**1. Run static analyzer** +Execute the Dart analyzer to identify all static errors in the target directory or file. +```bash +dart analyze . --fatal-infos +``` + +**2. Apply automated fixes** +Use the `dart fix` tool to automatically resolve standard linting and analysis issues. +```bash +# Preview changes +dart fix --dry-run +# Apply changes +dart fix --apply +``` + +**3. Resolve remaining errors manually** +Review the remaining analyzer output and apply conditional logic based on the error type: + +* **If the error is a Null Safety issue (e.g., "Property cannot be accessed on a nullable receiver"):** + * Verify if the variable can logically be null. + * If yes, use optional chaining (`?.`) or provide a fallback (`??`). + * If no, and initialization is guaranteed elsewhere, mark the declaration with `late`. +* **If the error is a Type Mismatch (e.g., "The argument type 'List' can't be assigned..."):** + * Trace the variable's initialization. + * Add explicit generic type annotations to the instantiation (e.g., `[]` instead of `[]`). +* **If the error is an Invalid Override (e.g., "The parameter type doesn't match the overridden method"):** + * Widen the parameter type to match the superclass, OR + * Add the `covariant` keyword to the parameter if tightening the type is intentionally required by the domain logic. + +**4. Verify fixes (Feedback Loop)** +Run the validator. Review errors. Fix. +```bash +dart analyze . +dart test +``` +* **If `dart analyze` reports errors:** Return to Step 3. +* **If `dart test` fails with a `TypeError`:** You have introduced an invalid explicit cast (`as T`) or accessed an uninitialized `late` variable. Locate the runtime failure and correct the type hierarchy or initialization order. + +## Examples + +### Example: Fixing Dynamic List Assignments +**Input (Fails Static Analysis):** +```dart +void printInts(List a) => print(a); + +void main() { + final list = []; // Inferred as List + list.add(1); + list.add(2); + printInts(list); // Error: List can't be assigned to List +} +``` + +**Output (Passes Static Analysis):** +```dart +void printInts(List a) => print(a); + +void main() { + final list = []; // Explicitly typed + list.add(1); + list.add(2); + printInts(list); +} +``` + +### Example: Fixing Method Overrides (Contravariance) +**Input (Fails Static Analysis):** +```dart +class Animal { + void chase(Animal a) {} +} + +class Cat extends Animal { + @override + void chase(Mouse a) {} // Error: Tightening parameter type +} +``` + +**Output (Passes Static Analysis):** +```dart +class Animal { + void chase(Animal a) {} +} + +class Cat extends Animal { + @override + void chase(covariant Mouse a) {} // Explicitly marked covariant +} +``` + +### Example: Fixing Null Safety with `late` +**Input (Fails Static Analysis):** +```dart +class Thermometer { + String temperature; // Error: Non-nullable instance field must be initialized + + void read() { + temperature = '20C'; + } +} +``` + +**Output (Passes Static Analysis):** +```dart +class Thermometer { + late String temperature; // Defers initialization check to runtime + + void read() { + temperature = '20C'; + } +} +``` diff --git a/.claude/skills/dart-generate-test-mocks/SKILL.md b/.claude/skills/dart-generate-test-mocks/SKILL.md new file mode 100644 index 0000000..fcd6d8b --- /dev/null +++ b/.claude/skills/dart-generate-test-mocks/SKILL.md @@ -0,0 +1,155 @@ +--- +name: dart-generate-test-mocks +description: Define and generate mock objects for external dependencies using `package:mockito` and `build_runner`. Use when unit testing classes that depend on complex external services like APIs or databases. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Fri, 24 Apr 2026 15:13:58 GMT +--- +# Testing and Mocking Dart Applications + +## Contents +- [Structuring Code for Testability](#structuring-code-for-testability) +- [Managing Dependencies](#managing-dependencies) +- [Generating Mocks](#generating-mocks) +- [Implementing Unit Tests](#implementing-unit-tests) +- [Workflow: Creating and Running Mocked Tests](#workflow-creating-and-running-mocked-tests) +- [Examples](#examples) + +## Structuring Code for Testability +Design Dart classes to support dependency injection. Isolate complex external dependencies (like API clients or databases) so they can be replaced with mock objects during testing. + +- Inject external services (e.g., `http.Client`) through class constructors. +- Represent URLs strictly as `Uri` objects using `Uri.parse(string)`. +- Utilize Dart's object-oriented features (classes, mixins) to define clear interfaces for external interactions. + +## Managing Dependencies +Configure the `pubspec.yaml` file with the necessary testing and code generation packages. + +- Add runtime dependencies (e.g., `package:http`) using `dart pub add http`. +- Add testing dependencies using `dart pub add dev:test dev:mockito dev:build_runner`. +- Import HTTP libraries with a prefix to avoid namespace collisions: `import 'package:http/http.dart' as http;`. + +## Generating Mocks +Use `package:mockito` and `build_runner` to automatically generate mock classes for fixed scenarios and behavior verification. + +- Always use the `@GenerateNiceMocks` annotation (preferable to `@GenerateMocks` to avoid missing stub exceptions). +- Place the annotation in the test file, passing a list of `MockSpec()` objects. +- Import the generated file using the `.mocks.dart` extension. +- Execute `build_runner` to generate the mock files: `dart run build_runner build`. + +## Implementing Unit Tests +Isolate the system under test using the generated mock objects. Use `package:test` to structure the test suite. + +- **Stubbing:** Configure mock behavior before interacting with the system under test. + - Use `when(mock.method()).thenReturn(value)` for synchronous methods. + - **CRITICAL:** Always use `thenAnswer((_) async => value)` for methods returning a `Future` or `Stream`. Never use `thenReturn` for asynchronous returns. +- **Verification:** Assert that the system under test interacted with the mock object correctly. + - Use `verify(mock.method()).called(1)` to check exact invocation counts. + - Use argument matchers like `any`, `anyNamed`, or `captureAny` for flexible verification. + +## Workflow: Creating and Running Mocked Tests + +Use the following checklist to implement and verify mocked unit tests. + +### Task Progress +- [ ] 1. Identify the external dependency to mock (e.g., `http.Client`). +- [ ] 2. Inject the dependency into the target class constructor. +- [ ] 3. Create a test file (e.g., `target_test.dart`) and add `@GenerateNiceMocks([MockSpec()])`. +- [ ] 4. Add the `part` or `import` directive for the generated `.mocks.dart` file. +- [ ] 5. Run `dart run build_runner build` to generate the mock classes. +- [ ] 6. Write the test cases using `group()` and `test()`. +- [ ] 7. Stub required behaviors using `when()`. +- [ ] 8. Execute the target method. +- [ ] 9. Verify interactions using `verify()` and assert outcomes using `expect()`. +- [ ] 10. Run the test suite using `dart test`. + +### Feedback Loop: Test Failures +If tests fail or `build_runner` encounters errors: +1. **Run validator:** Execute `dart test` or `dart run build_runner build`. +2. **Review errors:** Check for missing stubs, mismatched argument matchers, or syntax errors in the generated files. +3. **Fix:** + - If a mock method throws an unexpected null error, ensure you used `@GenerateNiceMocks`. + - If an async stub throws an `ArgumentError`, change `thenReturn` to `thenAnswer`. + - If `build_runner` fails, ensure the `.mocks.dart` import matches the file name exactly. +4. Repeat until all tests pass. + +## Examples + +### High-Fidelity Mocking and Testing Example + +**1. System Under Test (`lib/api_service.dart`)** +```dart +import 'dart:convert'; +import 'package:http/http.dart' as http; + +class ApiService { + final http.Client client; + + ApiService(this.client); + + Future fetchData(String urlString) async { + final uri = Uri.parse(urlString); + final response = await client.get(uri); + + if (response.statusCode == 200) { + return jsonDecode(response.body)['data']; + } else { + throw Exception('Failed to load data'); + } + } +} +``` + +**2. Test Implementation (`test/api_service_test.dart`)** +```dart +import 'package:test/test.dart'; +import 'package:mockito/annotations.dart'; +import 'package:mockito/mockito.dart'; +import 'package:http/http.dart' as http; +import 'package:my_app/api_service.dart'; + +// Generate the mock class for http.Client +@GenerateNiceMocks([MockSpec()]) +import 'api_service_test.mocks.dart'; + +void main() { + group('ApiService', () { + late ApiService apiService; + late MockClient mockHttpClient; + + setUp(() { + mockHttpClient = MockClient(); + apiService = ApiService(mockHttpClient); + }); + + test('returns data if the http call completes successfully', () async { + // Arrange: Stub the async HTTP GET request using thenAnswer + when(mockHttpClient.get(any)).thenAnswer( + (_) async => http.Response('{"data": "Success"}', 200), + ); + + // Act + final result = await apiService.fetchData('https://api.example.com/data'); + + // Assert + expect(result, 'Success'); + + // Verify the mock was called with the correct Uri + verify(mockHttpClient.get(Uri.parse('https://api.example.com/data'))).called(1); + }); + + test('throws an exception if the http call completes with an error', () { + // Arrange + when(mockHttpClient.get(any)).thenAnswer( + (_) async => http.Response('Not Found', 404), + ); + + // Act & Assert + expect( + apiService.fetchData('https://api.example.com/data'), + throwsException, + ); + }); + }); +} +``` diff --git a/.claude/skills/dart-migrate-to-checks-package/SKILL.md b/.claude/skills/dart-migrate-to-checks-package/SKILL.md new file mode 100644 index 0000000..dc39814 --- /dev/null +++ b/.claude/skills/dart-migrate-to-checks-package/SKILL.md @@ -0,0 +1,118 @@ +--- +name: dart-migrate-to-checks-package +description: Replace the usage of `expect` and similar functions from `package:matcher` to `package:checks` equivalents. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Fri, 24 Apr 2026 15:15:22 GMT +--- +# Migrating Dart Tests to Package Checks + +## Contents +- [Dependency Management](#dependency-management) +- [Syntax Migration Guidelines](#syntax-migration-guidelines) +- [Utilizing Dart MCP Tools](#utilizing-dart-mcp-tools) +- [Migration Workflow](#migration-workflow) +- [Examples](#examples) + +## Dependency Management +Manage dependencies using the Dart Tooling MCP Server `pub` tool or standard CLI commands. + +- Add `package:checks` as a `dev_dependency` using `dart pub add dev:checks`. +- Remove `package:matcher` if it is explicitly listed in the `pubspec.yaml` (note: it is often transitively included by `package:test`, which is fine). +- Import `package:checks/checks.dart` in all test files undergoing migration. + +## Syntax Migration Guidelines +Transition test assertions from the `package:matcher` syntax to the literate API provided by `package:checks`. + +- **Basic Equality:** Replace `expect(actual, equals(expected))` or `expect(actual, expected)` with `check(actual).equals(expected)`. +- **Type Checking:** Replace `expect(actual, isA())` with `check(actual).isA()`. +- **Property Extraction:** Replace `expect(actual.property, expected)` with `check(actual).has((a) => a.property, 'property name').equals(expected)`. +- **Cascades for Multiple Checks:** Use Dart's cascade operator (`..`) to chain multiple expectations on a single subject. +- **Asynchronous Expectations:** + - If checking a `Future`, `await` the `check` call: `await check(someFuture).completes((r) => r.equals(expected));`. + - If checking a `Stream`, wrap it in a `StreamQueue` for multiple checks, or use `.withQueue` for single/broadcast checks. + +## Migration Workflow + +Copy and use the following checklist to track progress when migrating a test suite: + +- [ ] **Task Progress** + - [ ] Add `package:checks` as a dev dependency. + - [ ] Identify all test files using `package:matcher` (`expect` calls). + - [ ] Import `package:checks/checks.dart` in target test files. + - [ ] Rewrite all `expect(...)` statements to `check(...)` statements. + - [ ] Run static analyzer (`analyze_files`). + - [ ] Run tests (`run_tests`). + +### Feedback Loop: Static Analysis +1. Run the `analyze_files` tool on the modified test directories. +2. Review any static analysis warnings or errors (e.g., missing imports, incorrect generic types on `isA`, unawaited futures). +3. Fix the warnings. +4. Repeat until the analyzer returns zero issues. + +### Feedback Loop: Test Validation +1. Run the `run_tests` tool. +2. If tests fail, review the failure output. `package:checks` provides detailed context (e.g., `Which: has length of <2>`). +3. Adjust the `check()` expectations or the underlying code to resolve the failure. +4. Repeat until all tests pass. + +## Examples + +### Basic Assertions +**Input (`matcher`):** +```dart +expect(someList.length, 1); +expect(someString, startsWith('a')); +expect(someObject, isA()); +``` + +**Output (`checks`):** +```dart +check(someList).length.equals(1); +check(someString).startsWith('a'); +check(someObject).isA(); +``` + +### Composed Expectations +**Input (`matcher`):** +```dart +expect('foo,bar,baz', allOf([ + contains('foo'), + isNot(startsWith('bar')), + endsWith('baz') +])); +``` + +**Output (`checks`):** +```dart +check('foo,bar,baz') + ..contains('foo') + ..not((s) => s.startsWith('bar')) + ..endsWith('baz'); +``` + +### Asynchronous Futures +**Input (`matcher`):** +```dart +expect(Future.value(10), completion(equals(10))); +expect(Future.error('oh no'), throwsA(equals('oh no'))); +``` + +**Output (`checks`):** +```dart +await check(Future.value(10)).completes((it) => it.equals(10)); +await check(Future.error('oh no')).throws().equals('oh no'); +``` + +### Asynchronous Streams +**Input (`matcher`):** +```dart +var stdout = StreamQueue(Stream.fromIterable(['Ready', 'Go'])); +await expectLater(stdout, emitsThrough('Ready')); +``` + +**Output (`checks`):** +```dart +var stdout = StreamQueue(Stream.fromIterable(['Ready', 'Go'])); +await check(stdout).emitsThrough((it) => it.equals('Ready')); +``` diff --git a/.claude/skills/dart-resolve-package-conflicts/SKILL.md b/.claude/skills/dart-resolve-package-conflicts/SKILL.md new file mode 100644 index 0000000..9a7ffdc --- /dev/null +++ b/.claude/skills/dart-resolve-package-conflicts/SKILL.md @@ -0,0 +1,116 @@ +--- +name: dart-resolve-package-conflicts +description: Workflow for fixing package version conflicts. Use this when `pub get` fails due to incompatible package versions. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Fri, 24 Apr 2026 15:11:14 GMT +--- +# Managing Dart Dependencies + +## Contents +- [Core Concepts](#core-concepts) +- [Version Constraints](#version-constraints) +- [Workflow: Auditing Dependencies](#workflow-auditing-dependencies) +- [Workflow: Upgrading Dependencies](#workflow-upgrading-dependencies) +- [Workflow: Resolving Version Conflicts](#workflow-resolving-version-conflicts) +- [Examples](#examples) + +## Core Concepts + +Dart enforces a strict single-version rule for dependencies: a project and all its transitive dependencies must resolve to a single, shared version of any given package. This prevents runtime type mismatches but introduces the risk of "version lock." + +To mitigate version lock, Dart relies on version constraints rather than pinned versions in the `pubspec.yaml`. The `pubspec.lock` file maintains the exact resolved versions for reproducible builds. + +Understand the output columns of `dart pub outdated`: +* **Current:** The version currently recorded in `pubspec.lock`. +* **Upgradable:** The latest version allowed by the constraints in `pubspec.yaml`. `dart pub upgrade` resolves to this. +* **Resolvable:** The absolute latest version that can be resolved when factoring in all other dependencies in the project. +* **Latest:** The latest published version of the package (excluding prereleases). + +## Version Constraints + +* **Use Caret Syntax:** Always use caret syntax (e.g., `^1.2.3`) for dependencies in `pubspec.yaml`. This allows `pub` to select newer, non-breaking versions (up to, but not including, the next major version) during resolution. +* **Tighten Dev Dependencies:** Set the lower bound of `dev_dependencies` to the exact version currently used. This reduces resolution complexity and prevents older, incompatible dev tools from being selected. +* **Enforce Lockfiles in CI:** Use `dart pub get --enforce-lockfile` in CI/CD pipelines to ensure the exact versions tested locally are used in production. + +## Workflow: Auditing Dependencies + +Run this workflow periodically to identify stale packages that may impact stability or performance. + +**Task Progress:** +- [ ] Run `dart pub outdated`. +- [ ] Review the **Upgradable** column to identify packages that can be updated without modifying `pubspec.yaml`. +- [ ] Review the **Resolvable** column to identify packages that require constraint modifications in `pubspec.yaml` to update. +- [ ] Identify any packages marked as retracted or discontinued. + +## Workflow: Upgrading Dependencies + +Use conditional logic based on the audit results to upgrade dependencies. + +**Task Progress:** +- [ ] **If updating to "Upgradable" versions:** + - [ ] Run `dart pub upgrade`. + - [ ] Run `dart pub upgrade --tighten` to automatically update the lower bounds in `pubspec.yaml` to match the newly resolved versions. +- [ ] **If updating to "Resolvable" versions (Major updates):** + - [ ] Manually edit `pubspec.yaml` to bump the version constraint to match the "Resolvable" column (e.g., change `^0.11.0` to `^0.12.1`). + - [ ] Run `dart pub upgrade` to resolve the new constraints and update `pubspec.lock`. +- [ ] **Feedback Loop:** + - [ ] Run `dart analyze` -> review errors -> fix breaking API changes. + - [ ] Run `dart test` -> review failures -> fix regressions. + +## Workflow: Resolving Version Conflicts + +When `pub` cannot find a set of concrete versions that satisfy all constraints, or when dealing with a retracted package version, manipulate the lockfile surgically. + +**NEVER** delete the entire `pubspec.lock` file and run `dart pub get`. This causes uncontrolled upgrades across the entire dependency graph. + +**Task Progress:** +- [ ] Open `pubspec.lock`. +- [ ] Locate the specific YAML block for the conflicting or retracted package. +- [ ] Delete ONLY that package's entry from the lockfile. +- [ ] Run `dart pub get` to fetch the newest compatible, non-retracted version for that specific package. +- [ ] **Feedback Loop:** + - [ ] Run `dart pub deps` -> verify the dependency graph resolves correctly. + - [ ] If resolution fails, identify the transitive dependency causing the lock, update its constraint in `pubspec.yaml`, and retry. + +## Examples + +### Tightening Constraints +When `dart pub outdated` shows a package is resolvable to a higher minor/patch version, use the `--tighten` flag to update the `pubspec.yaml` automatically. + +**Input (`pubspec.yaml`):** +```yaml +dependencies: + http: ^0.13.0 +``` + +**Command:** +```bash +dart pub upgrade --tighten http +``` + +**Output (`pubspec.yaml`):** +```yaml +dependencies: + http: ^0.13.5 +``` + +### Surgical Lockfile Removal +If `package_a` is retracted or locked in a conflict, remove only its block from `pubspec.lock`. + +**Before (`pubspec.lock`):** +```yaml +packages: + package_a: + dependency: "direct main" + description: + name: package_a + url: "https://pub.dev" + source: hosted + version: "1.0.0" # Retracted version + package_b: + dependency: "direct main" + # ... +``` + +**Action:** Delete the `package_a` block entirely. Leave `package_b` untouched. Run `dart pub get`. diff --git a/.claude/skills/dart-run-static-analysis/SKILL.md b/.claude/skills/dart-run-static-analysis/SKILL.md new file mode 100644 index 0000000..27ca654 --- /dev/null +++ b/.claude/skills/dart-run-static-analysis/SKILL.md @@ -0,0 +1,104 @@ +--- +name: dart-run-static-analysis +description: Execute `dart analyze` to identify warnings and errors, and use `dart fix --apply` to automatically resolve mechanical lint issues. Use during development to ensure code quality and before committing changes. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Fri, 24 Apr 2026 15:09:34 GMT +--- +# Analyzing and Fixing Dart Code + +## Contents +- [Analysis Configuration](#analysis-configuration) +- [Diagnostic Suppression](#diagnostic-suppression) +- [Workflow: Executing Static Analysis](#workflow-executing-static-analysis) +- [Workflow: Applying Automated Fixes](#workflow-applying-automated-fixes) +- [Examples](#examples) + +## Analysis Configuration + +Configure the Dart analyzer using the `analysis_options.yaml` file located at the package root. + +- **Base Configuration:** Always include a standard rule set (e.g., `package:lints/recommended.yaml` or `package:flutter_lints/flutter.yaml`) using the `include:` directive. +- **Strict Type Checks:** Enable strict type checks under the `analyzer: language:` node to prevent implicit downcasts and dynamic inferences. Set `strict-casts: true`, `strict-inference: true`, and `strict-raw-types: true`. +- **Linter Rules:** Explicitly enable or disable specific rules under the `linter: rules:` node. Use a key-value map (`rule_name: true/false`) when overriding included rules, or a list (`- rule_name`) when defining a fresh set. Do not mix list and map syntax in the same `rules` block. +- **Formatter Configuration:** Configure `dart format` behavior under the `formatter:` node. Set `page_width` (default 80) and `trailing_commas` (`automate` or `preserve`). +- **Analyzer Plugins:** Enable custom diagnostics by adding plugins under the `analyzer: plugins:` node. Ensure the plugin package is added as a `dev_dependency` in `pubspec.yaml`. + +## Diagnostic Suppression + +When a diagnostic (lint or warning) yields a false positive or applies to generated code, suppress it explicitly. + +- **File-level Exclusion:** Use the `analyzer: exclude:` node in `analysis_options.yaml` to exclude entire files or directories (e.g., `**/*.g.dart`) using glob patterns. +- **File-level Suppression:** Add `// ignore_for_file: ` at the top of a Dart file to suppress specific diagnostics for the entire file. Use `// ignore_for_file: type=lint` to suppress all linter rules. +- **Line-level Suppression:** Add `// ignore: ` on the line directly above the offending code, or appended to the end of the offending line. +- **Pubspec Suppression:** Add `# ignore: ` above the offending line in `pubspec.yaml` files (e.g., `# ignore: sort_pub_dependencies`). +- **Plugin Diagnostics:** Prefix the diagnostic code with the plugin name when suppressing plugin-specific issues (e.g., `// ignore: some_plugin/some_code`). + +## Workflow: Executing Static Analysis + +Use this workflow to identify type-related bugs, style violations, and potential runtime errors. + +**Task Progress:** +- [ ] 1. Verify `analysis_options.yaml` exists at the project root. +- [ ] 2. Run the analyzer using the `analyze_files` MCP tool (if available) or the CLI command `dart analyze `. +- [ ] 3. Review the diagnostic output. +- [ ] 4. If info-level issues must be treated as failures, append the `--fatal-infos` flag. +- [ ] 5. Resolve reported errors manually or proceed to the Automated Fixes workflow. + +## Workflow: Applying Automated Fixes + +Use this workflow to resolve outdated API usages, apply quick fixes, and migrate code (e.g., Dart 3 migrations). + +**Task Progress:** +- [ ] 1. Execute a dry run to preview proposed changes using the `dart_fix` MCP tool or CLI command `dart fix --dry-run`. +- [ ] 2. Review the proposed fixes to ensure they align with the intended architecture. +- [ ] 3. If additional fixes are required, verify that the corresponding linter rules are enabled in `analysis_options.yaml`. +- [ ] 4. Apply the fixes using the `dart_fix` MCP tool or CLI command `dart fix --apply`. +- [ ] 5. Format the modified code using the `dart_format` MCP tool or CLI command `dart format .`. +- [ ] 6. Run the static analysis workflow to verify all diagnostics are resolved. + +## Examples + +### Comprehensive `analysis_options.yaml` + +```yaml +include: package:flutter_lints/recommended.yaml + +analyzer: + exclude: + - "**/*.g.dart" + - "lib/generated/**" + language: + strict-casts: true + strict-inference: true + strict-raw-types: true + errors: + todo: ignore + invalid_assignment: warning + missing_return: error + +linter: + rules: + avoid_shadowing_type_parameters: false + await_only_futures: true + use_super_parameters: true + +formatter: + page_width: 100 + trailing_commas: preserve +``` + +### Inline Diagnostic Suppression + +```dart +// Suppress for the entire file +// ignore_for_file: unused_local_variable, dead_code + +void processData() { + // Suppress for a specific line + // ignore: invalid_assignment + int x = ''; + + const y = 10; // ignore: constant_identifier_names +} +``` diff --git a/.claude/skills/dart-use-pattern-matching/SKILL.md b/.claude/skills/dart-use-pattern-matching/SKILL.md new file mode 100644 index 0000000..7455620 --- /dev/null +++ b/.claude/skills/dart-use-pattern-matching/SKILL.md @@ -0,0 +1,146 @@ +--- +name: dart-use-pattern-matching +description: Use switch expressions and pattern matching where appropriate +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Fri, 24 Apr 2026 15:08:55 GMT +--- +# Implementing Dart Patterns + +## Contents +- [Pattern Selection Strategy](#pattern-selection-strategy) +- [Switch Statements vs. Expressions](#switch-statements-vs-expressions) +- [Core Pattern Implementations](#core-pattern-implementations) +- [Workflows](#workflows) +- [Examples](#examples) + +## Pattern Selection Strategy + +Apply specific pattern types based on the data structure and desired outcome. Follow these conditional guidelines: + +* **If validating and extracting from deserialized data (e.g., JSON):** Use Map and List patterns to simultaneously check structure and destructure key-value pairs. +* **If handling multiple return values:** Use Record patterns to destructure fields directly into local variables. +* **If executing type-specific behavior (Algebraic Data Types):** Use Object patterns combined with `sealed` classes to ensure exhaustiveness. +* **If matching numeric ranges or conditions:** Use Relational (`>=`, `<=`) and Logical-and (`&&`) patterns. +* **If multiple cases share logic:** Use Logical-or (`||`) patterns to share a single case body or guard clause. +* **If ignoring specific values:** Use the Wildcard pattern (`_`) or a non-matching Rest element (`...`) in collections. + +## Switch Statements vs. Expressions + +Select the appropriate switch construct based on the execution context: + +* **If producing a value:** Use a **switch expression**. + * Syntax: `switch (value) { pattern => expression, }` + * Rule: Each case must be a single expression. No implicit fallthrough. Must be exhaustive. +* **If executing statements or side effects:** Use a **switch statement**. + * Syntax: `switch (value) { case pattern: statements; }` + * Rule: Empty cases fall through to the next case. Non-empty cases implicitly break (no `break` keyword required). + +## Core Pattern Implementations + +Implement patterns using the following syntax and rules: + +* **Logical-or (`||`):** `pattern1 || pattern2`. Both branches must define the exact same set of variables. +* **Logical-and (`&&`):** `pattern1 && pattern2`. Branches must *not* define overlapping variables. +* **Relational:** `==`, `!=`, `<`, `>`, `<=`, `>=` followed by a constant expression. +* **Cast (`as`):** `pattern as Type`. Throws if the value does not match the type. Use to forcibly assert types during destructuring. +* **Null-check (`?`):** `pattern?`. Fails the match if the value is null. Binds the variable to the non-nullable base type. +* **Null-assert (`!`):** `pattern!`. Throws if the value is null. +* **Variable:** `var name` or `Type name`. Binds the matched value to a new local variable. +* **Wildcard (`_`):** Matches any value and discards it. +* **List:** `[pattern1, pattern2]`. Matches lists of exact length unless a Rest element (`...` or `...var rest`) is used. +* **Map:** `{"key": pattern}`. Matches maps containing the specified keys. Ignores unmatched keys. +* **Record:** `(pattern1, named: pattern2)`. Matches records of the exact shape. Use `:var name` to infer the getter name. +* **Object:** `ClassName(field: pattern)`. Matches instances of `ClassName`. Use `:var field` to infer the getter name. + +## Workflows + +### Task Progress: Implementing Pattern Matching +Copy this checklist to track progress when implementing complex pattern matching logic: + +- [ ] Identify the data structure being evaluated (JSON, Record, Class, Enum). +- [ ] Select the appropriate switch construct (Expression for values, Statement for side-effects). +- [ ] Define the required patterns (Object, Map, List, Record). +- [ ] Extract required data using Variable patterns (`var x`, `:var y`). +- [ ] Apply Guard clauses (`when condition`) for logic that cannot be expressed via patterns. +- [ ] Handle unmatched cases using a Wildcard (`_`) or `default` clause (if not using a sealed class). +- [ ] Run exhaustiveness validator. + +### Feedback Loop: Exhaustiveness Checking +When switching over `sealed` classes or enums, you must ensure all subtypes are handled. + +1. **Run validator:** Execute `dart analyze`. +2. **Review errors:** Look for "The type 'X' is not exhaustively matched by the switch cases" errors. +3. **Fix:** Add the missing Object patterns for the unhandled subtypes, or add a Wildcard (`_`) case if a default fallback is acceptable. + +## Examples + +### JSON Validation and Destructuring +Use Map and List patterns to validate structure and extract data in a single step. + +**Input:** +```dart +var data = { + 'user': ['Lily', 13], +}; +``` + +**Implementation:** +```dart +if (data case {'user': [String name, int age]}) { + print('User $name is $age years old.'); +} else { + print('Invalid JSON structure.'); +} +``` + +### Algebraic Data Types (Sealed Classes) +Use Object patterns with switch expressions to handle family types exhaustively. + +**Implementation:** +```dart +sealed class Shape {} + +class Square implements Shape { + final double length; + Square(this.length); +} + +class Circle implements Shape { + final double radius; + Circle(this.radius); +} + +// Switch expression guarantees exhaustiveness due to `sealed` modifier. +double calculateArea(Shape shape) => switch (shape) { + Square(length: var l) => l * l, + Circle(:var radius) => math.pi * radius * radius, +}; +``` + +### Variable Swapping and Destructuring +Use variable assignment patterns to swap values or extract record fields without temporary variables. + +**Implementation:** +```dart +var (a, b) = ('left', 'right'); +(b, a) = (a, b); // Swap values + +// Destructuring a function return +var (name, age) = getUserInfo(); +``` + +### Guard Clauses and Logical-or +Use `when` to evaluate arbitrary conditions after a pattern matches. + +**Implementation:** +```dart +switch (shape) { + case Square(size: var s) || Circle(size: var s) when s > 0: + print('Valid symmetric shape with size $s'); + case Square() || Circle(): + print('Invalid or empty shape'); + default: + print('Unknown shape'); +} +``` diff --git a/.claude/skills/databricks-agent-bricks/SKILL.md b/.claude/skills/databricks-agent-bricks/SKILL.md index 4aff7ac..026f204 100644 --- a/.claude/skills/databricks-agent-bricks/SKILL.md +++ b/.claude/skills/databricks-agent-bricks/SKILL.md @@ -28,7 +28,7 @@ Before creating Agent Bricks, ensure you have the required data: ### For Genie Spaces - **See the `databricks-genie` skill** for comprehensive Genie Space guidance - Tables in Unity Catalog with the data to explore -- Generate raw data using the `databricks-synthetic-data-generation` skill +- Generate raw data using the `databricks-synthetic-data-gen` skill - Create tables using the `databricks-spark-declarative-pipelines` skill ### For Supervisor Agents @@ -67,18 +67,19 @@ Actions: **For comprehensive Genie guidance, use the `databricks-genie` skill.** -Basic tools available: - -- `create_or_update_genie` - Create or update a Genie Space -- `get_genie` - Get Genie Space details -- `delete_genie` - Delete a Genie Space +Use `manage_genie` with actions: +- `create_or_update` - Create or update a Genie Space +- `get` - Get Genie Space details +- `list` - List all Genie Spaces +- `delete` - Delete a Genie Space +- `export` / `import` - For migration See `databricks-genie` skill for: - Table inspection workflow - Sample question best practices - Curation (instructions, certified queries) -**IMPORTANT**: There is NO system table for Genie spaces (e.g., `system.ai.genie_spaces` does not exist). To find a Genie space by name, use the `find_genie_by_name` tool. +**IMPORTANT**: There is NO system table for Genie spaces (e.g., `system.ai.genie_spaces` does not exist). Use `manage_genie(action="list")` to find spaces. ### Supervisor Agent Tool @@ -119,7 +120,7 @@ Before creating Agent Bricks, generate the required source data: **For Genie (SQL exploration)**: ``` -1. Use `databricks-synthetic-data-generation` skill to create raw parquet data +1. Use `databricks-synthetic-data-gen` skill to create raw parquet data 2. Use `databricks-spark-declarative-pipelines` skill to create bronze/silver/gold tables ``` @@ -199,7 +200,7 @@ manage_mas( - **[databricks-genie](../databricks-genie/SKILL.md)** - Comprehensive Genie Space creation, curation, and Conversation API guidance - **[databricks-unstructured-pdf-generation](../databricks-unstructured-pdf-generation/SKILL.md)** - Generate synthetic PDFs to feed into Knowledge Assistants -- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - Create raw data for Genie Space tables +- **[databricks-synthetic-data-gen](../databricks-synthetic-data-gen/SKILL.md)** - Create raw data for Genie Space tables - **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - Build bronze/silver/gold tables consumed by Genie Spaces - **[databricks-model-serving](../databricks-model-serving/SKILL.md)** - Deploy custom agent endpoints used as MAS agents - **[databricks-vector-search](../databricks-vector-search/SKILL.md)** - Build vector indexes for RAG applications paired with KAs diff --git a/.claude/skills/databricks-ai-functions/1-task-functions.md b/.claude/skills/databricks-ai-functions/1-task-functions.md new file mode 100644 index 0000000..a94159e --- /dev/null +++ b/.claude/skills/databricks-ai-functions/1-task-functions.md @@ -0,0 +1,385 @@ +# Task-Specific AI Functions — Full Reference + +These functions require no model endpoint selection. They call pre-configured Foundation Model APIs optimized for each task. All require DBR 15.1+ (15.4 ML LTS for batch); `ai_parse_document` requires DBR 17.1+. + +--- + +## `ai_analyze_sentiment` + +**Docs:** https://docs.databricks.com/aws/en/sql/language-manual/functions/ai_analyze_sentiment + +Returns one of: `positive`, `negative`, `neutral`, `mixed`, or `NULL`. + +```sql +SELECT ai_analyze_sentiment(review_text) AS sentiment +FROM customer_reviews; +``` + +```python +from pyspark.sql.functions import expr +df = spark.table("customer_reviews") +df.withColumn("sentiment", expr("ai_analyze_sentiment(review_text)")).display() +``` + +--- + +## `ai_classify` + +**Docs:** https://docs.databricks.com/aws/en/sql/language-manual/functions/ai_classify + +**Syntax:** `ai_classify(content, labels [, options])` +- `content`: VARIANT | STRING — raw text, or VARIANT from `ai_parse_document` / `ai_extract` +- `labels`: STRING — JSON labels definition: + - Simple array: `'["urgent", "not_urgent", "spam"]'` + - With descriptions: `'{"billing_error": "Payment, invoice, or refund issues", "product_defect": "Any malfunction or bug"}'` (descriptions up to 1000 chars each) + - 2–500 labels, each 1–100 characters +- `options`: optional MAP\: + - `instructions`: task context to improve accuracy (max 20,000 chars) + - `multilabel`: `"true"` to return multiple matching labels (default `"false"`) + +Returns VARIANT. Returns `NULL` if content is `NULL`. + +```sql +-- simple labels +SELECT ticket_text, + ai_classify(ticket_text, '["urgent", "not urgent", "spam"]') AS priority +FROM support_tickets; +-- {"response": ["urgent"], "error_message": null} + +-- labels with descriptions +SELECT ticket_text, + ai_classify( + ticket_text, + '{"billing_error": "Payment, invoice, or refund issues", + "product_defect": "Any malfunction, bug, or breakage", + "account_issue": "Login failures, password resets"}', + MAP('instructions', 'Customer support tickets for a SaaS product') + ) AS category +FROM support_tickets; +``` + +```python +from pyspark.sql.functions import expr +df = spark.table("support_tickets") +df.withColumn( + "priority", + expr("ai_classify(ticket_text, '[\"urgent\", \"not urgent\", \"spam\"]')") +).display() +``` + +**Tips:** +- Use label descriptions for ambiguous categories — they significantly improve accuracy +- `multilabel: "true"` enables multi-label classification without running multiple calls +- Up to 500 labels supported + +--- + +## `ai_extract` + +**Docs:** https://docs.databricks.com/aws/en/sql/language-manual/functions/ai_extract + +**Syntax:** `ai_extract(content, schema [, options])` +- `content`: VARIANT | STRING — raw text, or VARIANT from `ai_parse_document` +- `schema`: STRING — JSON schema definition: + - Simple (field names only): `'["invoice_id", "vendor_name", "total_amount"]'` + - Advanced (with types and descriptions): + ```json + { + "invoice_id": {"type": "string"}, + "total_amount": {"type": "number"}, + "currency": {"type": "enum", "labels": ["USD", "EUR", "GBP"]}, + "line_items": {"type": "array", "items": {"type": "object", "properties": {...}}} + } + ``` + - Supported types: `string`, `integer`, `number`, `boolean`, `enum` + - Max 128 fields, 7 nesting levels, 500 enum values +- `options`: optional MAP\: + - `instructions`: task context to improve extraction quality (max 20,000 chars) + +Returns VARIANT `{"response": {...}, "error_message": null}`. Returns `NULL` if content is `NULL`. + +```sql +-- simple schema +SELECT ai_extract( + 'Invoice #12345 from Acme Corp for $1,250.00', + '["invoice_id", "vendor_name", "total_amount"]' +) AS extracted; +-- {"response": {"invoice_id": "12345", "vendor_name": "Acme Corp", ...}, "error_message": null} + +-- composable with ai_parse_document +WITH parsed AS ( + SELECT ai_parse_document(content, MAP('version', '2.0')) AS parsed + FROM READ_FILES('/Volumes/finance/invoices/', format => 'binaryFile') +) +SELECT ai_extract( + parsed, + '["invoice_id", "vendor_name", "total_amount"]', + MAP('instructions', 'These are vendor invoices.') +) AS invoice_data +FROM parsed; +``` + +```python +from pyspark.sql.functions import expr +df = spark.table("messages") +df = df.withColumn( + "entities", + expr("ai_extract(message, '[\"person\", \"location\", \"date\"]')") +) +df.display() +``` + +--- + +## `ai_fix_grammar` + +**Docs:** https://docs.databricks.com/aws/en/sql/language-manual/functions/ai_fix_grammar + +**Syntax:** `ai_fix_grammar(content)` — Returns corrected STRING. + +Optimized for English. Useful for cleaning user-generated content before downstream processing. + +```sql +SELECT ai_fix_grammar(user_comment) AS corrected FROM user_feedback; +``` + +```python +from pyspark.sql.functions import expr +df = spark.table("user_feedback") +df.withColumn("corrected", expr("ai_fix_grammar(user_comment)")).display() +``` + +--- + +## `ai_gen` + +**Docs:** https://docs.databricks.com/aws/en/sql/language-manual/functions/ai_gen + +**Syntax:** `ai_gen(prompt)` — Returns a generated STRING. + +Use for free-form text generation where the output format doesn't need to be structured. For structured JSON output, use `ai_query` with `responseFormat`. + +```sql +SELECT product_name, + ai_gen(CONCAT('Write a one-sentence marketing tagline for: ', product_name)) AS tagline +FROM products; +``` + +```python +from pyspark.sql.functions import expr +df = spark.table("products") +df.withColumn( + "tagline", + expr("ai_gen(concat('Write a one-sentence marketing tagline for: ', product_name))") +).display() +``` + +--- + +## `ai_mask` + +**Docs:** https://docs.databricks.com/aws/en/sql/language-manual/functions/ai_mask + +**Syntax:** `ai_mask(content, labels)` +- `content`: STRING — text with sensitive data +- `labels`: ARRAY\ — entity types to redact + +Returns text with identified entities replaced by `[MASKED]`. + +Common label values: `'person'`, `'email'`, `'phone'`, `'address'`, `'ssn'`, `'credit_card'` + +```sql +SELECT ai_mask( + message_body, + ARRAY('person', 'email', 'phone', 'address') +) AS message_safe +FROM customer_messages; +``` + +```python +from pyspark.sql.functions import expr +df = spark.table("customer_messages") +df.withColumn( + "message_safe", + expr("ai_mask(message_body, array('person', 'email', 'phone'))") +).write.format("delta").mode("append").saveAsTable("catalog.schema.messages_safe") +``` + +--- + +## `ai_similarity` + +**Docs:** https://docs.databricks.com/aws/en/sql/language-manual/functions/ai_similarity + +**Syntax:** `ai_similarity(expr1, expr2)` — Returns a FLOAT between 0.0 and 1.0. + +Use for fuzzy deduplication, search result ranking, or item matching across datasets. + +```sql +-- Deduplicate company names (similarity > 0.85 = likely duplicate) +SELECT a.id, b.id, a.name, b.name, + ai_similarity(a.name, b.name) AS score +FROM companies a +JOIN companies b ON a.id < b.id +WHERE ai_similarity(a.name, b.name) > 0.85 +ORDER BY score DESC; +``` + +```python +from pyspark.sql.functions import expr +df = spark.table("product_search") +df.withColumn( + "match_score", + expr("ai_similarity(search_query, product_title)") +).orderBy("match_score", ascending=False).display() +``` + +--- + +## `ai_summarize` + +**Docs:** https://docs.databricks.com/aws/en/sql/language-manual/functions/ai_summarize + +**Syntax:** `ai_summarize(content [, max_words])` +- `content`: STRING — text to summarize +- `max_words`: INTEGER (optional) — word limit; default 50; use `0` for uncapped + +```sql +-- Default (50 words) +SELECT ai_summarize(article_body) AS summary FROM news_articles; + +-- Custom word limit +SELECT ai_summarize(article_body, 20) AS brief FROM news_articles; +SELECT ai_summarize(article_body, 0) AS full FROM news_articles; +``` + +```python +from pyspark.sql.functions import expr +df = spark.table("news_articles") +df.withColumn("summary", expr("ai_summarize(article_body, 30)")).display() +``` + +--- + +## `ai_translate` + +**Docs:** https://docs.databricks.com/aws/en/sql/language-manual/functions/ai_translate + +**Syntax:** `ai_translate(content, to_lang)` +- `content`: STRING — source text +- `to_lang`: STRING — target language code + +**Supported languages:** `en`, `de`, `fr`, `it`, `pt`, `hi`, `es`, `th` + +For unsupported languages, use `ai_query` with a multilingual model endpoint. + +```sql +-- Single language +SELECT ai_translate(product_description, 'es') AS description_es FROM products; + +-- Multi-language fanout +SELECT + description, + ai_translate(description, 'fr') AS description_fr, + ai_translate(description, 'de') AS description_de +FROM products; +``` + +```python +from pyspark.sql.functions import expr +df = spark.table("products") +df.withColumn( + "description_es", + expr("ai_translate(product_description, 'es')") +).display() +``` + +--- + +## `ai_parse_document` + +**Docs:** https://docs.databricks.com/aws/en/sql/language-manual/functions/ai_parse_document + +**Requires:** DBR 17.1+ + +**Syntax:** `ai_parse_document(content [, options])` +- `content`: BINARY — document content loaded from `read_files()` or `spark.read.format("binaryFile")` +- `options`: MAP\ (optional) — parsing configuration + +**Supported formats:** PDF, JPG/JPEG, PNG, DOCX, PPTX + +Returns a VARIANT with pages, elements (text paragraphs, tables, figures, headers, footers), bounding boxes, and error metadata. + +**Options:** + +| Key | Values | Description | +|-----|--------|-------------| +| `version` | `'2.0'` | Output schema version | +| `imageOutputPath` | Volume path | Save rendered page images | +| `descriptionElementTypes` | `''`, `'figure'`, `'*'` | AI-generated descriptions (default: `'*'` for all) | + +**Output schema:** + +``` +document +├── pages[] -- page id, image_uri +└── elements[] -- extracted content + ├── type -- "text", "table", "figure", etc. + ├── content -- extracted text + ├── bbox -- bounding box coordinates + └── description -- AI-generated description +metadata -- file info, schema version +error_status[] -- errors per page (if any) +``` + +```sql +-- Parse and extract text blocks +SELECT + path, + parsed:pages[*].elements[*].content AS text_blocks, + parsed:error AS parse_error +FROM ( + SELECT path, ai_parse_document(content) AS parsed + FROM read_files('/Volumes/catalog/schema/landing/docs/', format => 'binaryFile') +); + +-- Parse with options (image output + descriptions) +SELECT ai_parse_document( + content, + map( + 'version', '2.0', + 'imageOutputPath', '/Volumes/catalog/schema/volume/images/', + 'descriptionElementTypes', '*' + ) +) AS parsed +FROM read_files('/Volumes/catalog/schema/volume/invoices/', format => 'binaryFile'); +``` + +```python +from pyspark.sql.functions import expr + +df = ( + spark.read.format("binaryFile") + .load("/Volumes/catalog/schema/landing/docs/") + .withColumn("parsed", expr("ai_parse_document(content)")) + .selectExpr( + "path", + "parsed:pages[*].elements[*].content AS text_blocks", + "parsed:error AS parse_error", + ) + .filter("parse_error IS NULL") +) + +# Chain with task-specific functions on the extracted text +df = ( + df.withColumn("summary", expr("ai_summarize(text_blocks, 50)")) + .withColumn("entities", expr("ai_extract(text_blocks, array('date', 'amount', 'vendor'))")) + .withColumn("category", expr("ai_classify(text_blocks, array('invoice', 'contract', 'report'))")) +) +df.display() +``` + +**Limitations:** +- Processing is slow for dense or low-resolution documents +- Suboptimal for non-Latin alphabets and digitally signed PDFs +- Custom models not supported — always uses the built-in parsing model diff --git a/.claude/skills/databricks-ai-functions/2-ai-query.md b/.claude/skills/databricks-ai-functions/2-ai-query.md new file mode 100644 index 0000000..60d860f --- /dev/null +++ b/.claude/skills/databricks-ai-functions/2-ai-query.md @@ -0,0 +1,223 @@ +# `ai_query` — Full Reference + +**Docs:** https://docs.databricks.com/aws/en/sql/language-manual/functions/ai_query + +> Use `ai_query` only when no task-specific function fits. See the function selection table in [SKILL.md](SKILL.md). + +## When to Use `ai_query` + +- Output schema has **nested arrays or deeply nested STRUCTs** (e.g., `itens: [{codigo, descricao, qtde}]`) +- Calling a **custom Model Serving endpoint** (your own fine-tuned model) +- **Multimodal input** — passing binary image files via `files =>` +- **Cross-document reasoning** — prompt includes content from multiple sources +- Need **sampling parameters** (`temperature`, `max_tokens`) control + +## Syntax + +```sql +ai_query( + endpoint, + request + [, returnType => ddl_schema] + [, failOnError => boolean] + [, modelParameters => named_struct(...)] + [, responseFormat => json_string] + [, files => binary_column] +) +``` + +## Parameters + +| Parameter | Type | Runtime | Description | +|---|---|---|---| +| `endpoint` | STRING literal | — | Foundation Model name or custom endpoint name. Never guess — use exact names from the [model serving docs](https://docs.databricks.com/aws/en/machine-learning/foundation-models/supported-models.html). | +| `request` | STRING or STRUCT | — | Prompt string for chat models; STRUCT for custom ML endpoints | +| `returnType` | DDL schema (optional) | 15.2+ | Structures the parsed response like `from_json` | +| `failOnError` | BOOLEAN (optional, default `true`) | 15.3+ | If `false`, returns STRUCT `{response, error}` instead of raising on failure | +| `modelParameters` | STRUCT (optional) | 15.3+ | Sampling params: `temperature`, `max_tokens`, `top_p`, etc. | +| `responseFormat` | JSON string (optional) | 15.4+ | Forces structured JSON output: `'{"type":"json_object"}'` | +| `files` | binary column (optional) | — | Pass binary images directly (JPEG/PNG) — no upload step needed | + +## Foundation Model Names (Do Not Guess) + +| Use case | Endpoint name | +|---|---| +| General reasoning / extraction | `databricks-claude-sonnet-4` | +| Fast / cheap tasks | `databricks-meta-llama-3-1-8b-instruct` | +| Large context / complex | `databricks-meta-llama-3-3-70b-instruct` | +| Multimodal (vision + text) | `databricks-llama-4-maverick` | +| Embeddings | `databricks-gte-large-en` | + +## Patterns + +### Basic — single prompt + +```sql +SELECT ai_query( + 'databricks-meta-llama-3-3-70b-instruct', + 'Describe Databricks SQL in 30 words.' +) AS response; +``` + +### Applied to a table column + +```sql +SELECT ticket_id, + ai_query( + 'databricks-meta-llama-3-3-70b-instruct', + CONCAT('Summarize in one sentence: ', ticket_body) + ) AS summary +FROM support_tickets; +``` + +### Structured JSON output (`responseFormat`) + +Preferred over `returnType` for chat models (requires Runtime 15.4+): + +```sql +SELECT ai_query( + 'databricks-claude-sonnet-4', + CONCAT('Extract invoice fields as JSON. Fields: numero, fornecedor, total, ' + 'itens:[{codigo, descricao, qtde, vlrUnit}]. Input: ', text_blocks), + responseFormat => '{"type":"json_object"}', + failOnError => false +) AS ai_response +FROM parsed_documents; +``` + +Then parse with `from_json`: + +```python +from pyspark.sql.functions import from_json, col + +df = df.withColumn( + "invoice", + from_json( + col("ai_response.response"), + "STRUCT>>" + ) +) +# Access fields +df.select("invoice.numero", "invoice.total", "invoice.itens").display() +``` + +### With `failOnError` (always use in batch pipelines) + +```sql +SELECT + id, + ai_response.response, + ai_response.error +FROM ( + SELECT id, + ai_query( + 'databricks-claude-sonnet-4', + CONCAT('Classify: ', text), + failOnError => false + ) AS ai_response + FROM documents +) +-- Route errors to a separate table downstream +``` + +### With `modelParameters` (control sampling) + +```sql +SELECT ai_query( + 'databricks-meta-llama-3-3-70b-instruct', + CONCAT('Extract entities from: ', text), + failOnError => false, + modelParameters => named_struct('temperature', CAST(0.0 AS DOUBLE), 'max_tokens', 500) +) AS result +FROM documents; +``` + +### Multimodal — image files (`files =>`) + +No file upload step needed. Pass the binary column directly: + +```sql +SELECT + path, + ai_query( + 'databricks-llama-4-maverick', + 'Describe what is in this image in detail.', + files => content + ) AS description +FROM read_files('/Volumes/catalog/schema/images/', format => 'binaryFile'); +``` + +```python +from pyspark.sql.functions import expr + +df = ( + spark.read.format("binaryFile") + .load("/Volumes/catalog/schema/images/") + .withColumn("description", expr(""" + ai_query( + 'databricks-llama-4-maverick', + 'Describe the contents of this image.', + files => content + ) + """)) +) +``` + +### As a reusable SQL UDF + +```sql +CREATE FUNCTION catalog.schema.extract_invoice(text STRING) +RETURNS STRING +RETURN ai_query( + 'databricks-claude-sonnet-4', + CONCAT('Extract invoice JSON from: ', text), + responseFormat => '{"type":"json_object"}' +); + +SELECT extract_invoice(document_text) FROM raw_documents; +``` + +### PySpark with `expr` + +```python +from pyspark.sql.functions import expr + +df = spark.table("documents") +df = df.withColumn("result", expr(""" + ai_query( + 'databricks-claude-sonnet-4', + concat('Extract structured data from: ', content), + responseFormat => '{"type":"json_object"}', + failOnError => false + ) +""")) +``` + +## Error Handling Pattern for Batch Pipelines + +Always use `failOnError => false` in batch jobs. Write errors to a sidecar table: + +```python +import dlt +from pyspark.sql.functions import expr, col + +@dlt.table(comment="AI extraction results") +def extracted(): + return ( + dlt.read("raw") + .withColumn("ai_response", expr(""" + ai_query('databricks-claude-sonnet-4', prompt, + responseFormat => '{"type":"json_object"}', + failOnError => false) + """)) + ) + +@dlt.table(comment="Rows that failed AI extraction") +def extraction_errors(): + return ( + dlt.read("extracted") + .filter(col("ai_response.error").isNotNull()) + .select("id", "prompt", col("ai_response.error").alias("error")) + ) +``` diff --git a/.claude/skills/databricks-ai-functions/3-ai-forecast.md b/.claude/skills/databricks-ai-functions/3-ai-forecast.md new file mode 100644 index 0000000..9c1f9b1 --- /dev/null +++ b/.claude/skills/databricks-ai-functions/3-ai-forecast.md @@ -0,0 +1,162 @@ +# `ai_forecast` — Full Reference + +**Docs:** https://docs.databricks.com/aws/en/sql/language-manual/functions/ai_forecast + +> `ai_forecast` is a **table-valued function** — it returns a table of rows, not a scalar. Call it with `SELECT * FROM ai_forecast(...)`. + +## Requirements + +- **Pro or Serverless SQL warehouse** — not available on Classic or Starter +- Input data must have a DATE or TIMESTAMP time column and at least one numeric value column + +## Syntax + +```sql +SELECT * +FROM ai_forecast( + observed => TABLE(...) or query, + horizon => 'YYYY-MM-DD' or TIMESTAMP, + time_col => 'column_name', + value_col => 'column_name', + [group_col => 'column_name'], + [prediction_interval_width => 0.95] +) +``` + +## Parameters + +| Parameter | Type | Description | +|---|---|---| +| `observed` | TABLE reference or subquery | Training data with time + value columns | +| `horizon` | DATE, TIMESTAMP, or STRING | End date/time for the forecast period | +| `time_col` | STRING | Name of the DATE or TIMESTAMP column in `observed` | +| `value_col` | STRING | One or more numeric columns to forecast (up to 100 per group) | +| `group_col` | STRING (optional) | Column to partition forecasts by — produces one forecast series per group value | +| `prediction_interval_width` | DOUBLE (optional, default 0.95) | Confidence interval width between 0 and 1 | + +## Output Columns + +For each `value_col` named `metric`, the output includes: + +| Column | Type | Description | +|---|---|---| +| time_col | DATE or TIMESTAMP | The forecast timestamp (same type as input) | +| `metric_forecast` | DOUBLE | Point forecast | +| `metric_upper` | DOUBLE | Upper confidence bound | +| `metric_lower` | DOUBLE | Lower confidence bound | +| group_col | original type | Present when `group_col` is specified | + +## Patterns + +### Single Metric Forecast + +```sql +SELECT * +FROM ai_forecast( + observed => TABLE(SELECT order_date, revenue FROM daily_revenue), + horizon => '2026-12-31', + time_col => 'order_date', + value_col => 'revenue' +); +-- Returns: order_date, revenue_forecast, revenue_upper, revenue_lower +``` + +### Multi-Group Forecast + +Produces one forecast series per distinct value of `group_col`: + +```sql +SELECT * +FROM ai_forecast( + observed => TABLE(SELECT date, region, sales FROM regional_sales), + horizon => '2026-12-31', + time_col => 'date', + value_col => 'sales', + group_col => 'region' +); +-- Returns: date, region, sales_forecast, sales_upper, sales_lower +-- One row per date per region +``` + +### Multiple Value Columns + +```sql +SELECT * +FROM ai_forecast( + observed => TABLE(SELECT date, units, revenue FROM daily_kpis), + horizon => '2026-06-30', + time_col => 'date', + value_col => 'units,revenue' -- comma-separated +); +-- Returns: date, units_forecast, units_upper, units_lower, +-- revenue_forecast, revenue_upper, revenue_lower +``` + +### Custom Confidence Interval + +```sql +SELECT * +FROM ai_forecast( + observed => TABLE(SELECT ts, sensor_value FROM iot_readings), + horizon => '2026-03-31', + time_col => 'ts', + value_col => 'sensor_value', + prediction_interval_width => 0.80 -- narrower interval = less conservative +); +``` + +### Filtering Input Data (Subquery) + +```sql +SELECT * +FROM ai_forecast( + observed => TABLE( + SELECT date, sales + FROM daily_sales + WHERE region = 'BR' AND date >= '2024-01-01' + ), + horizon => '2026-12-31', + time_col => 'date', + value_col => 'sales' +); +``` + +### PySpark — Use `spark.sql()` + +`ai_forecast` is a table-valued function and must be called through `spark.sql()`: + +```python +result = spark.sql(""" + SELECT * + FROM ai_forecast( + observed => TABLE(SELECT date, sales FROM catalog.schema.daily_sales), + horizon => '2026-12-31', + time_col => 'date', + value_col => 'sales' + ) +""") +result.display() +``` + +### Save Forecast to Delta Table + +```python +result = spark.sql(""" + SELECT * + FROM ai_forecast( + observed => TABLE(SELECT date, region, revenue FROM catalog.schema.sales), + horizon => '2026-12-31', + time_col => 'date', + value_col => 'revenue', + group_col => 'region' + ) +""") +result.write.format("delta").mode("overwrite").saveAsTable("catalog.schema.revenue_forecast") +``` + +## Notes + +- The underlying model is a **prophet-like piecewise linear + seasonality model** — suitable for business time series with trend and weekly/yearly seasonality +- Handles "any number of groups" but up to **100 metrics per group** +- Output time column preserves the input type (DATE stays DATE, TIMESTAMP stays TIMESTAMP) +- Value columns are always cast to DOUBLE in output regardless of input type diff --git a/.claude/skills/databricks-ai-functions/4-document-processing-pipeline.md b/.claude/skills/databricks-ai-functions/4-document-processing-pipeline.md new file mode 100644 index 0000000..37498f4 --- /dev/null +++ b/.claude/skills/databricks-ai-functions/4-document-processing-pipeline.md @@ -0,0 +1,505 @@ +# Document Processing Pipeline with AI Functions + +End-to-end patterns for building batch document processing pipelines using AI Functions in a Lakeflow Declarative Pipeline (DLT). Covers function selection, `config.yml` centralization, error handling, and guidance on near-real-time variants with DSPy or LangChain. + +> For workflow migration context (e.g., migrating from n8n, LangChain, or other orchestration tools), see the companion skill `n8n-to-databricks`. + +--- + +## Function Selection for Document Pipelines + +When processing documents with AI Functions, apply this order of preference for each stage: + +| Stage | Preferred function | Use `ai_query` when... | +|---|---|---| +| Parse binary docs (PDF, DOCX, images) | `ai_parse_document` | Need image-level reasoning | +| Extract fields from text (flat or nested) | `ai_extract` | Schema exceeds 128 fields or 7 nesting levels | +| Classify document type or status | `ai_classify` | More than 20 categories | +| Score item similarity / matching | `ai_similarity` | Need cross-document reasoning | +| Summarize long sections | `ai_summarize` | — | +| Extract deeply nested JSON | `ai_query` with `responseFormat` | Schema exceeds `ai_extract` limits (128 fields, 7 levels) | + +--- + +## Centralized Configuration (`config.yml`) + +**Always centralize model names, volume paths, and prompts in a `config.yml`.** This makes model swaps a one-line change and keeps pipeline code free of hardcoded strings. + +```yaml +# config.yml +models: + default: "databricks-claude-sonnet-4" + mini: "databricks-meta-llama-3-1-8b-instruct" + vision: "databricks-llama-4-maverick" + +catalog: + name: "my_catalog" + schema: "document_processing" + +volumes: + input: "/Volumes/my_catalog/document_processing/landing/" + tmp: "/Volumes/my_catalog/document_processing/tmp/" + +output_tables: + results: "my_catalog.document_processing.processed_docs" + errors: "my_catalog.document_processing.processing_errors" + +prompts: + extract_invoice: | + Extract invoice fields and return ONLY valid JSON. + Fields: invoice_number, vendor_name, vendor_tax_id (digits only), + issue_date (dd/mm/yyyy), total_amount (numeric), + line_items: [{item_code, description, quantity, unit_price, total}]. + Return null for missing fields. + + classify_doc: | + Classify this document into exactly one category. +``` + +```python +# config_loader.py +import yaml + +def load_config(path: str = "config.yml") -> dict: + with open(path) as f: + return yaml.safe_load(f) + +CFG = load_config() +ENDPOINT = CFG["models"]["default"] +ENDPOINT_MINI = CFG["models"]["mini"] +VOLUME_INPUT = CFG["volumes"]["input"] +PROMPT_INV = CFG["prompts"]["extract_invoice"] +``` + +--- + +## Batch Pipeline — Lakeflow Declarative Pipeline + +Each logical step in your document workflow maps to a `@dlt.table` stage. Data flows through Delta tables between stages. + +``` +[Landing Volume] → Stage 1: ai_parse_document + → Stage 2: ai_classify (document type) + → Stage 3: ai_extract (flat fields) + ai_query (nested JSON) + → Stage 4: ai_similarity (item matching) + → Stage 5: Final Delta output table +``` + +### `pipeline.py` + +```python +import dlt +import yaml +from pyspark.sql.functions import expr, col, from_json + +CFG = yaml.safe_load(open("/Workspace/path/to/config.yml")) +ENDPOINT = CFG["models"]["default"] +VOL_IN = CFG["volumes"]["input"] +PROMPT = CFG["prompts"]["extract_invoice"] + + +# ── Stage 1: Parse binary documents ────────────────────────────────────────── +# Preferred: ai_parse_document — no model selection, no ai_query needed + +@dlt.table(comment="Parsed document text from all file types in the landing volume") +def raw_parsed(): + return ( + spark.read.format("binaryFile").load(VOL_IN) + .withColumn("parsed", expr("ai_parse_document(content, MAP('version', '2.0'))")) + .withColumn("text_blocks", expr(""" + concat_ws('\n', transform( + parsed:document:elements, + e -> e:content::STRING + )) + """)) + .selectExpr( + "path", + "text_blocks", + "parsed:error_status AS parse_error", + ) + .filter("parse_error IS NULL") + ) + + +# ── Stage 2: Classify document type ────────────────────────────────────────── +# Preferred: ai_classify — cheap, no endpoint selection + +@dlt.table(comment="Document type classification") +def classified_docs(): + return ( + dlt.read("raw_parsed") + .withColumn( + "doc_type", + expr(""" + ai_classify( + text_blocks, + '["invoice", "purchase_order", "receipt", "contract", "other"]', + MAP('version', '2.0') + ):response[0]::STRING + """) + ) + ) + + +# ── Stage 3a: Flat field extraction ────────────────────────────────────────── +# Preferred: ai_extract for flat fields (vendor, date, total) + +@dlt.table(comment="Flat header fields extracted from documents") +def extracted_flat(): + return ( + dlt.read("classified_docs") + .filter("doc_type = 'invoice'") + .filter("text_blocks IS NOT NULL") + .withColumn( + "result", + expr(""" + ai_extract( + text_blocks, + '{ + "invoice_number": {"type": "string"}, + "vendor_name": {"type": "string"}, + "issue_date": {"type": "string", "description": "dd/mm/yyyy"}, + "total_amount": {"type": "number"}, + "tax_id": {"type": "string"} + }', + MAP('version', '2.0') + ) + """) + ) + .selectExpr( + "path", "doc_type", "text_blocks", + "result:response AS header", + "result:error_message::STRING AS extract_error" + ) + ) + + +# ── Stage 3b: Nested JSON extraction (last resort: ai_query) ───────────────── +# Use ai_query only for deeply nested schemas that exceed ai_extract's 7-level limit + +@dlt.table(comment="Nested line items extracted — ai_query used for array schema only") +def extracted_line_items(): + return ( + dlt.read("extracted_flat") + .filter("extract_error IS NULL") + .withColumn( + "ai_response", + expr(f""" + ai_query( + '{ENDPOINT}', + concat('{PROMPT.strip()}', '\\n\\nDocument text:\\n', LEFT(text_blocks, 6000)), + responseFormat => '{{"type":"json_object"}}', + failOnError => false + ) + """) + ) + .withColumn( + "line_items", + from_json( + col("ai_response.response"), + "STRUCT>>" + ) + ) + .select("path", "doc_type", "header", "line_items", col("ai_response.error").alias("extraction_error")) + ) + + +# ── Stage 4: Similarity matching ───────────────────────────────────────────── +# Preferred: ai_similarity for fuzzy matching between extracted fields + +@dlt.table(comment="Vendor name similarity vs reference master data") +def vendor_matched(): + extracted = dlt.read("extracted_line_items") + # Join against a reference vendor table for fuzzy matching + vendors = spark.table("my_catalog.document_processing.vendor_master").select("vendor_id", "vendor_name") + + return ( + extracted.crossJoin(vendors) + .withColumn( + "name_similarity", + expr("ai_similarity(header:vendor_name::STRING, vendor_name)") + ) + .filter("name_similarity > 0.80") + .orderBy("name_similarity", ascending=False) + ) + + +# ── Stage 5: Final output + error sidecar ──────────────────────────────────── + +@dlt.table( + comment="Final processed documents ready for downstream consumption", + table_properties={"delta.enableChangeDataFeed": "true"}, +) +def processed_docs(): + return ( + dlt.read("extracted_line_items") + .filter("extraction_error IS NULL") + .selectExpr( + "path", + "doc_type", + "header:invoice_number::STRING AS invoice_number", + "header:vendor_name::STRING AS vendor_name", + "header:issue_date::STRING AS issue_date", + "header:total_amount::DOUBLE AS total_amount", + "line_items.line_items AS items", + ) + ) + + +@dlt.table(comment="Rows that failed at any extraction stage — review and reprocess") +def processing_errors(): + return ( + dlt.read("extracted_flat") + .filter("extract_error IS NOT NULL") + .select("path", "doc_type", col("extract_error").alias("error")) + .unionByName( + dlt.read("extracted_line_items") + .filter("extraction_error IS NOT NULL") + .select("path", "doc_type", col("extraction_error").alias("error")) + ) + ) +``` + +--- + +## Custom RAG Pipeline — Parse → Chunk → Index → Query + +When the goal is retrieval-augmented generation rather than field extraction, use this pipeline to parse documents, chunk them into a Delta table, and index with Vector Search. + +### Step 1 — Parse and Chunk into a Delta Table + +`ai_parse_document` returns a VARIANT. Use `variant_get` with an explicit `ARRAY` cast before calling `explode`, since `explode()` does not accept raw VARIANT values. + +```sql +CREATE OR REPLACE TABLE catalog.schema.parsed_chunks AS +WITH parsed AS ( + SELECT + path, + ai_parse_document(content) AS doc + FROM read_files('/Volumes/catalog/schema/volume/docs/', format => 'binaryFile') +), +elements AS ( + SELECT + path, + explode(variant_get(doc, '$.document.elements', 'ARRAY')) AS element + FROM parsed +) +SELECT + md5(concat(path, variant_get(element, '$.content', 'STRING'))) AS chunk_id, + path AS source_path, + variant_get(element, '$.content', 'STRING') AS content, + variant_get(element, '$.type', 'STRING') AS element_type, + current_timestamp() AS parsed_at +FROM elements +WHERE variant_get(element, '$.content', 'STRING') IS NOT NULL + AND length(trim(variant_get(element, '$.content', 'STRING'))) > 10; +``` + +### Step 1a (Production) — Incremental Parsing with Structured Streaming + +For production pipelines where new documents arrive over time, use Structured Streaming with checkpoints for exactly-once processing. Each run processes only new files, then stops with `trigger(availableNow=True)`. + +See the official bundle example: +[databricks/bundle-examples/contrib/job_with_ai_parse_document](https://github.com/databricks/bundle-examples/tree/main/contrib/job_with_ai_parse_document) + +**Stage 1 — Parse raw documents (streaming):** + +```python +from pyspark.sql.functions import col, current_timestamp, expr + +files_df = ( + spark.readStream.format("binaryFile") + .option("pathGlobFilter", "*.{pdf,jpg,jpeg,png}") + .option("recursiveFileLookup", "true") + .load("/Volumes/catalog/schema/volume/docs/") +) + +parsed_df = ( + files_df + .repartition(8, expr("crc32(path) % 8")) + .withColumn("parsed", expr(""" + ai_parse_document(content, map( + 'version', '2.0', + 'descriptionElementTypes', '*' + )) + """)) + .withColumn("parsed_at", current_timestamp()) + .select("path", "parsed", "parsed_at") +) + +( + parsed_df.writeStream.format("delta") + .outputMode("append") + .option("checkpointLocation", "/Volumes/catalog/schema/checkpoints/01_parse") + .option("mergeSchema", "true") + .trigger(availableNow=True) + .toTable("catalog.schema.parsed_documents_raw") +) +``` + +**Stage 2 — Extract text from parsed VARIANT (streaming):** + +Uses `transform()` to extract element content from the VARIANT array, and `try_cast` for safe access. Error rows are preserved but flagged. + +```python +from pyspark.sql.functions import col, concat_ws, expr, lit, when + +parsed_stream = spark.readStream.format("delta").table("catalog.schema.parsed_documents_raw") + +text_df = ( + parsed_stream + .withColumn("text", + when( + expr("try_cast(parsed:error_status AS STRING)").isNotNull(), lit(None) + ).otherwise( + concat_ws("\n\n", expr(""" + transform( + try_cast(parsed:document:elements AS ARRAY), + element -> try_cast(element:content AS STRING) + ) + """)) + ) + ) + .withColumn("error_status", expr("try_cast(parsed:error_status AS STRING)")) + .select("path", "text", "error_status", "parsed_at") +) + +( + text_df.writeStream.format("delta") + .outputMode("append") + .option("checkpointLocation", "/Volumes/catalog/schema/checkpoints/02_text") + .option("mergeSchema", "true") + .trigger(availableNow=True) + .toTable("catalog.schema.parsed_documents_text") +) +``` + +Key techniques: +- **`repartition` by file hash** — parallelizes `ai_parse_document` across workers +- **`trigger(availableNow=True)`** — processes all pending files then stops (batch-like) +- **Checkpoints** — exactly-once guarantee; no re-parsing on re-runs +- **`transform()` + `try_cast`** — safer than `explode` + `variant_get` for text extraction +- **Separate stages with independent checkpoints** — parse and text extraction can fail/retry independently + +### Step 1b — Enable Change Data Feed + +Required for Vector Search Delta Sync: + +```sql +ALTER TABLE catalog.schema.parsed_chunks +SET TBLPROPERTIES (delta.enableChangeDataFeed = true); +``` + +### Step 2 — Create a Vector Search Index and Query It + +Use the **[databricks-vector-search](../databricks-vector-search/SKILL.md)** skill to create a Delta Sync index on the chunked table and query it. Ensure CDF is enabled first (Step 1b above). + +### RAG-Specific Issues + +| Issue | Solution | +|-------|----------| +| `explode()` fails with VARIANT | `explode()` requires ARRAY, not VARIANT. Use `variant_get(doc, '$.document.elements', 'ARRAY')` to cast before exploding | +| Short/noisy chunks | Filter with `length(trim(...)) > 10` — parsing produces tiny fragments (page numbers, headers) that pollute the index | +| Re-parsing unchanged documents | Use Structured Streaming with checkpoints — see Step 1a above | +| Region not supported | US/EU regions only, or enable cross-geography routing | + +--- + +## Near-Real-Time Variant — DSPy + MLflow Agent + +When the pipeline must respond in seconds (triggered by a user action, API call, or form submission), use DSPy with an MLflow ChatAgent instead of a DLT pipeline. + +**When to use DSPy vs LangChain:** + +| Scenario | Stack | +|---|---| +| Fixed pipeline steps, well-defined I/O, want prompt optimization | **DSPy** | +| Needs tool-calling, memory, or multi-agent coordination | **LangChain LCEL** + MLflow ChatAgent | +| Single LLM call, simple task | Direct AI Function or `ai_query` in a notebook | + +### DSPy Signatures (replace LangChain agent system prompts) + +```python +# pip install dspy-ai mlflow databricks-sdk +import dspy, yaml + +CFG = yaml.safe_load(open("config.yml")) +lm = dspy.LM( + model=f"databricks/{CFG['models']['default']}", + api_base="https:///serving-endpoints", + api_key=dbutils.secrets.get("scope", "databricks-token"), +) +dspy.configure(lm=lm) + + +class ExtractInvoiceHeader(dspy.Signature): + """Extract invoice header fields from document text.""" + document_text: str = dspy.InputField(desc="Raw text from the document") + invoice_number: str = dspy.OutputField(desc="Invoice number, or null") + vendor_name: str = dspy.OutputField(desc="Vendor/supplier name, or null") + issue_date: str = dspy.OutputField(desc="Date as dd/mm/yyyy, or null") + total_amount: float = dspy.OutputField(desc="Total amount as float, or null") + + +class ClassifyDocument(dspy.Signature): + """Classify a document into one of the provided categories.""" + document_text: str = dspy.InputField() + category: str = dspy.OutputField( + desc="One of: invoice, purchase_order, receipt, contract, other" + ) + + +class DocumentPipeline(dspy.Module): + def __init__(self): + self.classify = dspy.Predict(ClassifyDocument) + self.extract = dspy.Predict(ExtractInvoiceHeader) + + def forward(self, document_text: str): + doc_type = self.classify(document_text=document_text).category + if doc_type == "invoice": + header = self.extract(document_text=document_text) + return {"doc_type": doc_type, "header": header.__dict__} + return {"doc_type": doc_type, "header": None} + + +pipeline = DocumentPipeline() +``` + +### Wrap and Register with MLflow + +```python +import mlflow, json + +class DSPyDocumentAgent(mlflow.pyfunc.PythonModel): + def load_context(self, context): + import dspy, yaml + cfg = yaml.safe_load(open(context.artifacts["config"])) + lm = dspy.LM(model=f"databricks/{cfg['models']['default']}") + dspy.configure(lm=lm) + self.pipeline = DocumentPipeline() + + def predict(self, context, model_input): + text = model_input.iloc[0]["document_text"] + return json.dumps(self.pipeline(document_text=text), ensure_ascii=False) + +mlflow.set_registry_uri("databricks-uc") +with mlflow.start_run(): + mlflow.pyfunc.log_model( + artifact_path="document_agent", + python_model=DSPyDocumentAgent(), + artifacts={"config": "config.yml"}, + registered_model_name="my_catalog.document_processing.document_agent", + ) +``` + +--- + +## Tips + +1. **Parse first, enrich second** — always run `ai_parse_document` as the first stage. Feed its text output to task-specific functions; never pass raw binary to `ai_query`. +2. **Flat or nested fields → `ai_extract`; deeply nested JSON exceeding 7 levels → `ai_query`** — pass `MAP('version', '2.0')` and access results through `:response`. +3. **`failOnError => false` is mandatory in batch** — write errors to a sidecar `_errors` table rather than crashing the pipeline. +4. **Truncate before sending to `ai_query`** — use `LEFT(text, 6000)` or chunk long documents to stay within context window limits. +5. **Prompts belong in `config.yml`** — never hardcode prompt strings in pipeline code. A prompt change should be a config change, not a code change. +6. **DSPy for agents** — when migrating from LangChain agent-based tools, DSPy typed `Signature` classes give you structured I/O contracts, testability, and optional prompt compilation/optimization. diff --git a/.claude/skills/databricks-ai-functions/SKILL.md b/.claude/skills/databricks-ai-functions/SKILL.md new file mode 100644 index 0000000..19897d8 --- /dev/null +++ b/.claude/skills/databricks-ai-functions/SKILL.md @@ -0,0 +1,195 @@ +--- +name: databricks-ai-functions +description: "Use Databricks built-in AI Functions (ai_classify, ai_extract, ai_summarize, ai_mask, ai_translate, ai_fix_grammar, ai_gen, ai_analyze_sentiment, ai_similarity, ai_parse_document, ai_query, ai_forecast) to add AI capabilities directly to SQL and PySpark pipelines without managing model endpoints. Also covers document parsing and building custom RAG pipelines (parse → chunk → index → query)." +--- + +# Databricks AI Functions + +> **Official Docs:** https://docs.databricks.com/aws/en/large-language-models/ai-functions +> Individual function reference: https://docs.databricks.com/aws/en/sql/language-manual/functions/ + +## Overview + +Databricks AI Functions are built-in SQL and PySpark functions that call Foundation Model APIs directly from your data pipelines — no model endpoint setup, no API keys, no boilerplate. They operate on table columns as naturally as `UPPER()` or `LENGTH()`, and are optimized for batch inference at scale. + +There are three categories: + +| Category | Functions | Use when | +|---|---|---| +| **Task-specific** | `ai_analyze_sentiment`, `ai_classify`, `ai_extract`, `ai_fix_grammar`, `ai_gen`, `ai_mask`, `ai_similarity`, `ai_summarize`, `ai_translate`, `ai_parse_document` | The task is well-defined — prefer these always | +| **General-purpose** | `ai_query` | Complex nested JSON, custom endpoints, multimodal — **last resort only** | +| **Table-valued** | `ai_forecast` | Time series forecasting | + +**Function selection rule — always prefer a task-specific function over `ai_query`:** + +| Task | Use this | Fall back to `ai_query` when... | +|---|---|---| +| Sentiment scoring | `ai_analyze_sentiment` | Never | +| Fixed-label routing | `ai_classify` (2–500 labels; add descriptions for accuracy) | Never | +| Entity / field extraction | `ai_extract` | Never | +| Summarization | `ai_summarize` | Never — use `max_words=0` for uncapped | +| Grammar correction | `ai_fix_grammar` | Never | +| Translation | `ai_translate` | Target language not in the supported list | +| PII redaction | `ai_mask` | Never | +| Free-form generation | `ai_gen` | Need structured JSON output | +| Semantic similarity | `ai_similarity` | Never | +| PDF / document parsing | `ai_parse_document` | Need image-level reasoning | +| Complex JSON / reasoning | — | **This is the intended use case for `ai_query`** | + +## Prerequisites + +- Databricks SQL warehouse (**not Classic**) or cluster with DBR **15.1+** +- DBR **15.4 ML LTS** recommended for batch workloads +- DBR **17.1+** required for `ai_parse_document` +- `ai_forecast` requires a **Pro or Serverless** SQL warehouse +- Workspace in a supported AWS/Azure region for batch AI inference +- Models run under Apache 2.0 or LLAMA 3.3 Community License — customers are responsible for compliance + +## Quick Start + +Classify, extract, and score sentiment from a text column in a single query: + +```sql +SELECT + ticket_id, + ticket_text, + ai_classify(ticket_text, ARRAY('urgent', 'not urgent', 'spam')) AS priority, + ai_extract(ticket_text, ARRAY('product', 'error_code', 'date')) AS entities, + ai_analyze_sentiment(ticket_text) AS sentiment +FROM support_tickets; +``` + +```python +from pyspark.sql.functions import expr + +df = spark.table("support_tickets") +df = ( + df.withColumn("priority", expr("ai_classify(ticket_text, array('urgent', 'not urgent', 'spam'))")) + .withColumn("entities", expr("ai_extract(ticket_text, array('product', 'error_code', 'date'))")) + .withColumn("sentiment", expr("ai_analyze_sentiment(ticket_text)")) +) +# Access nested STRUCT fields from ai_extract +df.select("ticket_id", "priority", "sentiment", + "entities.product", "entities.error_code", "entities.date").display() +``` + +## Common Patterns + +### Pattern 1: Text Analysis Pipeline + +Chain multiple task-specific functions to enrich a text column in one pass: + +```sql +SELECT + id, + content, + ai_analyze_sentiment(content) AS sentiment, + ai_summarize(content, 30) AS summary, + ai_classify(content, + ARRAY('technical', 'billing', 'other')) AS category, + ai_fix_grammar(content) AS content_clean +FROM raw_feedback; +``` + +### Pattern 2: PII Redaction Before Storage + +```python +from pyspark.sql.functions import expr + +df_clean = ( + spark.table("raw_messages") + .withColumn( + "message_safe", + expr("ai_mask(message, array('person', 'email', 'phone', 'address'))") + ) +) +df_clean.write.format("delta").mode("append").saveAsTable("catalog.schema.messages_safe") +``` + +### Pattern 3: Document Ingestion from a Unity Catalog Volume + +Parse PDFs/Office docs, then enrich with task-specific functions: + +```python +from pyspark.sql.functions import expr + +df = ( + spark.read.format("binaryFile") + .load("/Volumes/catalog/schema/landing/documents/") + .withColumn("parsed", expr("ai_parse_document(content)")) + .selectExpr("path", + "parsed:pages[*].elements[*].content AS text_blocks", + "parsed:error AS parse_error") + .filter("parse_error IS NULL") + .withColumn("summary", expr("ai_summarize(text_blocks, 50)")) + .withColumn("entities", expr("ai_extract(text_blocks, array('date', 'amount', 'vendor'))")) +) +``` + +### Pattern 4: Semantic Matching / Deduplication + +```sql +-- Find near-duplicate company names +SELECT a.id, b.id, ai_similarity(a.name, b.name) AS score +FROM companies a +JOIN companies b ON a.id < b.id +WHERE ai_similarity(a.name, b.name) > 0.85; +``` + +### Pattern 5: Complex JSON Extraction with `ai_query` (last resort) + +Use only when the output schema has nested arrays or requires multi-step reasoning that no task-specific function handles: + +```python +from pyspark.sql.functions import expr, from_json, col + +df = ( + spark.table("parsed_documents") + .withColumn("ai_response", expr(""" + ai_query( + 'databricks-claude-sonnet-4', + concat('Extract invoice as JSON with nested itens array: ', text_blocks), + responseFormat => '{"type":"json_object"}', + failOnError => false + ) + """)) + .withColumn("invoice", from_json( + col("ai_response.response"), + "STRUCT>>" + )) +) +``` + +### Pattern 6: Time Series Forecasting + +```sql +SELECT * +FROM ai_forecast( + observed => TABLE(SELECT date, sales FROM daily_sales), + horizon => '2026-12-31', + time_col => 'date', + value_col => 'sales' +); +-- Returns: date, sales_forecast, sales_upper, sales_lower +``` + +## Reference Files + +- [1-task-functions.md](1-task-functions.md) — Full syntax, parameters, SQL + PySpark examples for all 9 task-specific functions (`ai_analyze_sentiment`, `ai_classify`, `ai_extract`, `ai_fix_grammar`, `ai_gen`, `ai_mask`, `ai_similarity`, `ai_summarize`, `ai_translate`) and `ai_parse_document` +- [2-ai-query.md](2-ai-query.md) — `ai_query` complete reference: all parameters, structured output with `responseFormat`, multimodal `files =>`, UDF patterns, and error handling +- [3-ai-forecast.md](3-ai-forecast.md) — `ai_forecast` parameters, single-metric, multi-group, multi-metric, and confidence interval patterns +- [4-document-processing-pipeline.md](4-document-processing-pipeline.md) — End-to-end batch document processing pipeline using AI Functions in a Lakeflow Declarative Pipeline; includes `config.yml` centralization, function selection logic, custom RAG pipeline (parse → chunk → Vector Search), and DSPy/LangChain guidance for near-real-time variants + +## Common Issues + +| Issue | Solution | +|---|---| +| `ai_parse_document` not found | Requires DBR **17.1+**. Check cluster runtime. | +| `ai_forecast` fails | Requires **Pro or Serverless** SQL warehouse — not available on Classic or Starter. | +| All functions return NULL | Input column is NULL. Filter with `WHERE col IS NOT NULL` before calling. | +| `ai_translate` fails for a language | Supported: English, German, French, Italian, Portuguese, Hindi, Spanish, Thai. Use `ai_query` with a multilingual model for others. | +| `ai_classify` returns unexpected labels | Use clear, mutually exclusive label names. Fewer labels (2–5) produces more reliable results. | +| `ai_query` raises on some rows in a batch job | Add `failOnError => false` — returns a STRUCT with `.response` and `.error` instead of raising. | +| Batch job runs slowly | Use DBR **15.4 ML LTS** cluster (not serverless or interactive) for optimized batch inference throughput. | +| Want to swap models without editing pipeline code | Store all model names and prompts in `config.yml` — see [4-document-processing-pipeline.md](4-document-processing-pipeline.md) for the pattern. | diff --git a/.claude/skills/databricks-aibi-dashboards/1-widget-specifications.md b/.claude/skills/databricks-aibi-dashboards/1-widget-specifications.md new file mode 100644 index 0000000..8f23b16 --- /dev/null +++ b/.claude/skills/databricks-aibi-dashboards/1-widget-specifications.md @@ -0,0 +1,341 @@ +# Widget Specifications + +Core widget types for AI/BI dashboards. For advanced visualizations (area, scatter, choropleth map, combo), see [2-advanced-widget-specifications.md](2-advanced-widget-specifications.md). + +## Widget Naming and Display + +- `widget.name`: alphanumeric + hyphens + underscores ONLY (max 60 characters) +- `frame.title`: human-readable title (any characters allowed) +- `frame.showTitle`: always set to `true` so users understand the widget +- `displayName`: use in encodings to label axes/values clearly (e.g., "Revenue ($)", "Growth Rate (%)") +- `widget.queries[].name`: use `"main_query"` for chart/counter/table widgets. Filter widgets with multiple queries can use descriptive names (see [3-filters.md](3-filters.md)) + +**Always format values appropriately** - use `format` for currency, percentages, and large numbers (see [Axis Formatting](#axis-formatting)). + +## Version Requirements + +| Widget Type | Version | File | +|-------------|---------|------| +| text | N/A | this file | +| counter | 2 | this file | +| table | 2 | this file | +| bar | 3 | this file | +| line | 3 | this file | +| pie | 3 | this file | +| area | 3 | [2-advanced-widget-specifications.md](2-advanced-widget-specifications.md) | +| scatter | 3 | [2-advanced-widget-specifications.md](2-advanced-widget-specifications.md) | +| combo | 1 | [2-advanced-widget-specifications.md](2-advanced-widget-specifications.md) | +| choropleth-map | 1 | [2-advanced-widget-specifications.md](2-advanced-widget-specifications.md) | +| filter-* | 2 | [3-filters.md](3-filters.md) | + +--- + +## Text (Headers/Descriptions) + +- **CRITICAL: Text widgets do NOT use a spec block** - use `multilineTextboxSpec` directly +- Supports markdown: `#`, `##`, `###`, `**bold**`, `*italic*` +- **CRITICAL: Multiple items in the `lines` array are concatenated on a single line, NOT displayed as separate lines!** +- For title + subtitle, use **separate text widgets** at different y positions + +```json +// CORRECT: Separate widgets for title and subtitle +{ + "widget": { + "name": "title", + "multilineTextboxSpec": {"lines": ["## Dashboard Title"]} + }, + "position": {"x": 0, "y": 0, "width": 12, "height": 1} +}, +{ + "widget": { + "name": "subtitle", + "multilineTextboxSpec": {"lines": ["Description text here"]} + }, + "position": {"x": 0, "y": 1, "width": 12, "height": 1} +} + +// WRONG: Multiple lines concatenate into one line! +{ + "widget": { + "name": "title-widget", + "multilineTextboxSpec": { + "lines": ["## Dashboard Title", "Description text here"] // Becomes "## Dashboard TitleDescription text here" + } + }, + "position": {"x": 0, "y": 0, "width": 12, "height": 2} +} +``` + +--- + +## Counter (KPI) + +- `version`: **2** (NOT 3!) +- `widgetType`: "counter" +- Percent values must be 0-1 in the data (not 0-100) + +### Number Formatting + +```json +"encodings": { + "value": { + "fieldName": "revenue", + "displayName": "Total Revenue", + "format": { + "type": "number-currency", + "currencyCode": "USD", + "abbreviation": "compact", + "decimalPlaces": {"type": "max", "places": 2} + } + } +} +``` + +Format types: `number`, `number-currency`, `number-percent` + +### Counter Patterns + +**Pre-aggregated dataset (1 row)** - use `disaggregated: true`: +```json +{ + "widget": { + "name": "total-revenue", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "summary_ds", + "fields": [{"name": "revenue", "expression": "`revenue`"}], + "disaggregated": true + } + }], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": {"fieldName": "revenue", "displayName": "Total Revenue"} + }, + "frame": {"showTitle": true, "title": "Total Revenue"} + } + }, + "position": {"x": 0, "y": 0, "width": 4, "height": 3} +} +``` + +**Multi-row dataset with aggregation (supports filters)** - use `disaggregated: false`: +- Dataset returns multiple rows (e.g., grouped by a filter dimension) +- Use `"disaggregated": false` and aggregation expression +- **CRITICAL**: Field `name` MUST match `fieldName` exactly (e.g., `"sum(spend)"`) + +```json +{ + "widget": { + "name": "total-spend", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "by_category", + "fields": [{"name": "sum(spend)", "expression": "SUM(`spend`)"}], + "disaggregated": false + } + }], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": {"fieldName": "sum(spend)", "displayName": "Total Spend"} + }, + "frame": {"showTitle": true, "title": "Total Spend"} + } + }, + "position": {"x": 0, "y": 0, "width": 4, "height": 3} +} +``` + +--- + +## Table + +- `version`: **2** (NOT 1 or 3!) +- `widgetType`: "table" +- **Columns only need `fieldName` and `displayName`** - no other properties required +- Use `"disaggregated": true` for raw rows +- Default sort: use `ORDER BY` in dataset SQL + +```json +{ + "widget": { + "name": "details-table", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "details_ds", + "fields": [ + {"name": "name", "expression": "`name`"}, + {"name": "value", "expression": "`value`"} + ], + "disaggregated": true + } + }], + "spec": { + "version": 2, + "widgetType": "table", + "encodings": { + "columns": [ + {"fieldName": "name", "displayName": "Name"}, + {"fieldName": "value", "displayName": "Value"} + ] + }, + "frame": {"showTitle": true, "title": "Details"} + } + }, + "position": {"x": 0, "y": 0, "width": 12, "height": 6} +} +``` + +--- + +## Line / Bar Charts + +- `version`: **3** +- `widgetType`: "line" or "bar" +- Use `x`, `y`, optional `color` encodings +- `scale.type`: `"temporal"` (dates), `"quantitative"` (numbers), `"categorical"` (strings) +- Use `"disaggregated": true` with pre-aggregated dataset data + +**Multiple series - two approaches:** + +1. **Multi-Y Fields** (different metrics): +```json +"y": { + "scale": {"type": "quantitative"}, + "fields": [ + {"fieldName": "sum(orders)", "displayName": "Orders"}, + {"fieldName": "sum(returns)", "displayName": "Returns"} + ] +} +``` + +2. **Color Grouping** (same metric split by dimension): +```json +"y": {"fieldName": "sum(revenue)", "scale": {"type": "quantitative"}}, +"color": {"fieldName": "region", "scale": {"type": "categorical"}} +``` + +### Bar Chart Modes + +| Mode | Configuration | +|------|---------------| +| Stacked (default) | No `mark` field | +| Grouped | `"mark": {"layout": "group"}` | + +### Horizontal Bar Chart + +Swap `x` and `y` - put quantitative on `x`, categorical/temporal on `y`: +```json +"encodings": { + "x": {"scale": {"type": "quantitative"}, "fields": [...]}, + "y": {"fieldName": "category", "scale": {"type": "categorical"}} +} +``` + +### Color Scale + +> **CRITICAL**: For bar/line/pie, color scale ONLY supports `type` and `sort`. +> Do NOT use `scheme`, `colorRamp`, or `mappings` (only for choropleth-map). + +--- + +## Pie Chart + +- `version`: **3** +- `widgetType`: "pie" +- `angle`: quantitative field +- `color`: categorical dimension +- **Limit to 3-8 categories for readability** + +```json +"spec": { + "version": 3, + "widgetType": "pie", + "encodings": { + "angle": {"fieldName": "revenue", "scale": {"type": "quantitative"}}, + "color": {"fieldName": "category", "scale": {"type": "categorical"}} + } +} +``` + +--- + +## Axis Formatting + +Add `format` to any encoding to display values appropriately: + +| Data Type | Format Type | Example | +|-----------|-------------|---------| +| Currency | `number-currency` | $1.2M | +| Percentage | `number-percent` | 45.2% (data must be 0-1, not 0-100) | +| Large numbers | `number` with `abbreviation` | 1.5K, 2.3M | + +```json +"value": { + "fieldName": "revenue", + "displayName": "Revenue", + "format": { + "type": "number-currency", + "currencyCode": "USD", + "abbreviation": "compact", + "decimalPlaces": {"type": "max", "places": 2} + } +} +``` + +**Options:** +- `abbreviation`: `"compact"` (K/M/B) or omit for full numbers +- `decimalPlaces`: `{"type": "max", "places": N}` or `{"type": "fixed", "places": N}` + +--- + +## Dataset Parameters + +Use `:param` syntax in SQL for dynamic filtering: + +```json +{ + "name": "revenue_by_category", + "queryLines": ["SELECT ... WHERE returns_usd > :threshold GROUP BY category"], + "parameters": [{ + "keyword": "threshold", + "dataType": "INTEGER", + "defaultSelection": {} + }] +} +``` + +**Parameter types:** +- Single value: `"dataType": "INTEGER"` / `"DECIMAL"` / `"STRING"` +- Multi-select: Add `"complexType": "MULTI"` +- Range: `"dataType": "DATE", "complexType": "RANGE"` - use `:param.min` / `:param.max` + +--- + +## Widget Field Expressions + +Allowed in `query.fields` (no CAST or complex SQL): + +```json +// Aggregations +{"name": "sum(revenue)", "expression": "SUM(`revenue`)"} +{"name": "avg(price)", "expression": "AVG(`price`)"} +{"name": "count(id)", "expression": "COUNT(`id`)"} +{"name": "countdistinct(id)", "expression": "COUNT(DISTINCT `id`)"} + +// Date truncation +{"name": "daily(date)", "expression": "DATE_TRUNC(\"DAY\", `date`)"} +{"name": "weekly(date)", "expression": "DATE_TRUNC(\"WEEK\", `date`)"} +{"name": "monthly(date)", "expression": "DATE_TRUNC(\"MONTH\", `date`)"} + +// Simple reference +{"name": "category", "expression": "`category`"} +``` + +For conditional logic, compute in dataset SQL instead. diff --git a/.claude/skills/databricks-aibi-dashboards/2-advanced-widget-specifications.md b/.claude/skills/databricks-aibi-dashboards/2-advanced-widget-specifications.md new file mode 100644 index 0000000..3f65860 --- /dev/null +++ b/.claude/skills/databricks-aibi-dashboards/2-advanced-widget-specifications.md @@ -0,0 +1,177 @@ +# Advanced Widget Specifications + +Advanced visualization types for AI/BI dashboards. For core widgets (text, counter, table, bar, line, pie), see [1-widget-specifications.md](1-widget-specifications.md). + +--- + +## Area Chart + +- `version`: **3** +- `widgetType`: "area" +- Same structure as line chart - useful for showing cumulative values or emphasizing volume + +```json +"spec": { + "version": 3, + "widgetType": "area", + "encodings": { + "x": {"fieldName": "week_start", "scale": {"type": "temporal"}}, + "y": { + "scale": {"type": "quantitative"}, + "fields": [ + {"fieldName": "revenue_usd", "displayName": "Revenue"}, + {"fieldName": "returns_usd", "displayName": "Returns"} + ] + } + } +} +``` + +--- + +## Scatter Plot / Bubble Chart + +- `version`: **3** +- `widgetType`: "scatter" +- `x`, `y`: quantitative or temporal +- `size`: optional quantitative field for bubble size +- `color`: optional categorical or quantitative for grouping + +```json +"spec": { + "version": 3, + "widgetType": "scatter", + "encodings": { + "x": {"fieldName": "return_date", "scale": {"type": "temporal"}}, + "y": {"fieldName": "daily_returns", "scale": {"type": "quantitative"}}, + "size": {"fieldName": "count(*)", "scale": {"type": "quantitative"}}, + "color": {"fieldName": "category", "scale": {"type": "categorical"}} + } +} +``` + +--- + +## Combo Chart (Bar + Line) + +Combines bar and line visualizations on the same chart - useful for showing related metrics with different scales. + +- `version`: **1** +- `widgetType`: "combo" +- `y.primary`: bar chart fields +- `y.secondary`: line chart fields + +```json +{ + "widget": { + "name": "revenue-and-growth", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "metrics_ds", + "fields": [ + {"name": "daily(date)", "expression": "DATE_TRUNC(\"DAY\", `date`)"}, + {"name": "sum(revenue)", "expression": "SUM(`revenue`)"}, + {"name": "avg(growth_rate)", "expression": "AVG(`growth_rate`)"} + ], + "disaggregated": false + } + }], + "spec": { + "version": 1, + "widgetType": "combo", + "encodings": { + "x": {"fieldName": "daily(date)", "scale": {"type": "temporal"}}, + "y": { + "scale": {"type": "quantitative"}, + "primary": { + "fields": [{"fieldName": "sum(revenue)", "displayName": "Revenue ($)"}] + }, + "secondary": { + "fields": [{"fieldName": "avg(growth_rate)", "displayName": "Growth Rate"}] + } + }, + "label": {"show": false} + }, + "frame": {"title": "Revenue & Growth Rate", "showTitle": true} + } + }, + "position": {"x": 0, "y": 0, "width": 12, "height": 5} +} +``` + +--- + +## Choropleth Map + +Displays geographic regions colored by aggregate values. Requires a field with geographic names (state names, country names, etc.). + +- `version`: **1** +- `widgetType`: "choropleth-map" +- `region`: defines the geographic area mapping +- `color`: quantitative field for coloring regions + +```json +"spec": { + "version": 1, + "widgetType": "choropleth-map", + "encodings": { + "region": { + "regionType": "mapbox-v4-admin", + "admin0": { + "type": "value", + "value": "United States", + "geographicRole": "admin0-name" + }, + "admin1": { + "fieldName": "state_name", + "type": "field", + "geographicRole": "admin1-name" + } + }, + "color": { + "fieldName": "sum(revenue)", + "scale": {"type": "quantitative"} + } + } +} +``` + +### Region Configuration + +**Region levels:** +- `admin0`: Country level - use `"type": "value"` with fixed country name +- `admin1`: State/Province level - use `"type": "field"` with your data column +- `admin2`: County/District level + +**Geographic roles:** +- `admin0-name`, `admin1-name`, `admin2-name` - match by name +- `admin0-iso`, `admin1-iso` - match by ISO code + +**Supported countries for admin1:** United States, Japan (prefectures), and others. + +### Color Scale for Maps + +> **Note**: Unlike other charts, choropleth-map supports additional color scale properties: +> - `scheme`: color scheme name (e.g., "YIGnBu") +> - `colorRamp`: custom color gradient +> - `mappings`: explicit value-to-color mappings + +--- + +## Other Visualization Types + +The following visualization types are available in Databricks AI/BI dashboards but are less commonly used. Refer to [Databricks documentation](https://docs.databricks.com/aws/en/visualizations/visualization-types) for details: + +| Widget Type | Description | +|-------------|-------------| +| heatmap | Color intensity grid for numerical data | +| histogram | Frequency distribution with configurable bins | +| funnel | Stage-based metric analysis | +| sankey | Flow visualization between value sets | +| box | Distribution summary with quartiles | +| marker-map | Latitude/longitude point markers | +| pivot | Drag-and-drop aggregation table | +| word-cloud | Word frequency visualization | +| sunburst | Hierarchical data in concentric circles | +| cohort | Group outcome analysis over time | diff --git a/.claude/skills/databricks-aibi-dashboards/3-examples.md b/.claude/skills/databricks-aibi-dashboards/3-examples.md new file mode 100644 index 0000000..cb8791d --- /dev/null +++ b/.claude/skills/databricks-aibi-dashboards/3-examples.md @@ -0,0 +1,308 @@ +# Complete Dashboard Examples + +Production-ready templates you can adapt for your use case. + +## Basic Dashboard (NYC Taxi) + +```python +import json + +# Step 1: Check table schema +table_info = get_table_stats_and_schema(catalog="samples", schema="nyctaxi") + +# Step 2: Test queries +execute_sql("SELECT COUNT(*) as trips, AVG(fare_amount) as avg_fare, AVG(trip_distance) as avg_distance FROM samples.nyctaxi.trips") +execute_sql(""" + SELECT pickup_zip, COUNT(*) as trip_count + FROM samples.nyctaxi.trips + GROUP BY pickup_zip + ORDER BY trip_count DESC + LIMIT 10 +""") + +# Step 3: Build dashboard JSON +dashboard = { + "datasets": [ + { + "name": "summary", + "displayName": "Summary Stats", + "queryLines": [ + "SELECT COUNT(*) as trips, AVG(fare_amount) as avg_fare, ", + "AVG(trip_distance) as avg_distance ", + "FROM samples.nyctaxi.trips " + ] + }, + { + "name": "by_zip", + "displayName": "Trips by ZIP", + "queryLines": [ + "SELECT pickup_zip, COUNT(*) as trip_count ", + "FROM samples.nyctaxi.trips ", + "GROUP BY pickup_zip ", + "ORDER BY trip_count DESC ", + "LIMIT 10 " + ] + } + ], + "pages": [{ + "name": "overview", + "displayName": "NYC Taxi Overview", + "pageType": "PAGE_TYPE_CANVAS", + "layoutVersion": "GRID_V1", + "layout": [ + # Text header - NO spec block! Use SEPARATE widgets for title and subtitle! + { + "widget": { + "name": "title", + "multilineTextboxSpec": { + "lines": ["## NYC Taxi Dashboard"] + } + }, + "position": {"x": 0, "y": 0, "width": 12, "height": 1} + }, + { + "widget": { + "name": "subtitle", + "multilineTextboxSpec": { + "lines": ["Trip statistics and analysis"] + } + }, + "position": {"x": 0, "y": 1, "width": 12, "height": 1} + }, + # Counter - version 2, width 4! + { + "widget": { + "name": "total-trips", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "summary", + "fields": [{"name": "trips", "expression": "`trips`"}], + "disaggregated": True + } + }], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": {"fieldName": "trips", "displayName": "Total Trips"} + }, + "frame": {"title": "Total Trips", "showTitle": True} + } + }, + "position": {"x": 0, "y": 2, "width": 4, "height": 3} + }, + { + "widget": { + "name": "avg-fare", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "summary", + "fields": [{"name": "avg_fare", "expression": "`avg_fare`"}], + "disaggregated": True + } + }], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": {"fieldName": "avg_fare", "displayName": "Avg Fare"} + }, + "frame": {"title": "Average Fare", "showTitle": True} + } + }, + "position": {"x": 4, "y": 2, "width": 4, "height": 3} + }, + { + "widget": { + "name": "total-distance", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "summary", + "fields": [{"name": "avg_distance", "expression": "`avg_distance`"}], + "disaggregated": True + } + }], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": {"fieldName": "avg_distance", "displayName": "Avg Distance"} + }, + "frame": {"title": "Average Distance", "showTitle": True} + } + }, + "position": {"x": 8, "y": 2, "width": 4, "height": 3} + }, + # Bar chart - version 3 + { + "widget": { + "name": "trips-by-zip", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "by_zip", + "fields": [ + {"name": "pickup_zip", "expression": "`pickup_zip`"}, + {"name": "trip_count", "expression": "`trip_count`"} + ], + "disaggregated": True + } + }], + "spec": { + "version": 3, + "widgetType": "bar", + "encodings": { + "x": {"fieldName": "pickup_zip", "scale": {"type": "categorical"}, "displayName": "ZIP"}, + "y": {"fieldName": "trip_count", "scale": {"type": "quantitative"}, "displayName": "Trips"} + }, + "frame": {"title": "Trips by Pickup ZIP", "showTitle": True} + } + }, + "position": {"x": 0, "y": 5, "width": 12, "height": 5} + }, + # Table - version 2, minimal column props! + { + "widget": { + "name": "zip-table", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "by_zip", + "fields": [ + {"name": "pickup_zip", "expression": "`pickup_zip`"}, + {"name": "trip_count", "expression": "`trip_count`"} + ], + "disaggregated": True + } + }], + "spec": { + "version": 2, + "widgetType": "table", + "encodings": { + "columns": [ + {"fieldName": "pickup_zip", "displayName": "ZIP Code"}, + {"fieldName": "trip_count", "displayName": "Trip Count"} + ] + }, + "frame": {"title": "Top ZIP Codes", "showTitle": True} + } + }, + "position": {"x": 0, "y": 10, "width": 12, "height": 5} + } + ] + }] +} + +# Step 4: Deploy +result = manage_dashboard( + action="create_or_update", + display_name="NYC Taxi Dashboard", + parent_path="/Workspace/Users/me/dashboards", + serialized_dashboard=json.dumps(dashboard), + warehouse_id=manage_warehouse(action="get_best"), +) +print(result["url"]) +``` + +## Dashboard with Global Filters + +```python +import json + +# Dashboard with a global filter for region +dashboard_with_filters = { + "datasets": [ + { + "name": "sales", + "displayName": "Sales Data", + "queryLines": [ + "SELECT region, SUM(revenue) as total_revenue ", + "FROM catalog.schema.sales ", + "GROUP BY region" + ] + } + ], + "pages": [ + { + "name": "overview", + "displayName": "Sales Overview", + "pageType": "PAGE_TYPE_CANVAS", + "layoutVersion": "GRID_V1", + "layout": [ + { + "widget": { + "name": "total-revenue", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "sales", + "fields": [{"name": "total_revenue", "expression": "`total_revenue`"}], + "disaggregated": True + } + }], + "spec": { + "version": 2, # Version 2 for counters! + "widgetType": "counter", + "encodings": { + "value": {"fieldName": "total_revenue", "displayName": "Total Revenue"} + }, + "frame": {"title": "Total Revenue", "showTitle": True} + } + }, + "position": {"x": 0, "y": 0, "width": 12, "height": 3} + } + ] + }, + { + "name": "filters", + "displayName": "Filters", + "pageType": "PAGE_TYPE_GLOBAL_FILTERS", # Required for global filter page! + "layoutVersion": "GRID_V1", + "layout": [ + { + "widget": { + "name": "filter_region", + "queries": [{ + "name": "ds_sales_region", + "query": { + "datasetName": "sales", + "fields": [ + {"name": "region", "expression": "`region`"} + # DO NOT use associative_filter_predicate_group - causes SQL errors! + ], + "disaggregated": False # False for filters! + } + }], + "spec": { + "version": 2, # Version 2 for filters! + "widgetType": "filter-multi-select", # NOT "filter"! + "encodings": { + "fields": [{ + "fieldName": "region", + "displayName": "Region", + "queryName": "ds_sales_region" # Must match query name! + }] + }, + "frame": {"showTitle": True, "title": "Region"} # Always show title! + } + }, + "position": {"x": 0, "y": 0, "width": 4, "height": 2} + } + ] + } + ] +} + +# Deploy with filters +result = manage_dashboard( + action="create_or_update", + display_name="Sales Dashboard with Filters", + parent_path="/Workspace/Users/me/dashboards", + serialized_dashboard=json.dumps(dashboard_with_filters), + warehouse_id=manage_warehouse(action="get_best"), +) +print(result["url"]) +``` diff --git a/.claude/skills/databricks-aibi-dashboards/3-filters.md b/.claude/skills/databricks-aibi-dashboards/3-filters.md new file mode 100644 index 0000000..5a4ab49 --- /dev/null +++ b/.claude/skills/databricks-aibi-dashboards/3-filters.md @@ -0,0 +1,242 @@ +# Filters (Global vs Page-Level) + +> **CRITICAL**: Filter widgets use DIFFERENT widget types than charts! +> - Valid types: `filter-multi-select`, `filter-single-select`, `filter-date-range-picker` +> - **DO NOT** use `widgetType: "filter"` - this does not exist and will cause errors +> - Filters use `spec.version: 2` +> - **ALWAYS include `frame` with `showTitle: true`** for filter widgets + +**Filter widget types:** +- `filter-date-range-picker`: for DATE/TIMESTAMP fields (date range selection) +- `filter-single-select`: categorical with single selection +- `filter-multi-select`: categorical with multiple selections (preferred for drill-down) + +> **Performance note**: Global filters automatically apply `WHERE` clauses to dataset queries at runtime. You don't need to pre-filter data in your SQL - the dashboard engine handles this efficiently. + +--- + +## Global Filters vs Page-Level Filters + +| Type | Placement | Scope | Use Case | +|------|-----------|-------|----------| +| **Global Filter** | Dedicated page with `"pageType": "PAGE_TYPE_GLOBAL_FILTERS"` | Affects ALL pages that have datasets with the filter field | Cross-dashboard filtering (e.g., date range, campaign) | +| **Page-Level Filter** | Regular page with `"pageType": "PAGE_TYPE_CANVAS"` | Affects ONLY widgets on that same page | Page-specific filtering (e.g., platform filter on breakdown page only) | + +**Key Insight**: A filter only affects datasets that contain the filter field. To have a filter affect only specific pages: +1. Include the filter dimension in datasets for pages that should be filtered +2. Exclude the filter dimension from datasets for pages that should NOT be filtered + +--- + +## Filter Widget Structure + +> **CRITICAL**: Do NOT use `associative_filter_predicate_group` - it causes SQL errors! +> Use a simple field expression instead. + +```json +{ + "widget": { + "name": "filter_region", + "queries": [{ + "name": "ds_data_region", // Query name - must match queryName in encodings! + "query": { + "datasetName": "ds_data", + "fields": [ + {"name": "region", "expression": "`region`"} + ], + "disaggregated": false // CRITICAL: Always false for filters! + } + }], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [{ + "fieldName": "region", + "displayName": "Region", + "queryName": "ds_data_region" // Must match queries[].name above! + }] + }, + "frame": {"showTitle": true, "title": "Region"} + } + }, + "position": {"x": 0, "y": 0, "width": 4, "height": 2} +} +``` + +--- + +## Global Filter Example + +Place on a dedicated filter page: + +```json +{ + "name": "filters", + "displayName": "Filters", + "pageType": "PAGE_TYPE_GLOBAL_FILTERS", + "layoutVersion": "GRID_V1", + "layout": [ + { + "widget": { + "name": "filter_campaign", + "queries": [{ + "name": "ds_campaign", + "query": { + "datasetName": "overview", + "fields": [{"name": "campaign_name", "expression": "`campaign_name`"}], + "disaggregated": false + } + }], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [{ + "fieldName": "campaign_name", + "displayName": "Campaign", + "queryName": "ds_campaign" + }] + }, + "frame": {"showTitle": true, "title": "Campaign"} + } + }, + "position": {"x": 0, "y": 0, "width": 4, "height": 2} + } + ] +} +``` + +--- + +## Page-Level Filter Example + +Place filter widget directly on a `PAGE_TYPE_CANVAS` page (same widget structure as global filter, but only affects that page): + +```json +{ + "name": "platform_breakdown", + "displayName": "Platform Breakdown", + "pageType": "PAGE_TYPE_CANVAS", + "layoutVersion": "GRID_V1", + "layout": [ + {"widget": {...}, "position": {...}}, + { + "widget": { + "name": "filter_platform", + "queries": [{"name": "ds_platform", "query": {"datasetName": "platform_data", "fields": [{"name": "platform", "expression": "`platform`"}], "disaggregated": false}}], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": {"fields": [{"fieldName": "platform", "displayName": "Platform", "queryName": "ds_platform"}]}, + "frame": {"showTitle": true, "title": "Platform"} + } + }, + "position": {"x": 8, "y": 0, "width": 4, "height": 2} + } + ] +} +``` + +--- + +## Date Range Filtering + +> **Best Practice**: Most dashboards should include a date range filter. However, metrics that are not based on a time range (like "MRR" or "All-Time Total") should NOT be date-filtered - omit them from the filter's queries. + +**Two binding approaches** (can be combined in one filter): +- **Field-based**: Bind to a date column in SELECT → filter auto-applies `IN_RANGE()` +- **Parameter-based**: Use `:param.min`/`:param.max` in WHERE clause for pre-aggregation filtering + +```json +// Dataset with parameter (for aggregated queries) +{ + "name": "revenue_by_category", + "queryLines": [ + "SELECT category, SUM(revenue) as revenue FROM catalog.schema.orders ", + "WHERE order_date BETWEEN :date_range.min AND :date_range.max ", + "GROUP BY category" + ], + "parameters": [{ + "keyword": "date_range", "dataType": "DATE", "complexType": "RANGE", + "defaultSelection": {"range": {"dataType": "DATE", "min": {"value": "now-12M/M"}, "max": {"value": "now/M"}}} + }] +} + +// Filter widget binding to both field and parameter +{ + "widget": { + "name": "date_range_filter", + "queries": [ + {"name": "q_trend", "query": {"datasetName": "weekly_trend", "fields": [{"name": "week_start", "expression": "`week_start`"}], "disaggregated": false}}, + {"name": "q_category", "query": {"datasetName": "revenue_by_category", "parameters": [{"name": "date_range", "keyword": "date_range"}], "disaggregated": false}} + ], + "spec": { + "version": 2, + "widgetType": "filter-date-range-picker", + "encodings": { + "fields": [ + {"fieldName": "week_start", "queryName": "q_trend"}, + {"parameterName": "date_range", "queryName": "q_category"} + ] + }, + "frame": {"showTitle": true, "title": "Date Range"} + } + }, + "position": {"x": 0, "y": 0, "width": 4, "height": 2} +} +``` + +--- + +## Multi-Dataset Filters + +When a filter should affect multiple datasets (e.g., "Region" filter for both sales and customers data), add multiple queries - one per dataset: + +```json +{ + "widget": { + "name": "filter_region", + "queries": [ + { + "name": "sales_region", + "query": { + "datasetName": "sales", + "fields": [{"name": "region", "expression": "`region`"}], + "disaggregated": false + } + }, + { + "name": "customers_region", + "query": { + "datasetName": "customers", + "fields": [{"name": "region", "expression": "`region`"}], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [ + {"fieldName": "region", "displayName": "Region (Sales)", "queryName": "sales_region"}, + {"fieldName": "region", "displayName": "Region (Customers)", "queryName": "customers_region"} + ] + }, + "frame": {"showTitle": true, "title": "Region"} + } + }, + "position": {"x": 0, "y": 0, "width": 4, "height": 2} +} +``` + +Each `queryName` in `encodings.fields` binds the filter to that specific dataset. Datasets not bound will not be filtered. + +--- + +## Filter Layout Guidelines + +- Global filters: Position on dedicated filter page, stack vertically at `x=0` +- Page-level filters: Position in header area of page (e.g., top-right corner) +- Typical sizing: `width: 4, height: 2` diff --git a/.claude/skills/databricks-aibi-dashboards/4-examples.md b/.claude/skills/databricks-aibi-dashboards/4-examples.md new file mode 100644 index 0000000..8fa49c5 --- /dev/null +++ b/.claude/skills/databricks-aibi-dashboards/4-examples.md @@ -0,0 +1,498 @@ +# Complete Dashboard Example + +This is a **reference example** to understand the JSON structure and layout patterns. **Always adapt to what the user requests** - use their tables, metrics, and visualizations. This example demonstrates the correct syntax; your dashboard should reflect the user's actual requirements. + +## Key Patterns (Read First) + +### 1. Page Types (Required) +- `PAGE_TYPE_CANVAS` - Main content page with widgets +- `PAGE_TYPE_GLOBAL_FILTERS` - Dedicated filter page that affects all canvas pages + +### 2. Widget Versions (Critical!) +| Widget Type | Version | +|-------------|---------| +| `counter`, `table` | **2** | +| `bar`, `line`, `area`, `pie` | **3** | +| `filter-*` | **2** | + +### 3. KPI Counter with Currency Formatting +```json +"format": { + "type": "number-currency", + "currencyCode": "USD", + "abbreviation": "compact", + "decimalPlaces": {"type": "max", "places": 1} +} +``` + +### 4. Filter Binding to Multiple Datasets +Each filter query binds the filter to one dataset. Add multiple queries to filter multiple datasets: +```json +"queries": [ + {"name": "ds1_region", "query": {"datasetName": "dataset1", ...}}, + {"name": "ds2_region", "query": {"datasetName": "dataset2", ...}} +] +``` + +### 5. Layout Grid (12 columns) +``` +y=0: Header with title + description (w=12, h=2) +y=2: KPI(w=4,h=3) | KPI(w=4,h=3) | KPI(w=4,h=3) ← fills 12 +y=5: Section header (w=12, h=1) +y=6: Area chart (w=12, h=5) +y=11: Section header (w=12, h=1) +y=12: Pie(w=4,h=5) | Bar chart(w=8,h=5) ← fills 12 +``` + +Use `\n\n` in text widget lines array to create line breaks within a single widget. + +--- + +## Full Dashboard: Sales Analytics + +This example shows a complete dashboard with: +- Title and subtitle text widgets +- 3 KPI counters with currency/number formatting +- Area chart for time series trends +- Pie chart for category breakdown +- Bar chart with color grouping by region +- Data table for detailed records +- Global filters (date range, region, category) + +```json +{ + "datasets": [ + { + "name": "ds_daily_sales", + "displayName": "Daily Sales", + "queryLines": [ + "SELECT sale_date, region, department, total_orders, total_units, total_revenue, total_cost, profit_margin ", + "FROM catalog.schema.gold_daily_sales ", + "ORDER BY sale_date" + ] + }, + { + "name": "ds_products", + "displayName": "Product Performance", + "queryLines": [ + "SELECT product_id, product_name, department, region, units_sold, revenue, cost, profit ", + "FROM catalog.schema.gold_product_performance" + ] + } + ], + "pages": [ + { + "name": "sales_overview", + "displayName": "Sales Overview", + "pageType": "PAGE_TYPE_CANVAS", + "layoutVersion": "GRID_V1", + "layout": [ + { + "widget": { + "name": "header", + "multilineTextboxSpec": { + "lines": ["# Sales Dashboard\n\nMonitor daily sales, revenue, and profit margins across regions and departments."] + } + }, + "position": {"x": 0, "y": 0, "width": 12, "height": 2} + }, + { + "widget": { + "name": "kpi_revenue", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "ds_daily_sales", + "fields": [{"name": "sum(total_revenue)", "expression": "SUM(`total_revenue`)"}], + "disaggregated": false + } + }], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "sum(total_revenue)", + "displayName": "Total Revenue", + "format": { + "type": "number-currency", + "currencyCode": "USD", + "abbreviation": "compact", + "decimalPlaces": {"type": "max", "places": 1} + } + } + }, + "frame": {"title": "Total Revenue", "showTitle": true, "description": "For the selected period", "showDescription": true} + } + }, + "position": {"x": 0, "y": 2, "width": 4, "height": 3} + }, + { + "widget": { + "name": "kpi_orders", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "ds_daily_sales", + "fields": [{"name": "sum(total_orders)", "expression": "SUM(`total_orders`)"}], + "disaggregated": false + } + }], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "sum(total_orders)", + "displayName": "Total Orders", + "format": { + "type": "number", + "abbreviation": "compact", + "decimalPlaces": {"type": "max", "places": 0} + } + } + }, + "frame": {"title": "Total Orders", "showTitle": true, "description": "For the selected period", "showDescription": true} + } + }, + "position": {"x": 4, "y": 2, "width": 4, "height": 3} + }, + { + "widget": { + "name": "kpi_profit", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "ds_daily_sales", + "fields": [{"name": "avg(profit_margin)", "expression": "AVG(`profit_margin`)"}], + "disaggregated": false + } + }], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "avg(profit_margin)", + "displayName": "Avg Profit Margin", + "format": { + "type": "number-percent", + "decimalPlaces": {"type": "max", "places": 1} + } + } + }, + "frame": {"title": "Profit Margin", "showTitle": true, "description": "Average for period", "showDescription": true} + } + }, + "position": {"x": 8, "y": 2, "width": 4, "height": 3} + }, + { + "widget": { + "name": "section_trends", + "multilineTextboxSpec": { + "lines": ["## Revenue Trend"] + } + }, + "position": {"x": 0, "y": 5, "width": 12, "height": 1} + }, + { + "widget": { + "name": "chart_revenue_trend", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "ds_daily_sales", + "fields": [ + {"name": "sale_date", "expression": "`sale_date`"}, + {"name": "sum(total_revenue)", "expression": "SUM(`total_revenue`)"} + ], + "disaggregated": false + } + }], + "spec": { + "version": 3, + "widgetType": "area", + "encodings": { + "x": { + "fieldName": "sale_date", + "scale": {"type": "temporal"}, + "axis": {"title": "Date"}, + "displayName": "Date" + }, + "y": { + "fieldName": "sum(total_revenue)", + "scale": {"type": "quantitative"}, + "format": { + "type": "number-currency", + "currencyCode": "USD", + "abbreviation": "compact" + }, + "axis": {"title": "Revenue ($)"}, + "displayName": "Revenue ($)" + } + }, + "frame": { + "title": "Daily Revenue", + "showTitle": true, + "description": "Track daily revenue trends" + } + } + }, + "position": {"x": 0, "y": 6, "width": 12, "height": 5} + }, + { + "widget": { + "name": "section_breakdown", + "multilineTextboxSpec": { + "lines": ["## Breakdown"] + } + }, + "position": {"x": 0, "y": 11, "width": 12, "height": 1} + }, + { + "widget": { + "name": "chart_by_department", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "ds_daily_sales", + "fields": [ + {"name": "department", "expression": "`department`"}, + {"name": "sum(total_revenue)", "expression": "SUM(`total_revenue`)"} + ], + "disaggregated": false + } + }], + "spec": { + "version": 3, + "widgetType": "pie", + "encodings": { + "angle": { + "fieldName": "sum(total_revenue)", + "scale": {"type": "quantitative"}, + "displayName": "Revenue" + }, + "color": { + "fieldName": "department", + "scale": {"type": "categorical"}, + "displayName": "Department" + }, + "label": {"show": true} + }, + "frame": {"title": "Revenue by Department", "showTitle": true} + } + }, + "position": {"x": 0, "y": 12, "width": 4, "height": 5} + }, + { + "widget": { + "name": "chart_by_region", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "ds_daily_sales", + "fields": [ + {"name": "sale_date", "expression": "`sale_date`"}, + {"name": "region", "expression": "`region`"}, + {"name": "sum(total_revenue)", "expression": "SUM(`total_revenue`)"} + ], + "disaggregated": false + } + }], + "spec": { + "version": 3, + "widgetType": "bar", + "encodings": { + "x": { + "fieldName": "sale_date", + "scale": {"type": "temporal"}, + "axis": {"title": "Date"}, + "displayName": "Date" + }, + "y": { + "fieldName": "sum(total_revenue)", + "scale": {"type": "quantitative"}, + "format": { + "type": "number-currency", + "currencyCode": "USD", + "abbreviation": "compact" + }, + "axis": {"title": "Revenue ($)"}, + "displayName": "Revenue ($)" + }, + "color": { + "fieldName": "region", + "scale": {"type": "categorical"}, + "displayName": "Region" + } + }, + "frame": {"title": "Revenue by Region", "showTitle": true} + } + }, + "position": {"x": 4, "y": 12, "width": 8, "height": 5} + }, + { + "widget": { + "name": "section_products", + "multilineTextboxSpec": { + "lines": ["## Top Products"] + } + }, + "position": {"x": 0, "y": 17, "width": 12, "height": 1} + }, + { + "widget": { + "name": "table_products", + "queries": [{ + "name": "main_query", + "query": { + "datasetName": "ds_products", + "fields": [ + {"name": "product_name", "expression": "`product_name`"}, + {"name": "department", "expression": "`department`"}, + {"name": "units_sold", "expression": "`units_sold`"}, + {"name": "revenue", "expression": "`revenue`"}, + {"name": "profit", "expression": "`profit`"} + ], + "disaggregated": true + } + }], + "spec": { + "version": 2, + "widgetType": "table", + "encodings": { + "columns": [ + {"fieldName": "product_name", "displayName": "Product"}, + {"fieldName": "department", "displayName": "Department"}, + {"fieldName": "units_sold", "displayName": "Units Sold"}, + {"fieldName": "revenue", "displayName": "Revenue ($)"}, + {"fieldName": "profit", "displayName": "Profit ($)"} + ] + }, + "frame": { + "title": "Product Performance", + "showTitle": true, + "description": "Top products by revenue" + } + } + }, + "position": {"x": 0, "y": 18, "width": 12, "height": 6} + } + ] + }, + { + "name": "global_filters", + "displayName": "Filters", + "pageType": "PAGE_TYPE_GLOBAL_FILTERS", + "layoutVersion": "GRID_V1", + "layout": [ + { + "widget": { + "name": "filter_date_range", + "queries": [ + { + "name": "ds_sales_date", + "query": { + "datasetName": "ds_daily_sales", + "fields": [{"name": "sale_date", "expression": "`sale_date`"}], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-date-range-picker", + "encodings": { + "fields": [ + {"fieldName": "sale_date", "displayName": "Date", "queryName": "ds_sales_date"} + ] + }, + "selection": { + "defaultSelection": { + "range": { + "dataType": "DATE", + "min": {"value": "now/y"}, + "max": {"value": "now/y"} + } + } + }, + "frame": {"showTitle": true, "title": "Date Range"} + } + }, + "position": {"x": 0, "y": 0, "width": 4, "height": 2} + }, + { + "widget": { + "name": "filter_region", + "queries": [ + { + "name": "ds_sales_region", + "query": { + "datasetName": "ds_daily_sales", + "fields": [{"name": "region", "expression": "`region`"}], + "disaggregated": false + } + }, + { + "name": "ds_products_region", + "query": { + "datasetName": "ds_products", + "fields": [{"name": "region", "expression": "`region`"}], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [ + {"fieldName": "region", "displayName": "Region", "queryName": "ds_sales_region"}, + {"fieldName": "region", "displayName": "Region", "queryName": "ds_products_region"} + ] + }, + "frame": {"showTitle": true, "title": "Region"} + } + }, + "position": {"x": 4, "y": 0, "width": 4, "height": 2} + }, + { + "widget": { + "name": "filter_department", + "queries": [ + { + "name": "ds_sales_dept", + "query": { + "datasetName": "ds_daily_sales", + "fields": [{"name": "department", "expression": "`department`"}], + "disaggregated": false + } + }, + { + "name": "ds_products_dept", + "query": { + "datasetName": "ds_products", + "fields": [{"name": "department", "expression": "`department`"}], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-multi-select", + "encodings": { + "fields": [ + {"fieldName": "department", "displayName": "Department", "queryName": "ds_sales_dept"}, + {"fieldName": "department", "displayName": "Department", "queryName": "ds_products_dept"} + ] + }, + "frame": {"showTitle": true, "title": "Department"} + } + }, + "position": {"x": 8, "y": 0, "width": 4, "height": 2} + } + ] + } + ] +} +``` diff --git a/.claude/skills/databricks-aibi-dashboards/5-troubleshooting.md b/.claude/skills/databricks-aibi-dashboards/5-troubleshooting.md new file mode 100644 index 0000000..f6477c0 --- /dev/null +++ b/.claude/skills/databricks-aibi-dashboards/5-troubleshooting.md @@ -0,0 +1,100 @@ +# Troubleshooting + +Common errors and fixes for AI/BI dashboards. + +## Structural Errors (JSON Parse Failures) + +These errors occur when the JSON structure is wrong: + +| Error | Cause | Fix | +|-------|-------|-----| +| "failed to parse serialized dashboard" | Wrong JSON structure | Check: `queryLines` is array (not `"query": "string"`), widgets inline in `layout[].widget`, `pageType` on every page | +| "no selected fields to visualize" | `fields[].name` ≠ `encodings.fieldName` | Names must match exactly (e.g., both `"sum(spend)"`) | +| Widgets in wrong location | Used separate `"widgets"` array | Widgets must be INLINE: `layout[]: {widget: {...}, position: {...}}` | +| Missing page content | Omitted `pageType` | Add `"pageType": "PAGE_TYPE_CANVAS"` or `"PAGE_TYPE_GLOBAL_FILTERS"` | + +--- + +## Widget shows "no selected fields to visualize" + +**This is a field name mismatch error.** The `name` in `query.fields` must exactly match the `fieldName` in `encodings`. + +**Fix:** Ensure names match exactly: +```json +// WRONG - names don't match +"fields": [{"name": "spend", "expression": "SUM(`spend`)"}] +"encodings": {"value": {"fieldName": "sum(spend)", ...}} // ERROR! + +// CORRECT - names match +"fields": [{"name": "sum(spend)", "expression": "SUM(`spend`)"}] +"encodings": {"value": {"fieldName": "sum(spend)", ...}} // OK! +``` + +## Widget shows "Invalid widget definition" + +**Check version numbers:** +- Counters: `version: 2` (NOT 3!) +- Tables: `version: 2` (NOT 1 or 3!) +- Filters: `version: 2` +- Bar/Line/Pie/Area/Scatter charts: `version: 3` +- Combo/Choropleth-map: `version: 1` + +**Text widget errors:** +- Text widgets must NOT have a `spec` block +- Use `multilineTextboxSpec` directly on the widget object +- Do NOT use `widgetType: "text"` - this is invalid + +**Table widget errors:** +- Use `version: 2` (NOT 1 or 3) +- Column objects only need `fieldName` and `displayName` +- Do NOT add `type`, `numberFormat`, or other column properties + +**Counter widget errors:** +- Use `version: 2` (NOT 3) +- Ensure dataset returns exactly 1 row for `disaggregated: true` + +## Dashboard shows empty widgets + +- Run the dataset SQL query directly to check data exists +- Verify column aliases match widget field expressions +- Check `disaggregated` flag: + - `true` for pre-aggregated data (1 row) + - `false` when widget performs aggregation (multi-row) + +## Layout has gaps + +- Ensure each row sums to width=12 +- Check that y positions don't skip values + +## Filter shows "Invalid widget definition" + +- Check `widgetType` is one of: `filter-multi-select`, `filter-single-select`, `filter-date-range-picker` +- **DO NOT** use `widgetType: "filter"` - this is invalid +- Verify `spec.version` is `2` +- Ensure `queryName` in encodings matches the query `name` +- Confirm `disaggregated: false` in filter queries +- Ensure `frame` with `showTitle: true` is included + +## Filter not affecting expected pages + +- **Global filters** (on `PAGE_TYPE_GLOBAL_FILTERS` page) affect all datasets containing the filter field +- **Page-level filters** (on `PAGE_TYPE_CANVAS` page) only affect widgets on that same page +- A filter only works on datasets that include the filter dimension column + +## Filter shows "UNRESOLVED_COLUMN" error for `associative_filter_predicate_group` + +- **DO NOT** use `COUNT_IF(\`associative_filter_predicate_group\`)` in filter queries +- This internal expression causes SQL errors when the dashboard executes queries +- Use a simple field expression instead: `{"name": "field", "expression": "\`field\`"}` + +## Text widget shows title and description on same line + +- Multiple items in the `lines` array are **concatenated**, not displayed on separate lines +- Use **separate text widgets** for title and subtitle at different y positions +- Example: title at y=0 with height=1, subtitle at y=1 with height=1 + +## Chart unreadable (too many categories) + +- Use TOP-N + "Other" bucketing in dataset SQL +- Aggregate to a higher level (region instead of store) +- Use a table widget instead of a chart for high-cardinality data diff --git a/.claude/skills/databricks-aibi-dashboards/SKILL.md b/.claude/skills/databricks-aibi-dashboards/SKILL.md index 41dbeec..426e602 100644 --- a/.claude/skills/databricks-aibi-dashboards/SKILL.md +++ b/.claude/skills/databricks-aibi-dashboards/SKILL.md @@ -1,6 +1,6 @@ --- name: databricks-aibi-dashboards -description: "Create Databricks AI/BI dashboards. CRITICAL: You MUST test ALL SQL queries via execute_sql BEFORE deploying. Follow guidelines strictly." +description: "Create Databricks AI/BI dashboards. Use when creating, updating, or deploying Lakeview dashboards. CRITICAL: You MUST test ALL SQL queries via execute_sql BEFORE deploying. Follow guidelines strictly." --- # AI/BI Dashboard Skill @@ -13,7 +13,7 @@ Create Databricks AI/BI dashboards (formerly Lakeview dashboards). **Follow thes ``` ┌─────────────────────────────────────────────────────────────────────┐ -│ STEP 1: Get table schemas via get_table_details(catalog, schema) │ +│ STEP 1: Get table schemas via get_table_stats_and_schema(catalog, schema) │ ├─────────────────────────────────────────────────────────────────────┤ │ STEP 2: Write SQL queries for each dataset │ ├─────────────────────────────────────────────────────────────────────┤ @@ -24,7 +24,7 @@ Create Databricks AI/BI dashboards (formerly Lakeview dashboards). **Follow thes ├─────────────────────────────────────────────────────────────────────┤ │ STEP 4: Build dashboard JSON using ONLY verified queries │ ├─────────────────────────────────────────────────────────────────────┤ -│ STEP 5: Deploy via create_or_update_dashboard() │ +│ STEP 5: Deploy via manage_dashboard(action="create_or_update") │ └─────────────────────────────────────────────────────────────────────┘ ``` @@ -34,21 +34,55 @@ Create Databricks AI/BI dashboards (formerly Lakeview dashboards). **Follow thes | Tool | Description | |------|-------------| -| `get_table_details` | **STEP 1**: Get table schemas for designing queries | +| `get_table_stats_and_schema` | **STEP 1**: Get table schemas for designing queries | | `execute_sql` | **STEP 3**: Test SQL queries - MANDATORY before deployment! | -| `get_best_warehouse` | Get available warehouse ID | -| `create_or_update_dashboard` | **STEP 5**: Deploy dashboard JSON (only after validation!) | -| `get_dashboard` | Get dashboard details by ID | -| `list_dashboards` | List dashboards in workspace | -| `trash_dashboard` | Move dashboard to trash | -| `publish_dashboard` | Publish dashboard for viewers | -| `unpublish_dashboard` | Unpublish a dashboard | +| `manage_warehouse` (action="get_best") | Get available warehouse ID | +| `manage_dashboard` | **STEP 5**: Dashboard lifecycle management (see actions below) | + +### manage_dashboard Actions + +| Action | Description | Required Params | +|--------|-------------|-----------------| +| `create_or_update` | Deploy dashboard JSON (only after validation!) | display_name, parent_path, serialized_dashboard, warehouse_id | +| `get` | Get dashboard details by ID | dashboard_id | +| `list` | List all dashboards | (none) | +| `delete` | Move dashboard to trash | dashboard_id | +| `publish` | Publish a dashboard | dashboard_id, warehouse_id | +| `unpublish` | Unpublish a dashboard | dashboard_id | + +**Example usage:** +```python +# Create/update dashboard +manage_dashboard( + action="create_or_update", + display_name="Sales Dashboard", + parent_path="/Workspace/Users/me/dashboards", + serialized_dashboard=dashboard_json, + warehouse_id="abc123", + publish=True # auto-publish after create +) + +# Get dashboard details +manage_dashboard(action="get", dashboard_id="dashboard_123") + +# List all dashboards +manage_dashboard(action="list") +``` + +## Reference Files + +| What are you building? | Reference | +|------------------------|-----------| +| Any widget (text, counter, table, chart) | [1-widget-specifications.md](1-widget-specifications.md) | +| Dashboard with filters (global or page-level) | [2-filters.md](2-filters.md) | +| Need a complete working template to adapt | [3-examples.md](3-examples.md) | +| Debugging a broken dashboard | [4-troubleshooting.md](4-troubleshooting.md) | --- ## Implementation Guidelines -### 1) DATASET ARCHITECTURE (STRICT) +### 1) DATASET ARCHITECTURE - **One dataset per domain** (e.g., orders, customers, products) - **Exactly ONE valid SQL query per dataset** (no multiple queries separated by `;`) @@ -113,32 +147,44 @@ If you need conditional logic or multi-field formulas, compute a derived column - Date truncation: `DATE_TRUNC('DAY'|'WEEK'|'MONTH'|'QUARTER'|'YEAR', column)` - **AVOID** `INTERVAL` syntax - use functions instead -### 4) LAYOUT (6-Column Grid, NO GAPS) +### 4) LAYOUT (12-Column Grid, NO GAPS) + +**Every page must include `"layoutVersion": "GRID_V1"`** alongside `pageType`. + +```json +{ + "name": "overview", + "displayName": "Overview", + "pageType": "PAGE_TYPE_CANVAS", + "layoutVersion": "GRID_V1", + "layout": [...] +} +``` -Each widget has a position: `{"x": 0, "y": 0, "width": 2, "height": 4}` +Each widget has a position: `{"x": 0, "y": 0, "width": 4, "height": 4}` -**CRITICAL**: Each row must fill width=6 exactly. No gaps allowed. +**CRITICAL**: Each row must fill width=12 exactly. No gaps allowed. **Recommended widget sizes:** | Widget Type | Width | Height | Notes | |-------------|-------|--------|-------| -| Text header | 6 | 1 | Full width; use SEPARATE widgets for title and subtitle | -| Counter/KPI | 2 | **3-4** | **NEVER height=2** - too cramped! | -| Line/Bar chart | 3 | **5-6** | Pair side-by-side to fill row | -| Pie chart | 3 | **5-6** | Needs space for legend | -| Full-width chart | 6 | 5-7 | For detailed time series | -| Table | 6 | 5-8 | Full width for readability | +| Text header | 12 | 1 | Full width; use SEPARATE widgets for title and subtitle | +| Counter/KPI | 4 | **3-4** | **NEVER height=2** - too cramped! | +| Line/Bar chart | 6 | **5-6** | Pair side-by-side to fill row | +| Pie chart | 6 | **5-6** | Needs space for legend | +| Full-width chart | 12 | 5-7 | For detailed time series | +| Table | 12 | 5-8 | Full width for readability | **Standard dashboard structure:** ```text -y=0: Title (w=6, h=1) - Dashboard title (use separate widget!) -y=1: Subtitle (w=6, h=1) - Description (use separate widget!) -y=2: KPIs (w=2 each, h=3) - 3 key metrics side-by-side -y=5: Section header (w=6, h=1) - "Trends" or similar -y=6: Charts (w=3 each, h=5) - Two charts side-by-side -y=11: Section header (w=6, h=1) - "Details" -y=12: Table (w=6, h=6) - Detailed data +y=0: Title (w=12, h=1) - Dashboard title (use separate widget!) +y=1: Subtitle (w=12, h=1) - Description (use separate widget!) +y=2: KPIs (w=4 each, h=3) - 3 key metrics side-by-side +y=5: Section header (w=12, h=1) - "Trends" or similar +y=6: Charts (w=6 each, h=5) - Two charts side-by-side +y=11: Section header (w=12, h=1) - "Details" +y=12: Table (w=12, h=6) - Detailed data ``` ### 5) CARDINALITY & READABILITY (CRITICAL) @@ -152,770 +198,27 @@ y=12: Table (w=6, h=6) - Detailed data | High cardinality | **Table only** | customer_id, order_id, SKU | **Before creating any chart with color/grouping:** -1. Check column cardinality (use `get_table_details` to see distinct values) +1. Check column cardinality (use `get_table_stats_and_schema` to see distinct values) 2. If >10 distinct values, aggregate to higher level OR use TOP-N + "Other" bucket 3. For high-cardinality dimensions, use a table widget instead of a chart -### 6) WIDGET SPECIFICATIONS - -**Widget Naming Convention (CRITICAL):** -- `widget.name`: alphanumeric + hyphens + underscores ONLY (no spaces, parentheses, colons) -- `frame.title`: human-readable name (any characters allowed) -- `widget.queries[0].name`: always use `"main_query"` - -**CRITICAL VERSION REQUIREMENTS:** - -| Widget Type | Version | -|-------------|---------| -| counter | 2 | -| table | 2 | -| filter-multi-select | 2 | -| filter-single-select | 2 | -| filter-date-range-picker | 2 | -| bar | 3 | -| line | 3 | -| pie | 3 | -| text | N/A (no spec block) | - ---- - -**Text (Headers/Descriptions):** -- **CRITICAL: Text widgets do NOT use a spec block!** -- Use `multilineTextboxSpec` directly on the widget -- Supports markdown: `#`, `##`, `###`, `**bold**`, `*italic*` -- **CRITICAL: Multiple items in the `lines` array are concatenated on a single line, NOT displayed as separate lines!** -- For title + subtitle, use **separate text widgets** at different y positions - -```json -// CORRECT: Separate widgets for title and subtitle -{ - "widget": { - "name": "title", - "multilineTextboxSpec": { - "lines": ["## Dashboard Title"] - } - }, - "position": {"x": 0, "y": 0, "width": 6, "height": 1} -}, -{ - "widget": { - "name": "subtitle", - "multilineTextboxSpec": { - "lines": ["Description text here"] - } - }, - "position": {"x": 0, "y": 1, "width": 6, "height": 1} -} - -// WRONG: Multiple lines concatenate into one line! -{ - "widget": { - "name": "title-widget", - "multilineTextboxSpec": { - "lines": ["## Dashboard Title", "Description text here"] // Becomes "## Dashboard TitleDescription text here" - } - }, - "position": {"x": 0, "y": 0, "width": 6, "height": 2} -} -``` - ---- - -**Counter (KPI):** -- `version`: **2** (NOT 3!) -- `widgetType`: "counter" -- **Percent values must be 0-1** in the data (not 0-100) - -**Two patterns for counters:** - -**Pattern 1: Pre-aggregated dataset (1 row, no filters)** -- Dataset returns exactly 1 row -- Use `"disaggregated": true` and simple field reference -- Field `name` matches dataset column directly - -```json -{ - "widget": { - "name": "total-revenue", - "queries": [{ - "name": "main_query", - "query": { - "datasetName": "summary_ds", - "fields": [{"name": "revenue", "expression": "`revenue`"}], - "disaggregated": true - } - }], - "spec": { - "version": 2, - "widgetType": "counter", - "encodings": { - "value": {"fieldName": "revenue", "displayName": "Total Revenue"} - }, - "frame": {"showTitle": true, "title": "Total Revenue"} - } - }, - "position": {"x": 0, "y": 0, "width": 2, "height": 3} -} -``` - -**Pattern 2: Aggregating widget (multi-row dataset, supports filters)** -- Dataset returns multiple rows (e.g., grouped by a filter dimension) -- Use `"disaggregated": false` and aggregation expression -- **CRITICAL**: Field `name` MUST match `fieldName` exactly (e.g., `"sum(spend)"`) - -```json -{ - "widget": { - "name": "total-spend", - "queries": [{ - "name": "main_query", - "query": { - "datasetName": "by_category", - "fields": [{"name": "sum(spend)", "expression": "SUM(`spend`)"}], - "disaggregated": false - } - }], - "spec": { - "version": 2, - "widgetType": "counter", - "encodings": { - "value": {"fieldName": "sum(spend)", "displayName": "Total Spend"} - }, - "frame": {"showTitle": true, "title": "Total Spend"} - } - }, - "position": {"x": 0, "y": 0, "width": 2, "height": 3} -} -``` - ---- - -**Table:** -- `version`: **2** (NOT 1 or 3!) -- `widgetType`: "table" -- **Columns only need `fieldName` and `displayName`** - no other properties! -- Use `"disaggregated": true` for raw rows - -```json -{ - "widget": { - "name": "details-table", - "queries": [{ - "name": "main_query", - "query": { - "datasetName": "details_ds", - "fields": [ - {"name": "name", "expression": "`name`"}, - {"name": "value", "expression": "`value`"} - ], - "disaggregated": true - } - }], - "spec": { - "version": 2, - "widgetType": "table", - "encodings": { - "columns": [ - {"fieldName": "name", "displayName": "Name"}, - {"fieldName": "value", "displayName": "Value"} - ] - }, - "frame": {"showTitle": true, "title": "Details"} - } - }, - "position": {"x": 0, "y": 0, "width": 6, "height": 6} -} -``` - ---- - -**Line / Bar Charts:** -- `version`: **3** -- `widgetType`: "line" or "bar" -- Use `x`, `y`, optional `color` encodings -- `scale.type`: `"temporal"` (dates), `"quantitative"` (numbers), `"categorical"` (strings) -- Use `"disaggregated": true` with pre-aggregated dataset data - -**Multiple Lines - Two Approaches:** - -1. **Multi-Y Fields** (different metrics on same chart): -```json -"y": { - "scale": {"type": "quantitative"}, - "fields": [ - {"fieldName": "sum(orders)", "displayName": "Orders"}, - {"fieldName": "sum(returns)", "displayName": "Returns"} - ] -} -``` - -2. **Color Grouping** (same metric split by dimension): -```json -"y": {"fieldName": "sum(revenue)", "scale": {"type": "quantitative"}}, -"color": {"fieldName": "region", "scale": {"type": "categorical"}, "displayName": "Region"} -``` - -**Bar Chart Modes:** -- **Stacked** (default): No `mark` field - bars stack on top of each other -- **Grouped**: Add `"mark": {"layout": "group"}` - bars side-by-side for comparison - -**Pie Chart:** -- `version`: **3** -- `widgetType`: "pie" -- `angle`: quantitative aggregate -- `color`: categorical dimension -- Limit to 3-8 categories for readability - -### 7) FILTERS (Global vs Page-Level) - -> **CRITICAL**: Filter widgets use DIFFERENT widget types than charts! -> - Valid types: `filter-multi-select`, `filter-single-select`, `filter-date-range-picker` -> - **DO NOT** use `widgetType: "filter"` - this does not exist and will cause errors -> - Filters use `spec.version: 2` -> - **ALWAYS include `frame` with `showTitle: true`** for filter widgets - -**Filter widget types:** -- `filter-date-range-picker`: for DATE/TIMESTAMP fields -- `filter-single-select`: categorical with single selection -- `filter-multi-select`: categorical with multiple selections - ---- - -#### Global Filters vs Page-Level Filters - -| Type | Placement | Scope | Use Case | -|------|-----------|-------|----------| -| **Global Filter** | Dedicated page with `"pageType": "PAGE_TYPE_GLOBAL_FILTERS"` | Affects ALL pages that have datasets with the filter field | Cross-dashboard filtering (e.g., date range, campaign) | -| **Page-Level Filter** | Regular page with `"pageType": "PAGE_TYPE_CANVAS"` | Affects ONLY widgets on that same page | Page-specific filtering (e.g., platform filter on breakdown page only) | - -**Key Insight**: A filter only affects datasets that contain the filter field. To have a filter affect only specific pages: -1. Include the filter dimension in datasets for pages that should be filtered -2. Exclude the filter dimension from datasets for pages that should NOT be filtered - ---- - -#### Filter Widget Structure - -> **CRITICAL**: Do NOT use `associative_filter_predicate_group` - it causes SQL errors! -> Use a simple field expression instead. - -```json -{ - "widget": { - "name": "filter_region", - "queries": [{ - "name": "ds_data_region", - "query": { - "datasetName": "ds_data", - "fields": [ - {"name": "region", "expression": "`region`"} - ], - "disaggregated": false - } - }], - "spec": { - "version": 2, - "widgetType": "filter-multi-select", - "encodings": { - "fields": [{ - "fieldName": "region", - "displayName": "Region", - "queryName": "ds_data_region" - }] - }, - "frame": {"showTitle": true, "title": "Region"} - } - }, - "position": {"x": 0, "y": 0, "width": 2, "height": 2} -} -``` - ---- - -#### Global Filter Example - -Place on a dedicated filter page: - -```json -{ - "name": "filters", - "displayName": "Filters", - "pageType": "PAGE_TYPE_GLOBAL_FILTERS", - "layout": [ - { - "widget": { - "name": "filter_campaign", - "queries": [{ - "name": "ds_campaign", - "query": { - "datasetName": "overview", - "fields": [{"name": "campaign_name", "expression": "`campaign_name`"}], - "disaggregated": false - } - }], - "spec": { - "version": 2, - "widgetType": "filter-multi-select", - "encodings": { - "fields": [{ - "fieldName": "campaign_name", - "displayName": "Campaign", - "queryName": "ds_campaign" - }] - }, - "frame": {"showTitle": true, "title": "Campaign"} - } - }, - "position": {"x": 0, "y": 0, "width": 2, "height": 2} - } - ] -} -``` - ---- - -#### Page-Level Filter Example - -Place directly on a canvas page (affects only that page): - -```json -{ - "name": "platform_breakdown", - "displayName": "Platform Breakdown", - "pageType": "PAGE_TYPE_CANVAS", - "layout": [ - { - "widget": { - "name": "page-title", - "multilineTextboxSpec": {"lines": ["## Platform Breakdown"]} - }, - "position": {"x": 0, "y": 0, "width": 4, "height": 1} - }, - { - "widget": { - "name": "filter_platform", - "queries": [{ - "name": "ds_platform", - "query": { - "datasetName": "platform_data", - "fields": [{"name": "platform", "expression": "`platform`"}], - "disaggregated": false - } - }], - "spec": { - "version": 2, - "widgetType": "filter-multi-select", - "encodings": { - "fields": [{ - "fieldName": "platform", - "displayName": "Platform", - "queryName": "ds_platform" - }] - }, - "frame": {"showTitle": true, "title": "Platform"} - } - }, - "position": {"x": 4, "y": 0, "width": 2, "height": 2} - } - // ... other widgets on this page - ] -} -``` - ---- - -**Filter Layout Guidelines:** -- Global filters: Position on dedicated filter page, stack vertically at `x=0` -- Page-level filters: Position in header area of page (e.g., top-right corner) -- Typical sizing: `width: 2, height: 2` - -### 8) QUALITY CHECKLIST +### 6) QUALITY CHECKLIST Before deploying, verify: 1. All widget names use only alphanumeric + hyphens + underscores -2. All rows sum to width=6 with no gaps -3. KPIs use height 3-4, charts use height 5-6 -4. Chart dimensions have ≤8 distinct values -5. All widget fieldNames match dataset columns exactly -6. **Field `name` in query.fields matches `fieldName` in encodings exactly** (e.g., both `"sum(spend)"`) -7. Counter datasets: use `disaggregated: true` for 1-row datasets, `disaggregated: false` with aggregation for multi-row -8. Percent values are 0-1 (not 0-100) -9. SQL uses Spark syntax (date_sub, not INTERVAL) -10. **All SQL queries tested via `execute_sql` and return expected data** +2. **Every page has `"layoutVersion": "GRID_V1"`** +3. All rows sum to width=12 with no gaps +4. KPIs use height 3-4, charts use height 5-6 +5. Chart dimensions have ≤8 distinct values +6. All widget fieldNames match dataset columns exactly +7. **Field `name` in query.fields matches `fieldName` in encodings exactly** (e.g., both `"sum(spend)"`) +8. Counter datasets: use `disaggregated: true` for 1-row datasets, `disaggregated: false` with aggregation for multi-row +9. Percent values are 0-1 (not 0-100) +10. SQL uses Spark syntax (date_sub, not INTERVAL) +11. **All SQL queries tested via `execute_sql` and return expected data** --- -## Complete Example - -```python -import json - -# Step 1: Check table schema -table_info = get_table_details(catalog="samples", schema="nyctaxi") - -# Step 2: Test queries -execute_sql("SELECT COUNT(*) as trips, AVG(fare_amount) as avg_fare, AVG(trip_distance) as avg_distance FROM samples.nyctaxi.trips") -execute_sql(""" - SELECT pickup_zip, COUNT(*) as trip_count - FROM samples.nyctaxi.trips - GROUP BY pickup_zip - ORDER BY trip_count DESC - LIMIT 10 -""") - -# Step 3: Build dashboard JSON -dashboard = { - "datasets": [ - { - "name": "summary", - "displayName": "Summary Stats", - "queryLines": [ - "SELECT COUNT(*) as trips, AVG(fare_amount) as avg_fare, ", - "AVG(trip_distance) as avg_distance ", - "FROM samples.nyctaxi.trips " - ] - }, - { - "name": "by_zip", - "displayName": "Trips by ZIP", - "queryLines": [ - "SELECT pickup_zip, COUNT(*) as trip_count ", - "FROM samples.nyctaxi.trips ", - "GROUP BY pickup_zip ", - "ORDER BY trip_count DESC ", - "LIMIT 10 " - ] - } - ], - "pages": [{ - "name": "overview", - "displayName": "NYC Taxi Overview", - "pageType": "PAGE_TYPE_CANVAS", - "layout": [ - # Text header - NO spec block! Use SEPARATE widgets for title and subtitle! - { - "widget": { - "name": "title", - "multilineTextboxSpec": { - "lines": ["## NYC Taxi Dashboard"] - } - }, - "position": {"x": 0, "y": 0, "width": 6, "height": 1} - }, - { - "widget": { - "name": "subtitle", - "multilineTextboxSpec": { - "lines": ["Trip statistics and analysis"] - } - }, - "position": {"x": 0, "y": 1, "width": 6, "height": 1} - }, - # Counter - version 2, width 2! - { - "widget": { - "name": "total-trips", - "queries": [{ - "name": "main_query", - "query": { - "datasetName": "summary", - "fields": [{"name": "trips", "expression": "`trips`"}], - "disaggregated": True - } - }], - "spec": { - "version": 2, - "widgetType": "counter", - "encodings": { - "value": {"fieldName": "trips", "displayName": "Total Trips"} - }, - "frame": {"title": "Total Trips", "showTitle": True} - } - }, - "position": {"x": 0, "y": 2, "width": 2, "height": 3} - }, - { - "widget": { - "name": "avg-fare", - "queries": [{ - "name": "main_query", - "query": { - "datasetName": "summary", - "fields": [{"name": "avg_fare", "expression": "`avg_fare`"}], - "disaggregated": True - } - }], - "spec": { - "version": 2, - "widgetType": "counter", - "encodings": { - "value": {"fieldName": "avg_fare", "displayName": "Avg Fare"} - }, - "frame": {"title": "Average Fare", "showTitle": True} - } - }, - "position": {"x": 2, "y": 2, "width": 2, "height": 3} - }, - { - "widget": { - "name": "total-distance", - "queries": [{ - "name": "main_query", - "query": { - "datasetName": "summary", - "fields": [{"name": "avg_distance", "expression": "`avg_distance`"}], - "disaggregated": True - } - }], - "spec": { - "version": 2, - "widgetType": "counter", - "encodings": { - "value": {"fieldName": "avg_distance", "displayName": "Avg Distance"} - }, - "frame": {"title": "Average Distance", "showTitle": True} - } - }, - "position": {"x": 4, "y": 2, "width": 2, "height": 3} - }, - # Bar chart - version 3 - { - "widget": { - "name": "trips-by-zip", - "queries": [{ - "name": "main_query", - "query": { - "datasetName": "by_zip", - "fields": [ - {"name": "pickup_zip", "expression": "`pickup_zip`"}, - {"name": "trip_count", "expression": "`trip_count`"} - ], - "disaggregated": True - } - }], - "spec": { - "version": 3, - "widgetType": "bar", - "encodings": { - "x": {"fieldName": "pickup_zip", "scale": {"type": "categorical"}, "displayName": "ZIP"}, - "y": {"fieldName": "trip_count", "scale": {"type": "quantitative"}, "displayName": "Trips"} - }, - "frame": {"title": "Trips by Pickup ZIP", "showTitle": True} - } - }, - "position": {"x": 0, "y": 5, "width": 6, "height": 5} - }, - # Table - version 2, minimal column props! - { - "widget": { - "name": "zip-table", - "queries": [{ - "name": "main_query", - "query": { - "datasetName": "by_zip", - "fields": [ - {"name": "pickup_zip", "expression": "`pickup_zip`"}, - {"name": "trip_count", "expression": "`trip_count`"} - ], - "disaggregated": True - } - }], - "spec": { - "version": 2, - "widgetType": "table", - "encodings": { - "columns": [ - {"fieldName": "pickup_zip", "displayName": "ZIP Code"}, - {"fieldName": "trip_count", "displayName": "Trip Count"} - ] - }, - "frame": {"title": "Top ZIP Codes", "showTitle": True} - } - }, - "position": {"x": 0, "y": 10, "width": 6, "height": 5} - } - ] - }] -} - -# Step 4: Deploy -result = create_or_update_dashboard( - display_name="NYC Taxi Dashboard", - parent_path="/Workspace/Users/me/dashboards", - serialized_dashboard=json.dumps(dashboard), - warehouse_id=get_best_warehouse(), -) -print(result["url"]) -``` - -## Complete Example with Filters - -```python -import json - -# Dashboard with a global filter for region -dashboard_with_filters = { - "datasets": [ - { - "name": "sales", - "displayName": "Sales Data", - "queryLines": [ - "SELECT region, SUM(revenue) as total_revenue ", - "FROM catalog.schema.sales ", - "GROUP BY region" - ] - } - ], - "pages": [ - { - "name": "overview", - "displayName": "Sales Overview", - "pageType": "PAGE_TYPE_CANVAS", - "layout": [ - { - "widget": { - "name": "total-revenue", - "queries": [{ - "name": "main_query", - "query": { - "datasetName": "sales", - "fields": [{"name": "total_revenue", "expression": "`total_revenue`"}], - "disaggregated": True - } - }], - "spec": { - "version": 2, # Version 2 for counters! - "widgetType": "counter", - "encodings": { - "value": {"fieldName": "total_revenue", "displayName": "Total Revenue"} - }, - "frame": {"title": "Total Revenue", "showTitle": True} - } - }, - "position": {"x": 0, "y": 0, "width": 6, "height": 3} - } - ] - }, - { - "name": "filters", - "displayName": "Filters", - "pageType": "PAGE_TYPE_GLOBAL_FILTERS", # Required for global filter page! - "layout": [ - { - "widget": { - "name": "filter_region", - "queries": [{ - "name": "ds_sales_region", - "query": { - "datasetName": "sales", - "fields": [ - {"name": "region", "expression": "`region`"} - # DO NOT use associative_filter_predicate_group - causes SQL errors! - ], - "disaggregated": False # False for filters! - } - }], - "spec": { - "version": 2, # Version 2 for filters! - "widgetType": "filter-multi-select", # NOT "filter"! - "encodings": { - "fields": [{ - "fieldName": "region", - "displayName": "Region", - "queryName": "ds_sales_region" # Must match query name! - }] - }, - "frame": {"showTitle": True, "title": "Region"} # Always show title! - } - }, - "position": {"x": 0, "y": 0, "width": 2, "height": 2} - } - ] - } - ] -} - -# Deploy with filters -result = create_or_update_dashboard( - display_name="Sales Dashboard with Filters", - parent_path="/Workspace/Users/me/dashboards", - serialized_dashboard=json.dumps(dashboard_with_filters), - warehouse_id=get_best_warehouse(), -) -print(result["url"]) -``` - -## Troubleshooting - -### Widget shows "no selected fields to visualize" - -**This is a field name mismatch error.** The `name` in `query.fields` must exactly match the `fieldName` in `encodings`. - -**Fix:** Ensure names match exactly: -```json -// WRONG - names don't match -"fields": [{"name": "spend", "expression": "SUM(`spend`)"}] -"encodings": {"value": {"fieldName": "sum(spend)", ...}} // ERROR! - -// CORRECT - names match -"fields": [{"name": "sum(spend)", "expression": "SUM(`spend`)"}] -"encodings": {"value": {"fieldName": "sum(spend)", ...}} // OK! -``` - -### Widget shows "Invalid widget definition" - -**Check version numbers:** -- Counters: `version: 2` -- Tables: `version: 2` -- Filters: `version: 2` -- Bar/Line/Pie charts: `version: 3` - -**Text widget errors:** -- Text widgets must NOT have a `spec` block -- Use `multilineTextboxSpec` directly on the widget object -- Do NOT use `widgetType: "text"` - this is invalid - -**Table widget errors:** -- Use `version: 2` (NOT 1 or 3) -- Column objects only need `fieldName` and `displayName` -- Do NOT add `type`, `numberFormat`, or other column properties - -**Counter widget errors:** -- Use `version: 2` (NOT 3) -- Ensure dataset returns exactly 1 row - -### Dashboard shows empty widgets -- Run the dataset SQL query directly to check data exists -- Verify column aliases match widget field expressions -- Check `disaggregated` flag (should be `true` for pre-aggregated data) - -### Layout has gaps -- Ensure each row sums to width=6 -- Check that y positions don't skip values - -### Filter shows "Invalid widget definition" -- Check `widgetType` is one of: `filter-multi-select`, `filter-single-select`, `filter-date-range-picker` -- **DO NOT** use `widgetType: "filter"` - this is invalid -- Verify `spec.version` is `2` -- Ensure `queryName` in encodings matches the query `name` -- Confirm `disaggregated: false` in filter queries -- Ensure `frame` with `showTitle: true` is included - -### Filter not affecting expected pages -- **Global filters** (on `PAGE_TYPE_GLOBAL_FILTERS` page) affect all datasets containing the filter field -- **Page-level filters** (on `PAGE_TYPE_CANVAS` page) only affect widgets on that same page -- A filter only works on datasets that include the filter dimension column - -### Filter shows "UNRESOLVED_COLUMN" error for `associative_filter_predicate_group` -- **DO NOT** use `COUNT_IF(\`associative_filter_predicate_group\`)` in filter queries -- This internal expression causes SQL errors when the dashboard executes queries -- Use a simple field expression instead: `{"name": "field", "expression": "\`field\`"}` - -### Text widget shows title and description on same line -- Multiple items in the `lines` array are **concatenated**, not displayed on separate lines -- Use **separate text widgets** for title and subtitle at different y positions -- Example: title at y=0 with height=1, subtitle at y=1 with height=1 - ## Related Skills - **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - for querying the underlying data and system tables diff --git a/.claude/skills/databricks-app-apx/SKILL.md b/.claude/skills/databricks-app-apx/SKILL.md deleted file mode 100644 index 2ee96ba..0000000 --- a/.claude/skills/databricks-app-apx/SKILL.md +++ /dev/null @@ -1,253 +0,0 @@ ---- -name: databricks-app-apx -description: "Build full-stack Databricks applications using APX framework (FastAPI + React)." ---- - -# Databricks APX Application - -Build full-stack Databricks applications using APX framework (FastAPI + React). - -## Trigger Conditions - -**Invoke when user requests**: -- "Databricks app" or "Databricks application" -- Full-stack app for Databricks without specifying framework -- Mentions APX framework - -**Do NOT invoke if user specifies**: Streamlit, Dash, Node.js, Shiny, Gradio, Flask, or other frameworks. - -## Prerequisites Check - -Option A) -Repository configured for use with APX. -1.. Verify APX MCP available: `mcp-cli tools | grep apx` -2. Verify shadcn MCP available: `mcp-cli tools | grep shadcn` -3. Confirm APX project (check `pyproject.toml`) - -Option B) -Install APX -1. Verify uv available or prompt for install. On Mac, suggest: `brew install uv`. -2. Verify bun available or prompt for install. On Mac, suggest: -``` -brew tap oven-sh/bun -brew install bun -``` -3. Verify git available or prompt for install. -4. Run APX setup commands: -``` -uvx --from git+https://github.com/databricks-solutions/apx.git apx init -``` - - -## Workflow Overview - -Total time: 55-70 minutes - -1. **Initialize** (5 min) - Start servers, create todos -2. **Backend** (15-20 min) - Models + routes with mock data -3. **Frontend** (20-25 min) - Components + pages -4. **Test** (5-10 min) - Type check + manual verification -5. **Document** (10 min) - README + code structure guide - -## Phase 1: Initialize - -```bash -# Start APX development server -mcp-cli call apx/start '{}' -mcp-cli call apx/status '{}' -``` - -Create TodoWrite with tasks: -- Start servers ✓ -- Design models -- Create API routes -- Add UI components -- Create pages -- Test & document - -## Phase 2: Backend Development - -### Create Pydantic Models - -In `src/{app_name}/backend/models.py`: - -**Follow 3-model pattern**: -- `EntityIn` - Input validation -- `EntityOut` - Complete output with computed fields -- `EntityListOut` - Performance-optimized summary - -**See [backend-patterns.md](backend-patterns.md) for complete code templates.** - -### Create API Routes - -In `src/{app_name}/backend/router.py`: - -**Critical requirements**: -- Always include `response_model` (enables OpenAPI generation) -- Always include `operation_id` (becomes frontend hook name) -- Use naming pattern: `listX`, `getX`, `createX`, `updateX`, `deleteX` -- Initialize 3-4 mock data samples for testing - -**See [backend-patterns.md](backend-patterns.md) for complete CRUD templates.** - -### Type Check - -```bash -mcp-cli call apx/dev_check '{}' -``` - -Fix any Python type errors reported by basedpyright. - -## Phase 3: Frontend Development - -**Wait 5-10 seconds** after backend changes for OpenAPI client regeneration. - -### Add UI Components - -```bash -# Get shadcn add command -mcp-cli call shadcn/get_add_command_for_items '{ - "items": ["@shadcn/button", "@shadcn/card", "@shadcn/table", - "@shadcn/badge", "@shadcn/select", "@shadcn/skeleton"] -}' -``` - -Run the command from project root with `--yes` flag. - -### Create Pages - -**List page**: `src/{app_name}/ui/routes/_sidebar/{entity}.tsx` -- Table view with all entities -- Suspense boundaries with skeleton fallback -- Formatted data (currency, dates, status colors) - -**Detail page**: `src/{app_name}/ui/routes/_sidebar/{entity}.$id.tsx` -- Complete entity view with cards -- Update/delete mutations -- Back navigation - -**See [frontend-patterns.md](frontend-patterns.md) for complete page templates.** - -### Update Navigation - -In `src/{app_name}/ui/routes/_sidebar/route.tsx`, add new item to `navItems` array. - -## Phase 4: Testing - -```bash -# Type check both backend and frontend -mcp-cli call apx/dev_check '{}' - -# Test API endpoints -curl http://localhost:8000/api/{entities} | jq . -curl http://localhost:8000/api/{entities}/{id} | jq . - -# Get frontend URL -mcp-cli call apx/get_frontend_url '{}' -``` - -Manually verify in browser: -- List page displays data -- Detail page shows complete info -- Mutations work (update, delete) -- Loading states work (skeletons) -- Browser console errors are automatically captured in APX dev logs - -## Phase 5: Deployment & Monitoring - -### Deploy to Databricks - -Use DABs to deploy your APX application to Databricks. See the `databricks-asset-bundles` skill for complete deployment guidance. - -### Monitor Application Logs - -**Automated log checking with APX MCP:** - -The APX MCP server can automatically check deployed application logs. Simply ask: -"Please check the deployed app logs for " - - -The APX MCP will retrieve logs and identify issues automatically, including: -- Deployment status and errors -- Runtime exceptions and stack traces -- Both `[SYSTEM]` (deployment) and `[APP]` (application) logs -- Browser console errors (now included in APX dev logs) - -**Manual log checking (reference):** - -For direct CLI access: -```bash -databricks apps logs --profile -``` - -**Key patterns to look for:** -- ✅ `Deployment successful` - App deployed correctly -- ✅ `App started successfully` - Application is running -- ❌ `Error:` - Check stack traces for issues - -## Phase 6: Documentation - -Create two markdown files: - -**README.md**: -- Features overview -- Technology stack -- How app was created (AI tools + MCP servers used) -- Application architecture -- Getting started instructions -- API documentation -- Development workflow - -**CODE_STRUCTURE.md**: -- Directory structure explanation -- Backend structure (models, routes, patterns) -- Frontend structure (routes, components, hooks) -- Auto-generated files warnings -- Guide for adding new features -- Best practices -- Common patterns -- Troubleshooting guide - -## Key Patterns - -### Backend -- **3-model pattern**: Separate In, Out, and ListOut models -- **operation_id naming**: `listEntities` → `useListEntities()` -- **Type hints everywhere**: Enable validation and IDE support - -### Frontend -- **Suspense hooks**: `useXSuspense(selector())` -- **Suspense boundaries**: Always provide skeleton fallback -- **Formatters**: Currency, dates, status colors -- **Never edit**: `lib/api.ts` or `types/routeTree.gen.ts` - -## Success Criteria - -- [ ] Type checking passes (`apx dev check` succeeds) -- [ ] API endpoints return correct data (curl verification) -- [ ] Frontend displays and mutates data correctly -- [ ] Loading states work (skeletons display) -- [ ] Documentation complete - -## Common Issues - -**Deployed app not working**: Ask to check deployed app logs (APX MCP will automatically retrieve and analyze them) or manually use `databricks apps logs ` -**Python type errors**: Use explicit casting for dict access, check Optional fields -**TypeScript errors**: Wait for OpenAPI regen, verify hook names match operation_ids -**OpenAPI not updating**: Check watcher status with `apx dev status`, restart if needed -**Components not added**: Run shadcn from project root with `--yes` flag - -## Reference Materials - -- **[backend-patterns.md](backend-patterns.md)** - Complete backend code templates -- **[frontend-patterns.md](frontend-patterns.md)** - Complete frontend page templates -- **[best-practices.md](best-practices.md)** - Best practices, anti-patterns, debugging - -Read these files only when actively writing that type of code or debugging issues. - -## Related Skills - -- **[databricks-app-python](../databricks-app-python/SKILL.md)** - for Streamlit, Dash, Gradio, or Flask apps -- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - deploying APX apps via DABs -- **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - backend SDK integration -- **[databricks-lakebase-provisioned](../databricks-lakebase-provisioned/SKILL.md)** - adding persistent PostgreSQL state to apps diff --git a/.claude/skills/databricks-app-apx/backend-patterns.md b/.claude/skills/databricks-app-apx/backend-patterns.md deleted file mode 100644 index 1b8d6d0..0000000 --- a/.claude/skills/databricks-app-apx/backend-patterns.md +++ /dev/null @@ -1,225 +0,0 @@ -# Backend Code Patterns for APX - -Reference templates for backend development. **Only consult when writing backend code.** - -## Pydantic Models (models.py) - -### 3-Model Pattern - -```python -from pydantic import BaseModel, Field -from datetime import datetime -from enum import Enum -from typing import Optional - -# Enum for status -class EntityStatus(str, Enum): - STATUS_1 = "status_1" - STATUS_2 = "status_2" - -# Nested models -class ItemIn(BaseModel): - name: str - value: float = Field(gt=0) - -class ItemOut(BaseModel): - id: str - name: str - value: float - created_at: datetime - -# Main entity models -class EntityIn(BaseModel): - """Input for creating entities""" - title: str - items: list[ItemIn] - notes: Optional[str] = None - -class EntityOut(BaseModel): - """Complete entity output""" - id: str - entity_number: str - title: str - status: EntityStatus - items: list[ItemOut] - total: float # Computed field - notes: Optional[str] = None - created_at: datetime - updated_at: datetime - -class EntityListOut(BaseModel): - """Summary for list views (performance)""" - id: str - entity_number: str - title: str - status: EntityStatus - total: float - created_at: datetime -``` - -## API Routes (router.py) - -### Basic CRUD Structure - -```python -from typing import Annotated -from fastapi import APIRouter, Depends, HTTPException -from .models import EntityIn, EntityOut, EntityListOut, EntityStatus -from .config import conf -from datetime import datetime -import uuid - -api = APIRouter(prefix=conf.api_prefix) - -# In-memory storage (replace with database) -_entities_db: dict[str, EntityOut] = {} - -# List all -@api.get("/entities", response_model=list[EntityListOut], operation_id="listEntities") -async def list_entities(): - """Get all entities (summary view)""" - return [ - EntityListOut( - id=e.id, - entity_number=e.entity_number, - title=e.title, - status=e.status, - total=e.total, - created_at=e.created_at, - ) - for e in sorted(_entities_db.values(), key=lambda x: x.created_at, reverse=True) - ] - -# Get one -@api.get("/entities/{entity_id}", response_model=EntityOut, operation_id="getEntity") -async def get_entity(entity_id: str): - """Get a specific entity by ID""" - if entity_id not in _entities_db: - raise HTTPException(status_code=404, detail="Entity not found") - return _entities_db[entity_id] - -# Create -@api.post("/entities", response_model=EntityOut, operation_id="createEntity") -async def create_entity(entity_in: EntityIn): - """Create a new entity""" - entity_id = str(uuid.uuid4()) - - # Process items - items = [ - ItemOut( - id=str(uuid.uuid4()), - name=item.name, - value=item.value, - created_at=datetime.now() - ) - for item in entity_in.items - ] - - # Calculate total - total = sum(item.value for item in items) - - entity = EntityOut( - id=entity_id, - entity_number=f"ENT-{datetime.now().strftime('%Y%m%d')}-{len(_entities_db) + 1:04d}", - title=entity_in.title, - status=EntityStatus.STATUS_1, - items=items, - total=total, - notes=entity_in.notes, - created_at=datetime.now(), - updated_at=datetime.now(), - ) - - _entities_db[entity_id] = entity - return entity - -# Update -@api.patch("/entities/{entity_id}", response_model=EntityOut, operation_id="updateEntity") -async def update_entity(entity_id: str, entity_update: EntityIn): - """Update an entity""" - if entity_id not in _entities_db: - raise HTTPException(status_code=404, detail="Entity not found") - - entity = _entities_db[entity_id] - # Apply updates - entity.title = entity_update.title - entity.updated_at = datetime.now() - - return entity - -# Delete -@api.delete("/entities/{entity_id}", operation_id="deleteEntity") -async def delete_entity(entity_id: str): - """Delete an entity""" - if entity_id not in _entities_db: - raise HTTPException(status_code=404, detail="Entity not found") - - del _entities_db[entity_id] - return {"message": "Entity deleted successfully"} -``` - -### Mock Data Initialization - -```python -def _init_mock_data(): - """Initialize with sample data""" - if _entities_db: - return - - mock_data = [ - { - "title": "Sample Entity 1", - "status": EntityStatus.STATUS_1, - "items": [ - {"name": "Item A", "value": 100.0}, - {"name": "Item B", "value": 50.0}, - ], - "notes": "Sample note", - }, - # Add 2-3 more samples - ] - - for idx, data in enumerate(mock_data): - entity_id = str(uuid.uuid4()) - - items = [ - ItemOut( - id=str(uuid.uuid4()), - name=item["name"], - value=item["value"], - created_at=datetime.now() - ) - for item in data["items"] - ] - - entity = EntityOut( - id=entity_id, - entity_number=f"ENT-{datetime.now().strftime('%Y%m%d')}-{idx + 1:04d}", - title=data["title"], - status=data["status"], - items=items, - total=sum(item.value for item in items), - notes=data.get("notes"), - created_at=datetime.now(), - updated_at=datetime.now(), - ) - - _entities_db[entity_id] = entity - -# Call at module level -_init_mock_data() -``` - -## Naming Conventions - -### operation_id → Frontend Hook Name - -| operation_id | Generated Hook | -|--------------|----------------| -| `listEntities` | `useListEntities()`, `useListEntitiesSuspense()` | -| `getEntity` | `useGetEntity(id)`, `useGetEntitySuspense(id)` | -| `createEntity` | `useCreateEntity()` | -| `updateEntity` | `useUpdateEntity()` | -| `deleteEntity` | `useDeleteEntity()` | - -**Pattern**: Verb + EntityName in camelCase diff --git a/.claude/skills/databricks-app-apx/best-practices.md b/.claude/skills/databricks-app-apx/best-practices.md deleted file mode 100644 index ef71f0d..0000000 --- a/.claude/skills/databricks-app-apx/best-practices.md +++ /dev/null @@ -1,318 +0,0 @@ -# APX Best Practices & Anti-Patterns - -Guidelines for building high-quality APX applications. **Consult only when needed.** - -## Critical Rules - -### Backend - -1. **Always include `response_model` and `operation_id`** - ```python - # ✅ Correct - @api.get("/entities", response_model=list[EntityOut], operation_id="listEntities") - - # ❌ Wrong - missing both - @api.get("/entities") - ``` - -2. **Follow 3-model pattern** - - `EntityIn` - Input validation - - `EntityOut` - Complete output - - `EntityListOut` - Performance-optimized summary - -3. **Use descriptive operation_ids** - - Pattern: `` (camelCase) - - Examples: `listOrders`, `getOrder`, `createOrder`, `updateOrderStatus` - -4. **Always use type hints** - ```python - # ✅ Correct - def get_entity(entity_id: str) -> EntityOut: - - # ❌ Wrong - no types - def get_entity(entity_id): - ``` - -5. **Handle errors with HTTPException** - ```python - if entity_id not in db: - raise HTTPException(status_code=404, detail="Not found") - ``` - -### Frontend - -1. **Always use Suspense hooks** - ```typescript - // ✅ Correct - }> - - - - function DataComponent() { - const { data } = useListEntitiesSuspense(selector()); - return
{data.map(...)}
; - } - - // ❌ Wrong - no Suspense - const { data, isLoading } = useListEntities(); - if (isLoading) return
Loading...
; - ``` - -2. **Use selector() for destructuring** - ```typescript - // ✅ Correct - const { data: entities } = useListEntitiesSuspense(selector()); - - // ❌ Wrong - verbose - const result = useListEntitiesSuspense(); - const entities = result.data; - ``` - -3. **Provide matching skeleton fallbacks** - - Skeleton should mirror actual content structure - - Use same table/card layout - -4. **Never edit auto-generated files** - - `lib/api.ts` - Generated by Orval - - `types/routeTree.gen.ts` - Generated by TanStack Router - -5. **Implement proper formatters** - - Currency: `Intl.NumberFormat` - - Dates: `toLocaleDateString` - - Status colors: Tailwind classes with dark mode support - -## Anti-Patterns - -### Backend - -**❌ Missing response_model** -```python -@api.get("/entities") # OpenAPI won't generate correctly -async def list_entities(): - return [] -``` - -**❌ Generic operation_id** -```python -@api.get("/entities", operation_id="get") # Too generic -``` - -**❌ No type safety** -```python -def process(data): # Can't validate, no IDE support - return data["field"] -``` - -**❌ Using plain dicts instead of Pydantic** -```python -def create_entity(data: dict): # No validation - return {"id": "123", **data} -``` - -### Frontend - -**❌ Not using Suspense** -```typescript -const { data, isLoading } = useListEntities(); -if (isLoading) return ; // Manual loading state -``` - -**❌ Editing generated files** -```typescript -// In lib/api.ts -export function useListEntities() { - // Custom changes ❌ -} -``` - -**❌ No skeleton fallback** -```typescript - {/* No fallback - will show nothing */} - - -``` - -**❌ Inline styles or classes** -```typescript -
{/* No dark mode support */} -``` - -## Type Safety - -### Python Type Errors - -**Problem**: Dict access typing -```python -# ❌ Problem -item_data["field"] # Type checker doesn't know structure -``` - -**Solution**: Explicit casting -```python -# ✅ Solution -if not isinstance(item_data, dict): - continue -item_dict: dict[str, Any] = item_data -value = str(item_dict.get("field", "")) -``` - -**Problem**: Optional fields -```python -# ❌ Problem -entity.notes.upper() # notes is Optional[str] -``` - -**Solution**: Check before access -```python -# ✅ Solution -if entity.notes: - entity.notes.upper() -``` - -### TypeScript Type Errors - -**Problem**: Wrong destructuring -```typescript -// ❌ Problem -const { data: response } = useListEntitiesSuspense(selector()); -const entities = response.data; // response.data doesn't exist -``` - -**Solution**: Direct destructuring -```typescript -// ✅ Solution -const { data: entities } = useListEntitiesSuspense(selector()); -``` - -## Performance - -### Backend - -1. **Use EntityListOut for lists** - Don't return full EntityOut for performance -2. **Implement pagination** - For large datasets -3. **Use async** - For I/O operations -4. **Index database queries** - When replacing mock data - -### Frontend - -1. **Use EntityListOut endpoints** - Lists should use summary endpoints -2. **Implement virtual scrolling** - For very long lists -3. **Lazy load detail views** - Don't preload all details -4. **Use React.memo** - Only when profiling shows benefit - -## Code Organization - -### Backend - -``` -backend/ -├── models.py # All Pydantic models -├── router.py # All API routes -├── dependencies.py # Shared dependencies -├── config.py # Configuration -└── utils.py # Helper functions -``` - -**Don't**: Split models/routes across multiple files unless >1000 lines - -### Frontend - -``` -ui/ -├── routes/ -│ └── _sidebar/ -│ ├── entities.tsx # List page -│ └── entities.$entityId.tsx # Detail page -├── components/ -│ ├── ui/ # shadcn components (don't edit) -│ └── apx/ # Custom components -└── lib/ - ├── api.ts # Auto-generated (don't edit) - ├── utils.ts # Helpers (cn, etc.) - └── selector.ts # Query selector -``` - -**Do**: Keep list and detail pages together -**Don't**: Create deep nested route folders - -## Error Messages - -### Backend - -```python -# ✅ Descriptive -raise HTTPException( - status_code=404, - detail=f"Entity with ID {entity_id} not found" -) - -# ❌ Generic -raise HTTPException(status_code=404, detail="Not found") -``` - -### Frontend - -```typescript -// ✅ User-friendly -console.error("Failed to delete order:", error); -// Show toast/alert to user - -// ❌ Silent failure -try { - await deleteEntity.mutateAsync({ entityId }); -} catch {} // Silently swallows error -``` - -## Testing - -### Backend - -```bash -# Type check -uv run basedpyright --level error - -# Test endpoints -curl http://localhost:8000/api/entities | jq . -curl http://localhost:8000/api/entities/{id} | jq . -``` - -### Frontend - -```bash -# Type check -bun run tsc -b --incremental - -# Both -uv run apx dev check -``` - -## Common Pitfalls - -1. **Forgetting to wait for OpenAPI regeneration** - Wait 5-10 seconds after backend changes -2. **Running shadcn from wrong directory** - Must run from project root -3. **Not using --yes flag** - Shadcn will prompt for confirmation -4. **Editing auto-generated files** - Changes will be overwritten -5. **Not implementing skeleton fallbacks** - Page will appear broken while loading -6. **Inconsistent status colors** - Use same color scheme throughout -7. **No dark mode support** - Always use Tailwind dark: classes - -## Debugging Checklist - -**Backend issues**: -- [ ] All models use type hints -- [ ] All routes have response_model + operation_id -- [ ] Mock data initialized correctly -- [ ] Type checking passes - -**Frontend issues**: -- [ ] OpenAPI client regenerated (check timestamp on lib/api.ts) -- [ ] Using Suspense hooks -- [ ] Suspense boundaries in place -- [ ] Hook names match operation_ids -- [ ] Type checking passes - -**Integration issues**: -- [ ] Backend servers running (apx dev status) -- [ ] OpenAPI watcher running -- [ ] API returns correct data (curl test) -- [ ] Frontend URL accessible diff --git a/.claude/skills/databricks-app-apx/frontend-patterns.md b/.claude/skills/databricks-app-apx/frontend-patterns.md deleted file mode 100644 index 29b9685..0000000 --- a/.claude/skills/databricks-app-apx/frontend-patterns.md +++ /dev/null @@ -1,376 +0,0 @@ -# Frontend Code Patterns for APX - -Reference templates for frontend development. **Only consult when writing frontend code.** - -## List Page Template (routes/_sidebar/entities.tsx) - -```typescript -import { createFileRoute, Link } from "@tanstack/react-router"; -import { Suspense } from "react"; -import { useListEntitiesSuspense, EntityStatus } from "@/lib/api"; -import { selector } from "@/lib/selector"; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@/components/ui/table"; -import { Badge } from "@/components/ui/badge"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import { Skeleton } from "@/components/ui/skeleton"; - -export const Route = createFileRoute("/_sidebar/entities")({ - component: () => ( -
- - - Entities - - - }> - - - - -
- ), -}); - -function EntitiesTable() { - const { data: entities } = useListEntitiesSuspense(selector()); - - return ( -
- - - - Number - Title - Status - Total - Created - Actions - - - - {entities.length === 0 ? ( - - - No items found - - - ) : ( - entities.map((entity) => ( - - {entity.entity_number} - {entity.title} - - - {entity.status} - - - {formatCurrency(entity.total)} - {formatDate(entity.created_at)} - - - View - - - - )) - )} - -
-
- ); -} - -function TableSkeleton() { - return ( -
- - - - Number - Title - Status - Total - Created - Actions - - - - {[...Array(4)].map((_, i) => ( - - - - - - - - - ))} - -
-
- ); -} - -// Helper functions -const getStatusColor = (status: EntityStatus) => { - const colors = { - status_1: "bg-yellow-100 text-yellow-800 dark:bg-yellow-900 dark:text-yellow-300", - status_2: "bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-300", - }; - return colors[status] || "bg-gray-100 text-gray-800"; -}; - -const formatDate = (dateString: string) => { - return new Date(dateString).toLocaleDateString("en-US", { - year: "numeric", - month: "short", - day: "numeric", - hour: "2-digit", - minute: "2-digit", - }); -}; - -const formatCurrency = (amount: number) => { - return new Intl.NumberFormat("en-US", { - style: "currency", - currency: "USD", - }).format(amount); -}; -``` - -## Detail Page Template (routes/_sidebar/entities.$entityId.tsx) - -```typescript -import { createFileRoute, Link, useNavigate } from "@tanstack/react-router"; -import { Suspense } from "react"; -import { useGetEntitySuspense, useUpdateEntity, useDeleteEntity } from "@/lib/api"; -import { selector } from "@/lib/selector"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import { Button } from "@/components/ui/button"; -import { Skeleton } from "@/components/ui/skeleton"; -import { ArrowLeft } from "lucide-react"; - -export const Route = createFileRoute("/_sidebar/entities/$entityId")({ - component: () => ( -
- }> - - -
- ), -}); - -function EntityDetail() { - const { entityId } = Route.useParams(); - const navigate = useNavigate(); - const { data: entity } = useGetEntitySuspense(entityId, selector()); - - const updateMutation = useUpdateEntity(); - const deleteMutation = useDeleteEntity(); - - const handleDelete = async () => { - if (!confirm("Are you sure you want to delete this item?")) return; - - try { - await deleteMutation.mutateAsync({ entityId: entity.id }); - navigate({ to: "/entities" }); - } catch (error) { - console.error("Failed to delete:", error); - } - }; - - return ( -
- {/* Header */} -
-
- - - -
-

{entity.entity_number}

-

Entity Details

-
-
- -
- - {/* Content Cards */} -
- - - Information - - -
-

Title

-

{entity.title}

-
-
-

Status

-

{entity.status}

-
- - - - - - Items - - -
- {entity.items.map((item) => ( -
- {item.name} - {formatCurrency(item.value)} -
- ))} -
- - -
-
- ); -} - -function DetailSkeleton() { - return ( -
-
- -
- - -
-
-
- {[...Array(2)].map((_, i) => ( - - - - - - - - - - ))} -
-
- ); -} - -const formatCurrency = (amount: number) => { - return new Intl.NumberFormat("en-US", { - style: "currency", - currency: "USD", - }).format(amount); -}; -``` - -## Navigation Update (routes/_sidebar/route.tsx) - -Add to `navItems` array: - -```typescript -import { Package } from "lucide-react"; // Choose appropriate icon - -const navItems = [ - { - to: "/entities", - label: "Entities", - icon: , - match: (path: string) => path.startsWith("/entities"), - }, - // ... existing items -]; -``` - -## Common Formatters - -```typescript -// Currency -const formatCurrency = (amount: number) => { - return new Intl.NumberFormat("en-US", { - style: "currency", - currency: "USD", - }).format(amount); -}; - -// Date with time -const formatDate = (dateString: string) => { - return new Date(dateString).toLocaleDateString("en-US", { - year: "numeric", - month: "short", - day: "numeric", - hour: "2-digit", - minute: "2-digit", - }); -}; - -// Date only -const formatDateOnly = (dateString: string) => { - return new Date(dateString).toLocaleDateString("en-US", { - year: "numeric", - month: "long", - day: "numeric", - }); -}; - -// Number with commas -const formatNumber = (num: number) => { - return new Intl.NumberFormat("en-US").format(num); -}; -``` - -## Status Badge Colors - -```typescript -const getStatusColor = (status: string) => { - const colors: Record = { - pending: "bg-yellow-100 text-yellow-800 dark:bg-yellow-900 dark:text-yellow-300", - processing: "bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-300", - active: "bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-300", - completed: "bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-300", - cancelled: "bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-300", - inactive: "bg-gray-100 text-gray-800 dark:bg-gray-900 dark:text-gray-300", - }; - return colors[status] || "bg-gray-100 text-gray-800"; -}; -``` - -## Mutation Pattern with Error Handling - -```typescript -const createMutation = useCreateEntity(); - -const handleCreate = async (data: EntityIn) => { - try { - const result = await createMutation.mutateAsync({ data }); - // Success - navigate or show message - navigate({ to: `/entities/${result.data.id}` }); - } catch (error) { - console.error("Failed to create:", error); - // Show error to user - } -}; -``` diff --git a/.claude/skills/databricks-app-python/6-mcp-approach.md b/.claude/skills/databricks-app-python/6-mcp-approach.md deleted file mode 100644 index 23ffb67..0000000 --- a/.claude/skills/databricks-app-python/6-mcp-approach.md +++ /dev/null @@ -1,94 +0,0 @@ -# MCP Tools for App Lifecycle - -Use MCP tools to create, deploy, and manage Databricks Apps programmatically. This mirrors the CLI workflow but can be invoked by AI agents. - ---- - -## Workflow - -### Step 1: Write App Files Locally - -Create your app files in a local folder: - -``` -my_app/ -├── app.py # Main application -├── models.py # Pydantic models -├── backend.py # Data access layer -├── requirements.txt # Additional dependencies -└── app.yaml # Databricks Apps configuration -``` - -### Step 2: Upload to Workspace - -```python -# MCP Tool: upload_folder -upload_folder( - local_folder="/path/to/my_app", - workspace_folder="/Workspace/Users/user@example.com/my_app" -) -``` - -### Step 3: Create App - -```python -# MCP Tool: create_app -result = create_app( - name="my-dashboard", - description="Customer analytics dashboard" -) -# Returns: {"name": "my-dashboard", "url": "https://..."} -``` - -### Step 4: Deploy - -```python -# MCP Tool: deploy_app -result = deploy_app( - app_name="my-dashboard", - source_code_path="/Workspace/Users/user@example.com/my_app" -) -# Returns: {"deployment_id": "...", "status": "PENDING", ...} -``` - -### Step 5: Verify - -```python -# MCP Tool: get_app -app = get_app(name="my-dashboard") -# Returns: {"name": "...", "url": "...", "status": "RUNNING", ...} - -# MCP Tool: get_app_logs -logs = get_app_logs(app_name="my-dashboard") -# Returns: {"logs": "...", ...} -``` - -### Step 6: Iterate - -1. Fix issues in local files -2. Re-upload with `upload_folder` -3. Re-deploy with `deploy_app` -4. Check `get_app_logs` for errors -5. Repeat until app is healthy - ---- - -## Quick Reference: MCP Tools - -| Tool | Description | -|------|-------------| -| **`create_app`** | Create a new Databricks App | -| **`get_app`** | Get app details and status | -| **`list_apps`** | List all apps in the workspace | -| **`deploy_app`** | Deploy app from workspace source path | -| **`delete_app`** | Delete an app | -| **`get_app_logs`** | Get app deployment and runtime logs | -| **`upload_folder`** | Upload local folder to workspace (shared tool) | - ---- - -## Notes - -- Add resources (SQL warehouse, Lakebase, etc.) via the Databricks Apps UI after creating the app -- MCP tools use the service principal's permissions — ensure it has access to required resources -- For manual deployment, see [4-deployment.md](4-deployment.md) diff --git a/.claude/skills/databricks-app-python/1-authorization.md b/.claude/skills/databricks-apps-python/1-authorization.md similarity index 100% rename from .claude/skills/databricks-app-python/1-authorization.md rename to .claude/skills/databricks-apps-python/1-authorization.md diff --git a/.claude/skills/databricks-app-python/2-app-resources.md b/.claude/skills/databricks-apps-python/2-app-resources.md similarity index 100% rename from .claude/skills/databricks-app-python/2-app-resources.md rename to .claude/skills/databricks-apps-python/2-app-resources.md diff --git a/.claude/skills/databricks-app-python/3-frameworks.md b/.claude/skills/databricks-apps-python/3-frameworks.md similarity index 92% rename from .claude/skills/databricks-app-python/3-frameworks.md rename to .claude/skills/databricks-apps-python/3-frameworks.md index cb1ef87..b8e76c8 100644 --- a/.claude/skills/databricks-app-python/3-frameworks.md +++ b/.claude/skills/databricks-apps-python/3-frameworks.md @@ -25,7 +25,7 @@ app = dash.Dash( |--------|-------| | Pre-installed version | 2.18.1 | | app.yaml command | `["python", "app.py"]` | -| Default port | 8050 (set `DATABRICKS_APP_PORT=8080` or use `app.run(port=8080)`) | +| Default port | 8050 — override in code: `app.run(port=int(os.environ.get("DATABRICKS_APP_PORT", 8000)))` | | Auth header | `request.headers.get('x-forwarded-access-token')` (Flask under the hood) | **Databricks tips**: @@ -84,6 +84,7 @@ def get_connection(): **Critical**: Use `gr.Request` parameter to access auth headers. ```python +import os import gradio as gr import requests from databricks.sdk.core import Config @@ -102,14 +103,15 @@ def predict(message, request: gr.Request): return resp.json()["predictions"][0] demo = gr.Interface(fn=predict, inputs="text", outputs="text") -demo.launch(server_name="0.0.0.0", server_port=8080) +port = int(os.environ.get("DATABRICKS_APP_PORT", 8000)) +demo.launch(server_name="0.0.0.0", server_port=port) ``` | Detail | Value | |--------|-------| | Pre-installed version | 4.44.0 | | app.yaml command | `["python", "app.py"]` | -| Default port | 7860 (override with `server_port=8080` or `GRADIO_SERVER_PORT=8080`) | +| Default port | 7860 — override in code: `server_port=int(os.environ.get("DATABRICKS_APP_PORT", 8000))` | | Auth header | `request.headers.get('x-forwarded-access-token')` via `gr.Request` | **Databricks tips**: @@ -150,7 +152,7 @@ def get_data(): | Detail | Value | |--------|-------| | Pre-installed version | 3.0.3 | -| app.yaml command | `["gunicorn", "app:app", "-w", "4", "-b", "0.0.0.0:8080"]` | +| app.yaml command | `["gunicorn", "app:app", "-w", "4", "-b", "0.0.0.0:8000"]` | | Auth header | `request.headers.get('x-forwarded-access-token')` | **Databricks tips**: @@ -190,7 +192,7 @@ async def get_data(request: Request): | Detail | Value | |--------|-------| | Pre-installed version | 0.115.0 | -| app.yaml command | `["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"]` | +| app.yaml command | `["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]` | | Auth header | `request.headers.get('x-forwarded-access-token')` via `Request` | **Databricks tips**: @@ -241,6 +243,6 @@ class State(rx.State): - All frameworks are **pre-installed** — no need to add them to `requirements.txt` - Add only additional packages your app needs to `requirements.txt` - SDK `Config()` auto-detects credentials from injected environment variables -- Databricks Apps expects apps to listen on **port 8080** (configure your framework accordingly) +- Apps must bind to `DATABRICKS_APP_PORT` env var (defaults to 8000). Streamlit is auto-configured by the runtime; for other frameworks, read the env var in code or hardcode 8000 in `app.yaml` command. **Never use 8080** - For framework-specific deployment commands, see [4-deployment.md](4-deployment.md) - For authorization integration, see [1-authorization.md](1-authorization.md) diff --git a/.claude/skills/databricks-app-python/4-deployment.md b/.claude/skills/databricks-apps-python/4-deployment.md similarity index 83% rename from .claude/skills/databricks-app-python/4-deployment.md rename to .claude/skills/databricks-apps-python/4-deployment.md index 688f1f2..0d0ab9f 100644 --- a/.claude/skills/databricks-app-python/4-deployment.md +++ b/.claude/skills/databricks-apps-python/4-deployment.md @@ -31,17 +31,25 @@ env: | Dash | `["python", "app.py"]` | | Streamlit | `["streamlit", "run", "app.py"]` | | Gradio | `["python", "app.py"]` | -| Flask | `["gunicorn", "app:app", "-w", "4", "-b", "0.0.0.0:8080"]` | -| FastAPI | `["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"]` | +| Flask | `["gunicorn", "app:app", "-w", "4", "-b", "0.0.0.0:8000"]` | +| FastAPI | `["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]` | | Reflex | `["reflex", "run", "--env", "prod"]` | +### Excluded directories + +When uploading via the SDK's `upload_folder()` / `upload_to_workspace()`, the following directories are automatically skipped to keep uploads fast: + +`node_modules`, `__pycache__`, `.venv`, `venv`, `.tox`, `.pytest_cache`, `.mypy_cache`, `.ruff_cache`, `dist`, `build`, `.eggs`, `*.egg-info` + +If you use `databricks workspace import-dir` directly, it does **not** apply these exclusions. Either clean the directory first or use the SDK upload functions instead. + ### Step 2: Create and Deploy ```bash # Create the app databricks apps create -# Upload source code +# Upload source code (make sure to exclude node_modules, venv, etc.) databricks workspace mkdirs /Workspace/Users//apps/ databricks workspace import-dir . /Workspace/Users//apps/ @@ -103,7 +111,7 @@ databricks bundle run -t prod **Key difference from other resources**: environment variables go in `src/app/app.yaml`, not `databricks.yml`. -For complete DABs guidance, use the **databricks-asset-bundles** skill. +For complete DABs guidance, use the **databricks-bundles** skill. --- diff --git a/.claude/skills/databricks-app-python/5-lakebase.md b/.claude/skills/databricks-apps-python/5-lakebase.md similarity index 100% rename from .claude/skills/databricks-app-python/5-lakebase.md rename to .claude/skills/databricks-apps-python/5-lakebase.md diff --git a/.claude/skills/databricks-apps-python/6-mcp-approach.md b/.claude/skills/databricks-apps-python/6-mcp-approach.md new file mode 100644 index 0000000..943c49b --- /dev/null +++ b/.claude/skills/databricks-apps-python/6-mcp-approach.md @@ -0,0 +1,79 @@ +# MCP Tools for App Lifecycle + +Use MCP tools to create, deploy, and manage Databricks Apps programmatically. This mirrors the CLI workflow but can be invoked by AI agents. + +--- + +## manage_app - App Lifecycle Management + +| Action | Description | Required Params | +|--------|-------------|-----------------| +| `create_or_update` | Idempotent create, deploys if source_code_path provided | name | +| `get` | Get app details (with optional logs) | name | +| `list` | List all apps | (none, optional name_contains filter) | +| `delete` | Delete an app | name | + +--- + +## Workflow + +### Step 1: Write App Files Locally + +Create your app files in a local folder: + +``` +my_app/ +├── app.py # Main application +├── models.py # Pydantic models +├── backend.py # Data access layer +├── requirements.txt # Additional dependencies +└── app.yaml # Databricks Apps configuration +``` + +### Step 2: Upload to Workspace + +```python +# MCP Tool: manage_workspace_files +manage_workspace_files( + action="upload", + local_path="/path/to/my_app", + workspace_path="/Workspace/Users/user@example.com/my_app" +) +``` + +### Step 3: Create and Deploy App + +```python +# MCP Tool: manage_app (creates if needed + deploys) +result = manage_app( + action="create_or_update", + name="my-dashboard", + description="Customer analytics dashboard", + source_code_path="/Workspace/Users/user@example.com/my_app" +) +# Returns: {"name": "my-dashboard", "url": "...", "created": True, "deployment": {...}} +``` + +### Step 4: Verify + +```python +# MCP Tool: manage_app (get with logs) +app = manage_app(action="get", name="my-dashboard", include_logs=True) +# Returns: {"name": "...", "url": "...", "status": "RUNNING", "logs": "...", ...} +``` + +### Step 5: Iterate + +1. Fix issues in local files +2. Re-upload with `manage_workspace_files(action="upload", ...)` +3. Re-deploy with `manage_app(action="create_or_update", ...)` (will update existing + deploy) +4. Check `manage_app(action="get", name=..., include_logs=True)` for errors +5. Repeat until app is healthy + +--- + +## Notes + +- Add resources (SQL warehouse, Lakebase, etc.) via the Databricks Apps UI after creating the app +- MCP tools use the service principal's permissions — ensure it has access to required resources +- For manual deployment, see [4-deployment.md](4-deployment.md) diff --git a/.claude/skills/databricks-app-python/SKILL.md b/.claude/skills/databricks-apps-python/SKILL.md similarity index 71% rename from .claude/skills/databricks-app-python/SKILL.md rename to .claude/skills/databricks-apps-python/SKILL.md index eb62551..161dbd8 100644 --- a/.claude/skills/databricks-app-python/SKILL.md +++ b/.claude/skills/databricks-apps-python/SKILL.md @@ -1,24 +1,71 @@ --- -name: databricks-app-python -description: "Builds Python-based Databricks applications using Dash, Streamlit, Gradio, Flask, FastAPI, or Reflex. Handles OAuth authorization (app and user auth), app resources, SQL warehouse and Lakebase connectivity, model serving integration, and deployment. Use when building Python web apps, dashboards, ML demos, or REST APIs for Databricks, or when the user mentions Streamlit, Dash, Gradio, Flask, FastAPI, Reflex, or Databricks app." +name: databricks-apps-python +description: "Builds Databricks applications. Prefers AppKit (TypeScript + React SDK) for new apps; falls back to Python frameworks (Dash, Streamlit, Gradio, Flask, FastAPI, Reflex) when Python is required. Handles OAuth authorization, app resources, SQL warehouse and Lakebase connectivity, model serving, foundation model APIs, and deployment. Use when building web apps, dashboards, ML demos, or REST APIs for Databricks, or when the user mentions AppKit, Streamlit, Dash, Gradio, Flask, FastAPI, Reflex, or Databricks app." --- -# Databricks Python Application +# Databricks Applications Build Python-based Databricks applications. For full examples and recipes, see the **[Databricks Apps Cookbook](https://apps-cookbook.dev/)**. --- -## Critical Rules (always follow) +## AppKit (Preferred for New Apps) -- **MUST** confirm framework choice or use [Framework Selection](#framework-selection) below +**[AppKit](https://github.com/databricks/appkit)** is the recommended SDK for new Databricks apps. It is a TypeScript + React SDK with a plugin architecture, built-in caching, telemetry, and end-to-end type safety. + +### Requirements +- Node.js v22+ +- Databricks CLI v0.295.0+ + +### Scaffold a new app +```bash +databricks apps init +``` +This interactive command scaffolds the full project, installs dependencies, and optionally deploys. + +### Deploy +```bash +databricks apps deploy +``` + +### AppKit plugins +| Plugin | Purpose | +|--------|---------| +| **Analytics** | SQL queries against Databricks SQL Warehouses — file-based, typed, cached | +| **Genie** | Conversational AI/BI interface with natural language queries | +| **Files** | Browse/upload Unity Catalog Volumes | +| **Lakebase** | OLTP PostgreSQL via Lakebase with OAuth token management | + +### AI-assisted development +```bash +# Install agent skills for AI-powered scaffolding +databricks experimental aitools skills install + +# Query AppKit docs inline +npx @databricks/appkit docs "your question here" +``` + +### AppKit documentation +- **[AppKit Docs](https://databricks.github.io/appkit/docs/)** — getting started, plugins, API reference +- **[AI-assisted development](https://databricks.github.io/appkit/docs/development/ai-assisted-development)** — guidance for code assistants +- **[llms.txt](https://databricks.github.io/appkit/llms.txt)** — machine-readable docs for AI context + +--- + +## Python Apps (alternative) + +Use Python when: the team is Python-only, you need Streamlit/Dash/Gradio/Gradio, or you are extending an existing Python app. + +## Critical Rules for Python apps (always follow) + +- **MUST** confirm framework choice or use [Python Framework Selection](#python-framework-selection) below - **MUST** use SDK `Config()` for authentication (never hardcode tokens) - **MUST** use `app.yaml` `valueFrom` for resources (never hardcode resource IDs) - **MUST** use `dash-bootstrap-components` for Dash app layout and styling - **MUST** use `@st.cache_resource` for Streamlit database connections - **MUST** deploy Flask with Gunicorn, FastAPI with uvicorn (not dev servers) -## Required Steps +## Required Steps for Python apps Copy this checklist and verify each item: ``` @@ -31,15 +78,15 @@ Copy this checklist and verify each item: --- -## Framework Selection +## Python Framework Selection | Framework | Best For | app.yaml Command | |-----------|----------|------------------| | **Dash** | Production dashboards, BI tools, complex interactivity | `["python", "app.py"]` | | **Streamlit** | Rapid prototyping, data science apps, internal tools | `["streamlit", "run", "app.py"]` | | **Gradio** | ML demos, model interfaces, chat UIs | `["python", "app.py"]` | -| **Flask** | Custom REST APIs, lightweight apps, webhooks | `["gunicorn", "app:app", "-w", "4", "-b", "0.0.0.0:8080"]` | -| **FastAPI** | Async APIs, auto-generated OpenAPI docs | `["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"]` | +| **Flask** | Custom REST APIs, lightweight apps, webhooks | `["gunicorn", "app:app", "-w", "4", "-b", "0.0.0.0:8000"]` | +| **FastAPI** | Async APIs, auto-generated OpenAPI docs | `["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]` | | **Reflex** | Full-stack Python apps without JavaScript | `["reflex", "run", "--env", "prod"]` | **Default**: Recommend **Streamlit** for prototypes, **Dash** for production dashboards, **FastAPI** for APIs, **Gradio** for ML demos. @@ -74,18 +121,21 @@ Copy this checklist and verify each item: **MCP tools**: Use [6-mcp-approach.md](6-mcp-approach.md) for managing app lifecycle via MCP tools — covers creating, deploying, monitoring, and deleting apps programmatically. (Keywords: MCP, create app, deploy app, app logs) +**Foundation Models**: See [examples/llm_config.py](examples/llm_config.py) for calling Databricks foundation model APIs — covers OAuth M2M auth, OpenAI-compatible client wiring, and token caching. (Keywords: foundation model, LLM, OpenAI client, chat completions) + --- ## Workflow 1. Determine the task type: - **New app from scratch?** → Use [Framework Selection](#framework-selection), then read [3-frameworks.md](3-frameworks.md) + **New app from scratch?** → Use [AppKit](#appkit-preferred-for-new-apps) (`databricks apps init`). Fall back to [Python Framework Selection](#python-framework-selection) only if Python is required. **Setting up authorization?** → Read [1-authorization.md](1-authorization.md) **Connecting to data/resources?** → Read [2-app-resources.md](2-app-resources.md) **Using Lakebase (PostgreSQL)?** → Read [5-lakebase.md](5-lakebase.md) **Deploying to Databricks?** → Read [4-deployment.md](4-deployment.md) **Using MCP tools?** → Read [6-mcp-approach.md](6-mcp-approach.md) + **Calling foundation model/LLM APIs?** → See [examples/llm_config.py](examples/llm_config.py) 2. Follow the instructions in the relevant guide 3. For full code examples, browse https://apps-cookbook.dev/ @@ -170,7 +220,7 @@ class EntityIn(BaseModel): | **Resource not accessible** | Add resource via UI, verify SP has permissions, use `valueFrom` in app.yaml | | **Import error on deploy** | Add missing packages to `requirements.txt` (pre-installed packages don't need listing) | | **Lakebase app crashes on start** | `psycopg2`/`asyncpg` are NOT pre-installed — MUST add to `requirements.txt` | -| **Port conflict** | Databricks Apps expects port 8080; configure your framework accordingly | +| **Port conflict** | Apps must bind to `DATABRICKS_APP_PORT` env var (defaults to 8000). Never use 8080. Streamlit is auto-configured; for others, read the env var in code or use 8000 in app.yaml command | | **Streamlit: set_page_config error** | `st.set_page_config()` must be the first Streamlit command | | **Dash: unstyled layout** | Add `dash-bootstrap-components`; use `dbc.themes.BOOTSTRAP` | | **Slow queries** | Use Lakebase for transactional/low-latency; SQL warehouse for analytical queries | @@ -192,6 +242,7 @@ class EntityIn(BaseModel): ## Official Documentation +- **[AppKit](https://databricks.github.io/appkit/docs/)** — preferred SDK for new apps (TypeScript + React) - **[Databricks Apps Overview](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/)** — main docs hub - **[Apps Cookbook](https://apps-cookbook.dev/)** — ready-to-use code snippets (Streamlit, Dash, Reflex, FastAPI) - **[Authorization](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/auth)** — app auth and user auth @@ -202,7 +253,7 @@ class EntityIn(BaseModel): ## Related Skills - **[databricks-app-apx](../databricks-app-apx/SKILL.md)** - full-stack apps with FastAPI + React -- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - deploying apps via DABs +- **[databricks-bundles](../databricks-bundles/SKILL.md)** - deploying apps via DABs - **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - backend SDK integration - **[databricks-lakebase-provisioned](../databricks-lakebase-provisioned/SKILL.md)** - adding persistent PostgreSQL state - **[databricks-model-serving](../databricks-model-serving/SKILL.md)** - serving ML models for app integration diff --git a/.claude/skills/databricks-apps-python/examples/fm-minimal-chat.py b/.claude/skills/databricks-apps-python/examples/fm-minimal-chat.py new file mode 100644 index 0000000..920a405 --- /dev/null +++ b/.claude/skills/databricks-apps-python/examples/fm-minimal-chat.py @@ -0,0 +1,182 @@ +""" +Minimal Databricks Foundation Model Chat App + +A complete, deployable Streamlit app demonstrating Foundation Model API integration +in Databricks Apps. This is a working example extracted from databricksters-check-and-pub. + +Features: +- Validated dual-mode auth (OAuth M2M in Apps, PAT for local dev) +- OpenAI SDK wired to Databricks serving endpoints +- Token caching with expiry check +- Multi-turn chat with conversation history +- Viewer identity display +- Latency tracking + +Local Development: + export DATABRICKS_TOKEN="dapi..." + export DATABRICKS_SERVING_BASE_URL="https:///serving-endpoints" + export DATABRICKS_MODEL="" # See databricks-model-serving + streamlit run fm-minimal-chat.py + +Databricks Apps Deployment: + 1. Create app.yaml: + command: ["streamlit", "run", "fm-minimal-chat.py"] + env: + - name: DATABRICKS_SERVING_BASE_URL + value: "https:///serving-endpoints" + - name: DATABRICKS_MODEL + value: "" # See databricks-model-serving + + 2. Create requirements.txt: + streamlit>=1.38,<2.0 + openai>=1.30,<2.0 + requests>=2.31,<3.0 # Needed for endpoint validation and OAuth fallback + + 3. Deploy: + databricks apps create foundation-chat --source-code-path . + + 4. Add service principal via UI for OAuth M2M auth +""" + +import time +from typing import Dict, List, Optional, Tuple + +import streamlit as st +from openai import OpenAI + +from llm_config import create_foundation_model_client, get_model_name + + +def _get_forwarded_headers() -> Dict[str, str]: + try: + return dict(getattr(st, "context").headers) + except Exception: + return {} + + +def get_viewer_identity() -> Tuple[Optional[str], Optional[str]]: + headers = _get_forwarded_headers() + email = headers.get("X-Forwarded-Email") or headers.get("x-forwarded-email") + token = headers.get("X-Forwarded-Access-Token") or headers.get( + "x-forwarded-access-token" + ) + return email, token + + +# ============================================================================= +# LLM Helper +# ============================================================================= +def llm_chat( + client: OpenAI, + *, + model: str, + messages: List[Dict[str, str]], + max_tokens: int = 1000, + temperature: float = 0.7, +) -> Tuple[str, int]: + """Call foundation model and return (response, latency_ms).""" + t0 = time.perf_counter() + resp = client.chat.completions.create( + model=model, + messages=messages, + max_tokens=max_tokens, + temperature=temperature, + ) + elapsed_ms = int((time.perf_counter() - t0) * 1000) + content = resp.choices[0].message.content or "" + return content, elapsed_ms + + +# ============================================================================= +# Streamlit App +# ============================================================================= +def main(): + st.set_page_config( + page_title="Databricks Foundation Model Chat", + page_icon="💬", + layout="centered", + ) + + st.title("💬 Foundation Model Chat") + st.caption("Powered by Databricks Apps") + + # Sidebar: viewer identity + viewer_email, _ = get_viewer_identity() + if viewer_email: + st.sidebar.success(f"Logged in as: {viewer_email}") + else: + st.sidebar.info("Local dev mode (no viewer identity)") + + # Sidebar: model config + with st.sidebar: + st.subheader("Configuration") + st.code(f"Model: {get_model_name()}", language=None) + + if st.button("🗑️ Clear Chat History"): + st.session_state.messages = [] + st.rerun() + + with st.expander("ℹ️ About"): + st.markdown( + """ + This app demonstrates calling Databricks Foundation Model APIs + from a Streamlit app using: + - Shared dual-mode auth (PAT + OAuth M2M) + - Shared OpenAI client wiring + - Viewer identity extraction + """ + ) + + # Initialize chat history + if "messages" not in st.session_state: + st.session_state.messages = [] + + # Display chat history + for message in st.session_state.messages: + with st.chat_message(message["role"]): + st.markdown(message["content"]) + if message.get("latency_ms"): + st.caption(f"⏱️ {message['latency_ms']}ms") + + # Chat input + if prompt := st.chat_input("Ask me anything..."): + # Add user message to chat history + st.session_state.messages.append({"role": "user", "content": prompt}) + with st.chat_message("user"): + st.markdown(prompt) + + # Generate assistant response + with st.chat_message("assistant"): + with st.spinner("Thinking..."): + try: + client = create_foundation_model_client(cache=st.session_state) + + # Call foundation model + response, latency_ms = llm_chat( + client, + model=get_model_name(), + messages=st.session_state.messages, + max_tokens=1000, + temperature=0.7, + ) + + # Display response + st.markdown(response) + st.caption(f"⏱️ {latency_ms}ms") + + # Add to chat history + st.session_state.messages.append( + { + "role": "assistant", + "content": response, + "latency_ms": latency_ms, + } + ) + + except Exception as e: + st.error(f"Error calling foundation model: {e}") + st.session_state.messages.pop() # Remove failed user message + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/databricks-apps-python/examples/fm-parallel-calls.py b/.claude/skills/databricks-apps-python/examples/fm-parallel-calls.py new file mode 100644 index 0000000..71cd81b --- /dev/null +++ b/.claude/skills/databricks-apps-python/examples/fm-parallel-calls.py @@ -0,0 +1,267 @@ +""" +Parallel Foundation Model Calls + +This example demonstrates how to make multiple foundation model API calls in parallel +for improved performance. It uses the same bounded job-runner pattern as the +production Databricks App, but keeps the example generic enough to reuse in +other review, extraction, or scoring workflows. + +Use cases: +- Document evaluation with multiple independent checks +- Batch processing of independent prompts +- Multi-aspect analysis of the same content +- A/B testing different prompts + +Performance impact: +- Serial: 5 calls × 2s each = 10s total +- Parallel (max_workers=5): ~2s to 3s total depending on endpoint overhead + +Configuration: +- LLM_MAX_CONCURRENCY env var controls parallelism (positive integer, default: 5) +- Balance between throughput and rate limits +- DATABRICKS_MODEL must be set to a valid serving endpoint name +""" + +import time +from typing import Any, Callable, Dict, List, Tuple + +from openai import OpenAI + +from llm_config import ( + create_foundation_model_client, + get_model_name, + run_jobs_parallel, +) + + +# ============================================================================= +# LLM Call Helper +# ============================================================================= +def llm_call( + client: OpenAI, + prompt: str, + model: str | None = None, + max_tokens: int = 500, +) -> Tuple[str, int]: + """Make a single LLM call and return (response, latency_ms).""" + t0 = time.perf_counter() + resp = client.chat.completions.create( + model=model or get_model_name(), + messages=[{"role": "user", "content": prompt}], + max_tokens=max_tokens, + temperature=0.2, + ) + elapsed_ms = int((time.perf_counter() - t0) * 1000) + content = resp.choices[0].message.content or "" + return content, elapsed_ms + + +# ============================================================================= +# Example: Generic Technical Document Checks +# ============================================================================= +def check_structure(client: OpenAI, text: str) -> Dict[str, Any]: + """Check if a technical document has clear section structure.""" + prompt = f"""Evaluate the structure of this technical document. Does it have clear section headings and a logical progression? + +DOCUMENT: +{text[:2000]} + +Answer with: PASS or FAIL, then brief explanation.""" + + response, latency_ms = llm_call(client, prompt) + passed = "PASS" in response.upper().split("\n")[0] + + return { + "check": "structure", + "passed": passed, + "response": response, + "latency_ms": latency_ms, + } + + +def check_summary(client: OpenAI, text: str) -> Dict[str, Any]: + """Check if content has a concise executive summary near the top.""" + prompt = f"""Does this technical document start with a concise summary or key takeaways section in the first 10 percent? + +DOCUMENT: +{text[:2000]} + +Answer with: PASS or FAIL, then brief explanation.""" + + response, latency_ms = llm_call(client, prompt) + passed = "PASS" in response.upper().split("\n")[0] + + return { + "check": "summary", + "passed": passed, + "response": response, + "latency_ms": latency_ms, + } + + +def check_examples(client: OpenAI, text: str) -> Dict[str, Any]: + """Check if content includes concrete examples.""" + prompt = f"""Does this technical document include concrete examples, code, or step-by-step guidance readers can adapt? + +DOCUMENT: +{text[:2000]} + +Answer with: PASS or FAIL, then brief explanation.""" + + response, latency_ms = llm_call(client, prompt) + passed = "PASS" in response.upper().split("\n")[0] + + return { + "check": "examples", + "passed": passed, + "response": response, + "latency_ms": latency_ms, + } + + +def check_troubleshooting(client: OpenAI, text: str) -> Dict[str, Any]: + """Check if content covers troubleshooting or failure modes.""" + prompt = f"""Does this technical document include troubleshooting guidance, failure modes, or common pitfalls? + +DOCUMENT: +{text[:2000]} + +Answer with: PASS or FAIL, then brief explanation.""" + + response, latency_ms = llm_call(client, prompt) + passed = "PASS" in response.upper().split("\n")[0] + + return { + "check": "troubleshooting", + "passed": passed, + "response": response, + "latency_ms": latency_ms, + } + + +def check_audience_fit(client: OpenAI, text: str) -> Dict[str, Any]: + """Check if content matches a technical practitioner audience.""" + prompt = f"""Does this technical document appear written for practitioners, with the right level of specificity and useful context? + +DOCUMENT: +{text[:2000]} + +Answer with: PASS or FAIL, then brief explanation.""" + + response, latency_ms = llm_call(client, prompt) + passed = "PASS" in response.upper().split("\n")[0] + + return { + "check": "audience_fit", + "passed": passed, + "response": response, + "latency_ms": latency_ms, + } + + +# ============================================================================= +# Example Usage: Parallel Execution +# ============================================================================= +if __name__ == "__main__": + # Sample technical document + sample_text = """ + Summary: This guide shows how to deploy a Databricks App in three steps. + + ## Introduction + Databricks Apps provides a way to deploy web applications... + + ## Step 1: Create Your App + First, create an app.py file... + + ## Step 2: Configure app.yaml + Next, set up your configuration... + + ## Step 3: Deploy + Finally, deploy using the CLI... + """ + + client = create_foundation_model_client() + + print("Making 5 parallel LLM calls...") + print(f"Model: {get_model_name()}\n") + + # Define independent parallel jobs + jobs = { + "structure": (check_structure, (client, sample_text), {}), + "summary": (check_summary, (client, sample_text), {}), + "examples": (check_examples, (client, sample_text), {}), + "troubleshooting": (check_troubleshooting, (client, sample_text), {}), + "audience_fit": (check_audience_fit, (client, sample_text), {}), + } + + # Execute in parallel using the shared bounded job runner. + start = time.perf_counter() + results, errors = run_jobs_parallel(jobs) + total_time = time.perf_counter() - start + + # Display results + print("=" * 60) + print(f"Completed in {total_time:.2f}s (parallel execution)") + print("=" * 60) + + if errors: + print("\nErrors encountered:") + for error in errors: + print(f" ❌ {error}") + + print("\nResults:") + for job_name, result in results.items(): + if result: + status = "✅ PASS" if result["passed"] else "❌ FAIL" + print(f"\n{job_name.upper()}: {status}") + print(f" Latency: {result['latency_ms']}ms") + print(f" Response: {result['response'][:150]}...") + else: + print(f"\n{job_name.upper()}: ❌ FAILED (see errors above)") + + # Calculate time saved + total_latency = sum(r["latency_ms"] for r in results.values() if r) + time_saved = (total_latency / 1000) - total_time + print(f"\n{'='*60}") + print(f"Time saved vs serial execution: {time_saved:.2f}s") + if total_time > 0: + print(f"Speedup: {(total_latency/1000) / total_time:.1f}×") + else: + print("Speedup: N/A (total_time below resolution)") + print(f"{'='*60}") + + +# ============================================================================= +# Production Best Practices +# ============================================================================= +# +# Best practices from databricksters-check-and-pub: +# +# 1. Configurable concurrency +# - Use LLM_MAX_CONCURRENCY env var (default: 5 in the production app) +# - Balance throughput vs rate limits +# - Too high = rate limit errors +# - Too low = underutilized resources +# +# 2. Error handling +# - Capture exceptions per job +# - Return None for failed jobs +# - Collect error messages for debugging +# - Continue execution even if some jobs fail +# +# 3. Bounded execution +# - Only parallelize independent checks +# - Cap concurrency with an env var rather than firing unlimited requests +# - Keep the job contract simple: name -> (callable, args, kwargs) +# +# 4. When to use parallel calls +# - Multiple independent evaluations of same content +# - Batch processing multiple documents +# - A/B testing different prompts +# - Multi-aspect analysis +# +# 5. When NOT to use parallel calls +# - Dependent/sequential operations +# - Single evaluation needed +# - Rate limits are very strict +# - Debugging (use serial for easier troubleshooting) diff --git a/.claude/skills/databricks-apps-python/examples/fm-structured-outputs.py b/.claude/skills/databricks-apps-python/examples/fm-structured-outputs.py new file mode 100644 index 0000000..90fe6d2 --- /dev/null +++ b/.claude/skills/databricks-apps-python/examples/fm-structured-outputs.py @@ -0,0 +1,337 @@ +""" +Structured Outputs and Robust Response Parsing + +Production patterns for getting structured data (JSON) from foundation models. +Extracted from databricksters-check-and-pub production app. + +Key patterns: +1. Robust JSON parsing (handles code fences, smart quotes, malformed JSON) +2. Retry logic on parse failure with stricter prompts +3. Content normalization (handles various response formats) +4. temperature=0.0 for deterministic structured outputs +5. Streamlit caching for expensive API calls +6. Consistent timeout handling + +Use cases: +- Content evaluation/scoring +- Data extraction from text +- Classification tasks +- Compliance checking +- Any task requiring structured model output + +Set `DATABRICKS_MODEL` to a valid serving endpoint name before running. +""" + +import json +import re +import time +from typing import Any, Dict, List, Tuple + +import streamlit as st +from openai import OpenAI + +from llm_config import create_foundation_model_client, get_model_name + + +# ============================================================================= +# Pattern 1: Content Normalization +# ============================================================================= +def _content_to_text(content: Any) -> str: + """Normalize model message content to a string. + + Handles various content types returned by foundation models: + - str: return as-is + - bytes: decode to UTF-8 + - list: extract text from content parts (handles multi-modal responses) + + This is critical for handling different response formats consistently. + """ + if isinstance(content, str): + return content + + if isinstance(content, (bytes, bytearray)): + return content.decode("utf-8", errors="replace") + + if isinstance(content, list): + parts: List[str] = [] + for item in content: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict): + # Handle content part objects + if "text" in item and isinstance(item["text"], str): + parts.append(item["text"]) + elif "content" in item and isinstance(item["content"], str): + parts.append(item["content"]) + return "".join(parts) + + return str(content) + + +# ============================================================================= +# Pattern 2: Robust JSON Parsing +# ============================================================================= +def _parse_json_object(response_text: str) -> Dict[str, Any]: + """Best-effort parse of a JSON object from a model response. + + Handles common failure modes: + 1. Model wraps JSON in markdown code fences (```json ... ```) + 2. Model uses smart/curly quotes instead of straight quotes + 3. Model includes extra text before/after JSON + 4. Model returns malformed JSON + + This is THE critical pattern for production structured outputs. + """ + text = (response_text or "").strip() + if not text: + raise ValueError("Empty model response (expected JSON object)") + + # Strip markdown code fences if present + if text.startswith("```"): + text = re.sub(r"^```[a-zA-Z]*\n", "", text) + text = re.sub(r"```$", "", text).strip() + + # Try direct parse first + try: + obj = json.loads(text) + if isinstance(obj, dict): + return obj + except Exception: + pass + + # Extract first {...} block (handles extra text around JSON) + start = text.find("{") + end = text.rfind("}") + if start != -1 and end != -1 and end > start: + candidate = text[start : end + 1] + else: + candidate = text + + # Normalize smart quotes (common LLM formatting issue) + candidate = ( + candidate.replace("\u201c", '"') # Left double quote + .replace("\u201d", '"') # Right double quote + .replace("\u2018", "'") # Left single quote + .replace("\u2019", "'") # Right single quote + ) + + # Final parse attempt + obj = json.loads(candidate) + if not isinstance(obj, dict): + raise ValueError("Model did not return a JSON object") + return obj + + +# ============================================================================= +# Pattern 3: Structured LLM Call with Retry +# ============================================================================= +def llm_structured_call( + client: OpenAI, + system_prompt: str, + user_prompt: str, + model: str | None = None, +) -> Tuple[Dict[str, Any], int]: + """Call foundation model for structured output with retry on parse failure. + + Returns: + (parsed_json_dict, latency_ms) + + Critical pattern: + - Use temperature=0.0 for deterministic structured outputs + - If JSON parse fails, retry with stricter instructions + - Combine latencies from both attempts + """ + # First attempt + t0 = time.perf_counter() + response = client.chat.completions.create( + model=model or get_model_name(), + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + max_tokens=2000, + temperature=0.0, # Deterministic for structured outputs + ) + elapsed_ms = int((time.perf_counter() - t0) * 1000) + + content = _content_to_text(response.choices[0].message.content) + + # Try to parse response + try: + return _parse_json_object(content), elapsed_ms + except Exception as e: + # Retry with stricter prompt + print(f"Parse failed (attempt 1): {e}. Retrying with stricter prompt...") + + t0_retry = time.perf_counter() + retry_response = client.chat.completions.create( + model=model or get_model_name(), + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": "Return ONLY minified JSON object. Strings must be JSON-escaped. No extra text."}, + {"role": "user", "content": user_prompt}, + ], + max_tokens=2000, + temperature=0.0, + ) + retry_elapsed_ms = int((time.perf_counter() - t0_retry) * 1000) + + retry_content = _content_to_text(retry_response.choices[0].message.content) + return _parse_json_object(retry_content), elapsed_ms + retry_elapsed_ms + + +# ============================================================================= +# Pattern 4: Caching Expensive Calls (Streamlit) +# ============================================================================= +@st.cache_data(ttl=60 * 60) # Cache for 1 hour +def cached_structured_call( + prompt: str, + model: str | None = None, +) -> Dict[str, Any]: + """Cache expensive structured LLM calls. + + Use @st.cache_data with TTL for: + - Expensive/slow API calls + - Calls with same inputs (idempotent) + - Data that doesn't need real-time freshness + + TTL examples: + - 60 * 10 = 10 minutes (frequently changing data) + - 60 * 60 = 1 hour (moderate freshness) + - 60 * 60 * 24 = 24 hours (stable data) + """ + client = create_foundation_model_client() + system = "You are a data extraction assistant. Return ONLY valid JSON." + result, _ = llm_structured_call(client, system, prompt, model or get_model_name()) + return result + + +# ============================================================================= +# Example: Content Quality Evaluation +# ============================================================================= +def evaluate_content_quality( + client: OpenAI, text: str +) -> Tuple[Dict[str, Any], int]: + """Evaluate content quality with structured output.""" + + system_prompt = """You are a content quality evaluator. +You must return ONLY valid JSON that exactly matches the schema below. +No commentary. No markdown. No explanations.""" + + user_prompt = f"""Evaluate this content and return JSON with this exact schema: +{{ + "overall_score": 0-100, + "readability": "poor"|"fair"|"good"|"excellent", + "has_clear_structure": true|false, + "has_actionable_takeaways": true|false, + "strengths": ["string", "string"], + "weaknesses": ["string", "string"], + "suggestions": ["string", "string"] +}} + +Content to evaluate: +{text[:2000]} +""" + + return llm_structured_call(client, system_prompt, user_prompt) + + +# ============================================================================= +# Example: Entity Extraction +# ============================================================================= +def extract_entities(client: OpenAI, text: str) -> Tuple[Dict[str, Any], int]: + """Extract structured entities from text.""" + + system_prompt = """You are an entity extraction system. +Return ONLY valid JSON. Do not include explanations.""" + + user_prompt = f"""Extract entities from this text and return JSON: +{{ + "people": ["name1", "name2"], + "organizations": ["org1", "org2"], + "technologies": ["tech1", "tech2"], + "key_concepts": ["concept1", "concept2"] +}} + +Text: +{text[:2000]} +""" + + return llm_structured_call(client, system_prompt, user_prompt) + + +# ============================================================================= +# Example Usage +# ============================================================================= +if __name__ == "__main__": + sample_text = """ + Databricks Lakehouse Platform combines data warehousing and AI with open + data formats like Delta Lake. Apache Spark and MLflow are key components. + Jane Smith, VP of Engineering at Acme Corp, recently shared their migration story. + """ + + client = create_foundation_model_client() + + print("=" * 60) + print("Example 1: Content Quality Evaluation") + print("=" * 60) + try: + quality_data, latency_ms = evaluate_content_quality(client, sample_text) + print(f"✓ Completed in {latency_ms}ms") + print(json.dumps(quality_data, indent=2)) + except Exception as e: + print(f"❌ Error: {e}") + + print("\n" + "=" * 60) + print("Example 2: Entity Extraction") + print("=" * 60) + try: + entity_data, latency_ms = extract_entities(client, sample_text) + print(f"✓ Completed in {latency_ms}ms") + print(json.dumps(entity_data, indent=2)) + except Exception as e: + print(f"❌ Error: {e}") + + +# ============================================================================= +# Production Best Practices Summary +# ============================================================================= +""" +Key takeaways from databricksters-check-and-pub: + +1. Content Normalization (_content_to_text) + - Handle str, bytes, list content types + - Essential for multi-modal or varying response formats + +2. Robust JSON Parsing (_parse_json_object) + - Strip markdown code fences (```json) + - Normalize smart quotes + - Extract {...} from surrounding text + - This ONE function prevents 90% of parsing errors in production + +3. Retry on Parse Failure + - If first attempt fails to parse, retry with stricter prompt + - Add latencies together for accurate tracking + - Shows user total cost, not just successful attempt + +4. Temperature Settings + - Use temperature=0.0 for structured outputs (deterministic) + - Use temperature=0.2-0.7 for creative/generative tasks + - Compliance checks = 0.0, content generation = 0.7 + +5. Caching with TTL + - Use @st.cache_data(ttl=...) for expensive calls + - Choose TTL based on data freshness needs + - Dramatically improves app responsiveness + +6. Timeouts + - Set timeout=30 on all HTTP requests + - Prevents hanging connections + - Provides better error messages to users + +7. System Prompts for Structure + - Clearly state: "Return ONLY valid JSON" + - Provide exact schema in prompt + - Use examples when needed + - Be explicit about constraints +""" diff --git a/.claude/skills/databricks-apps-python/examples/llm_config.py b/.claude/skills/databricks-apps-python/examples/llm_config.py new file mode 100644 index 0000000..200aaef --- /dev/null +++ b/.claude/skills/databricks-apps-python/examples/llm_config.py @@ -0,0 +1,354 @@ +import concurrent.futures +import os +import threading +import time +from collections.abc import MutableMapping as MutableMappingABC +from dataclasses import dataclass +from typing import Any, Callable, Dict, MutableMapping, Tuple +from urllib.parse import urlsplit + +from openai import OpenAI + +CACHE_KEY = "dbx_oauth" +VALIDATION_TTL_SECONDS = 300 + + +class DatabricksLLMConfigError(RuntimeError): + """Raised when Databricks LLM configuration is invalid.""" + + +@dataclass(frozen=True) +class DatabricksLLMConfig: + serving_base_url: str + workspace_host: str + model: str + auth_mode: str + + +_token_lock = threading.Lock() +_token_cache: Dict[str, Any] = {} +_validation_cache: Dict[Tuple[str, str], int] = {} + + +def _requests_module(): + import requests + + return requests + + +def _normalize_host(raw_host: str) -> str: + host = (raw_host or "").strip().rstrip("/") + if not host: + raise DatabricksLLMConfigError("Databricks workspace host is empty.") + if not host.startswith(("http://", "https://")): + host = "https://" + host + parts = urlsplit(host) + if not parts.scheme or not parts.netloc: + raise DatabricksLLMConfigError(f"Invalid Databricks workspace host: {raw_host!r}") + return f"{parts.scheme}://{parts.netloc}" + + +def _normalize_serving_base_url(raw_url: str) -> str: + value = (raw_url or "").strip() + if not value: + raise DatabricksLLMConfigError( + "DATABRICKS_SERVING_BASE_URL must be set to https:///serving-endpoints." + ) + if not value.startswith(("http://", "https://")): + value = "https://" + value + parts = urlsplit(value) + if not parts.scheme or not parts.netloc: + raise DatabricksLLMConfigError(f"Invalid DATABRICKS_SERVING_BASE_URL: {raw_url!r}") + path = parts.path.rstrip("/") + if path != "/serving-endpoints": + raise DatabricksLLMConfigError( + "DATABRICKS_SERVING_BASE_URL must end with /serving-endpoints for the target workspace." + ) + return f"{parts.scheme}://{parts.netloc}/serving-endpoints" + + +def get_databricks_llm_config() -> DatabricksLLMConfig: + serving_base_url = _normalize_serving_base_url( + os.environ.get("DATABRICKS_SERVING_BASE_URL", "") + ) + workspace_host = serving_base_url[: -len("/serving-endpoints")] + + configured_host = os.environ.get("DATABRICKS_HOST", "").strip() + if configured_host: + normalized_host = _normalize_host(configured_host) + if normalized_host != workspace_host: + raise DatabricksLLMConfigError( + "DATABRICKS_HOST must match the workspace host in DATABRICKS_SERVING_BASE_URL." + ) + + model = os.environ.get("DATABRICKS_MODEL", "").strip() + if not model: + raise DatabricksLLMConfigError( + "DATABRICKS_MODEL must be set to a serving endpoint available in the workspace." + ) + + client_id = os.environ.get("DATABRICKS_CLIENT_ID", "").strip() + client_secret = os.environ.get("DATABRICKS_CLIENT_SECRET", "").strip() + token = os.environ.get("DATABRICKS_TOKEN", "").strip() + + if client_id and client_secret: + auth_mode = "oauth-m2m" + elif token: + auth_mode = "pat" + else: + raise DatabricksLLMConfigError( + "No Databricks auth configured. Set DATABRICKS_CLIENT_ID and " + "DATABRICKS_CLIENT_SECRET, or provide DATABRICKS_TOKEN." + ) + + return DatabricksLLMConfig( + serving_base_url=serving_base_url, + workspace_host=workspace_host, + model=model, + auth_mode=auth_mode, + ) + + +def get_serving_base_url() -> str: + return get_databricks_llm_config().serving_base_url + + +def get_model_name() -> str: + return get_databricks_llm_config().model + + +def _is_token_fresh(cache: MutableMapping[str, Any] | Dict[str, Any]) -> bool: + return bool( + cache.get("access_token") + and int(cache.get("expires_at", 0)) > int(time.time()) + 30 + ) + + +def _write_token_cache( + access_token: str, + expires_at: int, + config: DatabricksLLMConfig, + cache: MutableMapping[str, Any] | None = None, +) -> None: + token_record = { + "access_token": access_token, + "expires_at": expires_at, + "workspace_host": config.workspace_host, + "auth_mode": config.auth_mode, + "client_id": os.environ.get("DATABRICKS_CLIENT_ID", "").strip(), + } + _token_cache.clear() + _token_cache.update(token_record) + if cache is not None: + cache[CACHE_KEY] = dict(token_record) + + +def _token_cache_matches( + cache: MutableMapping[str, Any] | Dict[str, Any], + config: DatabricksLLMConfig, +) -> bool: + return bool( + cache.get("workspace_host") == config.workspace_host + and cache.get("auth_mode") == config.auth_mode + and cache.get("client_id", "") == os.environ.get("DATABRICKS_CLIENT_ID", "").strip() + ) + + +def get_databricks_bearer_token( + cache: MutableMapping[str, Any] | None = None, +) -> str: + config = get_databricks_llm_config() + + if config.auth_mode == "pat": + return os.environ["DATABRICKS_TOKEN"].strip() + + if cache: + cached = cache.get(CACHE_KEY, {}) + if ( + isinstance(cached, MutableMappingABC) + and _token_cache_matches(cached, config) + and _is_token_fresh(cached) + ): + _write_token_cache( + str(cached["access_token"]), + int(cached["expires_at"]), + config, + cache=cache, + ) + return str(cached["access_token"]) + + if _token_cache_matches(_token_cache, config) and _is_token_fresh(_token_cache): + access_token = str(_token_cache["access_token"]) + expires_at = int(_token_cache["expires_at"]) + _write_token_cache(access_token, expires_at, config, cache=cache) + return access_token + + with _token_lock: + if _token_cache_matches(_token_cache, config) and _is_token_fresh(_token_cache): + access_token = str(_token_cache["access_token"]) + expires_at = int(_token_cache["expires_at"]) + _write_token_cache(access_token, expires_at, config, cache=cache) + return access_token + + requests = _requests_module() + try: + response = requests.post( + f"{config.workspace_host}/oidc/v1/token", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + data={"grant_type": "client_credentials", "scope": "all-apis"}, + auth=( + os.environ["DATABRICKS_CLIENT_ID"].strip(), + os.environ["DATABRICKS_CLIENT_SECRET"].strip(), + ), + timeout=30, + ) + except Exception as exc: + raise DatabricksLLMConfigError( + f"Could not reach Databricks OAuth token endpoint for " + f"{config.workspace_host}: {type(exc).__name__}: {str(exc)[:200]}" + ) from exc + if response.status_code >= 400: + raise DatabricksLLMConfigError( + f"Failed Databricks OAuth authentication for {config.workspace_host} " + f"(HTTP {response.status_code}). Check the service principal credentials " + "for that workspace." + ) + + payload = response.json() + access_token = payload.get("access_token") + expires_in = int(payload.get("expires_in", 300)) + if not access_token: + payload_keys = sorted(payload.keys()) if isinstance(payload, dict) else [] + raise DatabricksLLMConfigError( + "Token endpoint response is missing access_token " + f"(keys present: {payload_keys})" + ) + + expires_at = int(time.time()) + expires_in + _write_token_cache(str(access_token), expires_at, config, cache=cache) + return str(access_token) + + +def validate_databricks_llm_config( + cache: MutableMapping[str, Any] | None = None, +) -> DatabricksLLMConfig: + config = get_databricks_llm_config() + cache_key = (config.serving_base_url, config.model) + + cached_expiry = _validation_cache.get(cache_key, 0) + if cached_expiry > int(time.time()): + return config + + requests = _requests_module() + token = get_databricks_bearer_token(cache=cache) + headers = {"Authorization": f"Bearer {token}"} + endpoint_url = f"{config.workspace_host}/api/2.0/serving-endpoints/{config.model}" + try: + response = requests.get(endpoint_url, headers=headers, timeout=30) + except Exception as exc: + raise DatabricksLLMConfigError( + f"Could not validate DATABRICKS_MODEL={config.model!r} in workspace " + f"{config.workspace_host}: {type(exc).__name__}: {str(exc)[:200]}" + ) from exc + + if response.status_code == 404: + try: + list_response = requests.get( + f"{config.workspace_host}/api/2.0/serving-endpoints", + headers=headers, + timeout=30, + ) + except Exception: + list_response = None + available: list[str] = [] + if list_response is not None and list_response.status_code < 400: + try: + payload = list_response.json() + available = sorted( + endpoint.get("name", "").strip() + for endpoint in payload.get("endpoints", []) + if endpoint.get("name", "").strip() + ) + except Exception: + available = [] + available_text = ", ".join(available[:10]) if available else "no endpoints were returned" + raise DatabricksLLMConfigError( + f"DATABRICKS_MODEL={config.model!r} was not found in workspace " + f"{config.workspace_host}. Available endpoints include: {available_text}." + ) + + if response.status_code >= 400: + raise DatabricksLLMConfigError( + f"Failed to validate DATABRICKS_MODEL={config.model!r} in workspace " + f"{config.workspace_host} (HTTP {response.status_code})." + ) + + _validation_cache[cache_key] = int(time.time()) + VALIDATION_TTL_SECONDS + return config + + +def build_openai_client( + *, + validate: bool = True, + cache: MutableMapping[str, Any] | None = None, +) -> OpenAI: + config = ( + validate_databricks_llm_config(cache=cache) + if validate + else get_databricks_llm_config() + ) + token = get_databricks_bearer_token(cache=cache) + return OpenAI(api_key=token, base_url=config.serving_base_url) + + +def create_foundation_model_client( + cache: MutableMapping[str, Any] | None = None, +) -> OpenAI: + return build_openai_client(validate=True, cache=cache) + + +def resolve_bearer_token(cache: MutableMapping[str, Any] | None = None) -> str: + return get_databricks_bearer_token(cache=cache) + + +def run_jobs_parallel( + jobs: Dict[str, Tuple[Callable[..., Any], Tuple[Any, ...], Dict[str, Any]]], + max_workers: int | None = None, +) -> Tuple[Dict[str, Any], list[str]]: + """Run independent jobs in parallel and collect per-job failures.""" + if max_workers is None: + raw_worker_count = os.environ.get("LLM_MAX_CONCURRENCY", "5") + try: + worker_count = int(raw_worker_count) + except ValueError as exc: + raise DatabricksLLMConfigError( + "LLM_MAX_CONCURRENCY must be a positive integer." + ) from exc + else: + worker_count = max_workers + + if worker_count < 1: + raise DatabricksLLMConfigError( + "LLM_MAX_CONCURRENCY must be a positive integer." + ) + + results: Dict[str, Any] = {} + errors: list[str] = [] + + def _call(fn: Callable[..., Any], args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> Any: + return fn(*args, **kwargs) + + with concurrent.futures.ThreadPoolExecutor(max_workers=worker_count) as executor: + futures = { + executor.submit(_call, fn, args, kwargs): name + for name, (fn, args, kwargs) in jobs.items() + } + concurrent.futures.wait(list(futures.keys())) + for future, name in [(future, futures[future]) for future in futures]: + try: + results[name] = future.result() + except Exception as exc: + errors.append(f"{name}: {type(exc).__name__}: {str(exc)[:200]}") + results[name] = None + + return results, errors diff --git a/.claude/skills/databricks-asset-bundles/SDP_guidance.md b/.claude/skills/databricks-bundles/SDP_guidance.md similarity index 100% rename from .claude/skills/databricks-asset-bundles/SDP_guidance.md rename to .claude/skills/databricks-bundles/SDP_guidance.md diff --git a/.claude/skills/databricks-asset-bundles/SKILL.md b/.claude/skills/databricks-bundles/SKILL.md similarity index 94% rename from .claude/skills/databricks-asset-bundles/SKILL.md rename to .claude/skills/databricks-bundles/SKILL.md index 4253e8e..3cff53a 100644 --- a/.claude/skills/databricks-asset-bundles/SKILL.md +++ b/.claude/skills/databricks-bundles/SKILL.md @@ -1,9 +1,9 @@ --- -name: databricks-asset-bundles -description: "Create and configure Databricks Asset Bundles (DABs) with best practices for multi-environment deployments. Use when working with: (1) Creating new DAB projects, (2) Adding resources (dashboards, pipelines, jobs, alerts), (3) Configuring multi-environment deployments, (4) Setting up permissions, (5) Deploying or running bundle resources" +name: databricks-bundles +description: "Create and configure Declarative Automation Bundles (formerly Asset Bundles) with best practices for multi-environment deployments (CICD). Use when working with: (1) Creating new DAB projects, (2) Adding resources (dashboards, pipelines, jobs, alerts), (3) Configuring multi-environment deployments, (4) Setting up permissions, (5) Deploying or running bundle resources" --- -# Databricks Asset Bundle (DABs) Writer +# DABs Writer ## Overview Create DABs for multi-environment deployment (dev/staging/prod). @@ -311,13 +311,13 @@ databricks bundle destroy -t prod --auto-approve - **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - pipeline definitions referenced by DABs - **[databricks-app-apx](../databricks-app-apx/SKILL.md)** - app deployment via DABs -- **[databricks-app-python](../databricks-app-python/SKILL.md)** - Python app deployment via DABs +- **[databricks-apps-python](../databricks-apps-python/SKILL.md)** - Python app deployment via DABs - **[databricks-config](../databricks-config/SKILL.md)** - profile and authentication setup for CLI/SDK - **[databricks-jobs](../databricks-jobs/SKILL.md)** - job orchestration managed through bundles ## Resources -- [Databricks Asset Bundles Documentation](https://docs.databricks.com/dev-tools/bundles/) +- [DABs Documentation](https://docs.databricks.com/dev-tools/bundles/) - [Bundle Resources Reference](https://docs.databricks.com/dev-tools/bundles/resources) - [Bundle Configuration Reference](https://docs.databricks.com/dev-tools/bundles/settings) - [Supported Resource Types](https://docs.databricks.com/aws/en/dev-tools/bundles/resources#resource-types) diff --git a/.claude/skills/databricks-asset-bundles/alerts_guidance.md b/.claude/skills/databricks-bundles/alerts_guidance.md similarity index 100% rename from .claude/skills/databricks-asset-bundles/alerts_guidance.md rename to .claude/skills/databricks-bundles/alerts_guidance.md diff --git a/.claude/skills/databricks-config/SKILL.md b/.claude/skills/databricks-config/SKILL.md index 2053f15..118713d 100644 --- a/.claude/skills/databricks-config/SKILL.md +++ b/.claude/skills/databricks-config/SKILL.md @@ -1,81 +1,22 @@ --- name: databricks-config -description: Configure Databricks profile and authenticate for Databricks Connect, Databricks CLI, and Databricks SDK. +description: "Manage Databricks workspace connections: check current workspace, switch profiles, list available workspaces, or authenticate to a new workspace. Use when the user mentions \"switch workspace\", \"which workspace\", \"current profile\", \"databrickscfg\", \"connect to workspace\", or \"databricks auth\"." --- -Configure the Databricks profile in ~/.databrickscfg for use with Databricks Connect. +Use the `manage_workspace` MCP tool for all workspace operations. Do NOT edit `~/.databrickscfg`, use Bash, or use the Databricks CLI. -**Usage:** `/databricks-config [profile_name|workspace_host]` +## Steps -Examples: -- `/databricks-config` - Configure DEFAULT profile (interactive) -- `/databricks-config DEFAULT` - Configure DEFAULT profile -- `/databricks-config my-workspace` - Configure profile named "my-workspace" -- `/databricks-config https://adb-1234567890123456.7.azuredatabricks.net/` - Configure using workspace host URL +1. Call `ToolSearch` with query `select:mcp__databricks__manage_workspace` to load the tool. -## Task +2. Map user intent to action: + - status / which workspace / current → `action="status"` + - list / available workspaces → `action="list"` + - switch to X → call `list` first to find the profile name, then `action="switch", profile=""` (or `host=""` if a URL was given) + - login / connect / authenticate → `action="login", host=""` -1. Determine the profile and host: - - If a parameter is provided and it starts with `https://`, treat it as a workspace host: - - Extract profile name from the host (e.g., `adb-1234567890123456.7.azuredatabricks.net` → `adb-1234567890123456`, `my-company-dev.cloud.databricks.com` → `my-company-dev`) - - Use this as the profile name and configure it with the provided host - - If a parameter is provided and it doesn't start with `https://`, treat it as a profile name - - If no parameter is provided, ask the user which profile they want to configure (default: DEFAULT) +3. Call `mcp__databricks__manage_workspace` with the action and any parameters. -2. Run `databricks auth login -p ` with the determined profile name - - If a workspace host was provided, add `--host ` to the command - - This ensures authentication is completed and the profile works -3. Check if the profile exists in ~/.databrickscfg -4. Ask the user to choose ONE of the following compute options: - - **Cluster ID**: Provide a specific cluster ID for an interactive/all-purpose cluster - - **Serverless**: Use serverless compute (sets `serverless_compute_id = auto`) -5. Update the profile in ~/.databrickscfg with the selected configuration -6. Verify the configuration by displaying the updated profile section +4. Present the result. For `status`/`switch`/`login`: show host, profile, username. For `list`: formatted table with the active profile marked. -## Important Notes - -- Use the AskUserQuestion tool to present the compute options as a choice -- Only add ONE of: `cluster_id` OR `serverless_compute_id` (never both) -- For serverless, set `serverless_compute_id = auto` (not just `serverless = true`) -- Preserve all existing settings in the profile (host, auth_type, etc.) -- Format the configuration file consistently with proper spacing -- The `databricks auth login` command will open a browser for OAuth authentication -- **SECURITY: NEVER print token values in plain text** - - When displaying configuration, redact any `token` field values (e.g., `token = [REDACTED]`) - - Inform the user they can view the full configuration at `~/.databrickscfg` - - This applies to any output showing the profile configuration - -## Example Configurations - -**With Cluster ID:** -``` -[DEFAULT] -host = https://adb-123456789.11.azuredatabricks.net/ -cluster_id = 1217-064531-c9c3ngyn -auth_type = databricks-cli -``` - -**With Serverless:** -``` -[DEFAULT] -host = https://adb-123456789.11.azuredatabricks.net/ -serverless_compute_id = auto -auth_type = databricks-cli -``` - -**With Token (display as redacted):** -``` -[DEFAULT] -host = https://adb-123456789.11.azuredatabricks.net/ -token = [REDACTED] -cluster_id = 1217-064531-c9c3ngyn - -View full configuration at: ~/.databrickscfg -``` - -## Related Skills - -- **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - uses profiles configured by this skill -- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - references workspace profiles for deployment targets -- **[databricks-app-apx](../databricks-app-apx/SKILL.md)** - apps that connect via configured profiles -- **[databricks-app-python](../databricks-app-python/SKILL.md)** - Python apps using configured profiles +> **Note:** The switch is session-scoped — it resets on MCP server restart. For permanent profile setup, use `databricks auth login -p ` and update `~/.databrickscfg` with `cluster_id` or `serverless_compute_id = auto`. diff --git a/.claude/skills/databricks-docs/SKILL.md b/.claude/skills/databricks-docs/SKILL.md index 54bb157..ceca11e 100644 --- a/.claude/skills/databricks-docs/SKILL.md +++ b/.claude/skills/databricks-docs/SKILL.md @@ -1,6 +1,6 @@ --- name: databricks-docs -description: "Databricks documentation reference. Use as a lookup resource alongside other skills and MCP tools for comprehensive guidance." +description: "Databricks documentation reference via llms.txt index. Use when other skills do not cover a topic, looking up unfamiliar Databricks features, or needing authoritative docs on APIs, configurations, or platform capabilities." --- # Databricks Documentation Reference @@ -16,7 +16,7 @@ This is a **reference skill**, not an action skill. Use it to: - Find detailed information to inform how you use MCP tools - Discover features and capabilities you may not know about -**Always prefer using MCP tools for actions** (execute_sql, create_or_update_pipeline, etc.) and **load specific skills for workflows** (databricks-python-sdk, databricks-spark-declarative-pipelines, etc.). Use this skill when you need reference documentation. +**Always prefer using MCP tools for actions** (execute_sql, manage_pipeline, etc.) and **load specific skills for workflows** (databricks-python-sdk, databricks-spark-declarative-pipelines, etc.). Use this skill when you need reference documentation. ## How to Use @@ -47,7 +47,7 @@ The llms.txt file is organized by category: 1. Load `databricks-spark-declarative-pipelines` skill for workflow patterns 2. Use this skill to fetch docs if you need clarification on specific DLT features -3. Use `create_or_update_pipeline` MCP tool to actually create the pipeline +3. Use `manage_pipeline(action="create_or_update")` MCP tool to actually create the pipeline **Scenario:** User asks about an unfamiliar Databricks feature diff --git a/.claude/skills/databricks-execution-compute/SKILL.md b/.claude/skills/databricks-execution-compute/SKILL.md new file mode 100644 index 0000000..c351838 --- /dev/null +++ b/.claude/skills/databricks-execution-compute/SKILL.md @@ -0,0 +1,82 @@ +--- +name: databricks-execution-compute +description: >- + Execute code and manage compute on Databricks. Use this skill when the user + mentions: "run code", "execute", "run on databricks", "serverless", "no + cluster", "run python", "run scala", "run sql", "run R", "run file", "push + and run", "notebook run", "batch script", "model training", "run script on + cluster", "create cluster", "new cluster", "resize cluster", "modify cluster", + "delete cluster", "terminate cluster", "create warehouse", "new warehouse", + "resize warehouse", "delete warehouse", "node types", "runtime versions", + "DBR versions", "spin up compute", "provision cluster". +--- + +# Databricks Execution & Compute + +Run code on Databricks. Three execution modes—choose based on workload. + +## Execution Mode Decision Matrix + +| Aspect | [Databricks Connect](references/1-databricks-connect.md) ⭐ | [Serverless Job](references/2-serverless-job.md) | [Interactive Cluster](references/3-interactive-cluster.md) | +|--------|-------------------|----------------|---------------------| +| **Use for** | Spark code (ETL, data gen) | Heavy processing (ML) | State across tool calls, Scala/R | +| **Startup** | Instant | ~25-50s cold start | ~5min if stopped | +| **State** | Within Python process | None | Via context_id | +| **Languages** | Python (PySpark) | Python, SQL | Python, Scala, SQL, R | +| **Dependencies** | `withDependencies()` | CLI with environments spec | Install on cluster | + +### Decision Flow + +``` +Spark-based code? → Databricks Connect (fastest) + └─ Python 3.12 missing? → Install it + databricks-connect + └─ Install fails? → Ask user (don't auto-switch modes) + +Heavy/long-running (ML)? → Serverless Job (independent) +Need state across calls? → Interactive Cluster (list and ask which one to use) +Scala/R? → Interactive Cluster (list and ask which one to use) +``` + + +## How to Run Code + +**Read the reference file for your chosen mode before proceeding.** + +### Databricks Connect (no MCP tool, run locally) → [reference](references/1-databricks-connect.md) + +```bash +python my_spark_script.py +``` + +### Serverless Job → [reference](references/2-serverless-job.md) + +```python +execute_code(file_path="/path/to/script.py") +``` + +### Interactive Cluster → [reference](references/3-interactive-cluster.md) + +```python +# Check for running clusters first (or use the one instructed) +list_compute(resource="clusters") +# Ask the customer which one to use + +# Run code, reuse context_id for follow-up MCP call +result = execute_code(code="...", compute_type="cluster", cluster_id="...") +execute_code(code="...", context_id=result["context_id"], cluster_id=result["cluster_id"]) +``` + +## MCP Tools + +| Tool | For | Purpose | +|------|-----|---------| +| `execute_code` | Serverless, Interactive | Run code remotely | +| `list_compute` | Interactive | List clusters, check status, auto-select running cluster | +| `manage_cluster` | Interactive | Create, start, terminate, delete. **COSTLY:** `start` takes 3-8 min—ask user | +| `manage_sql_warehouse` | SQL | Create, modify, delete SQL warehouses | + +## Related Skills + +- **[databricks-synthetic-data-gen](../databricks-synthetic-data-gen/SKILL.md)** — Data generation using Spark + Faker +- **[databricks-jobs](../databricks-jobs/SKILL.md)** — Production job orchestration +- **[databricks-dbsql](../databricks-dbsql/SKILL.md)** — SQL warehouse and AI functions diff --git a/.claude/skills/databricks-execution-compute/references/1-databricks-connect.md b/.claude/skills/databricks-execution-compute/references/1-databricks-connect.md new file mode 100644 index 0000000..838d2a7 --- /dev/null +++ b/.claude/skills/databricks-execution-compute/references/1-databricks-connect.md @@ -0,0 +1,72 @@ +# Databricks Connect (Recommended Default) + +**Use when:** Running Spark code locally that executes on Databricks serverless compute. This is the fastest, cleanest approach for data generation, ETL, and any Spark workload. + +## Why Databricks Connect First? + +- **Instant iteration** — Edit file, re-run immediately +- **Local debugging** — IDE debugger, breakpoints work +- **No cold start** — Session stays warm across executions +- **Clean dependencies** — `withDependencies()` installs packages on remote compute + +## Requirements + +- **Python 3.12** (databricks-connect >= 16.4 requires it) +- **databricks-connect >= 16.4** package +- **~/.databrickscfg** with serverless config + +## Setup + +**Python 3.12 required.** If not available, install it (uv or other). If install fails, ask user—don't auto-switch modes. + +Use default profile, if not setup you can add it `~/.databrickscfg` (never overwrite it without conscent) +```ini +[DEFAULT] +host = https://your-workspace.cloud.databricks.com/ +serverless_compute_id = auto +auth_type = databricks-cli +``` + +## Usage Pattern + +```python +from databricks.connect import DatabricksSession, DatabricksEnv + +# Declare dependencies installed on serverless compute +# CRITICAL: Include ALL packages used inside UDFs (pandas/numpy are there by default) +env = DatabricksEnv().withDependencies("faker", "holidays") + +spark = ( + DatabricksSession.builder + .profile("my-workspace") # optional: run on a specific profile from ~/.databrickscfg instead of default + .withEnvironment(env) + .serverless(True) + .getOrCreate() +) + +# Spark code now executes on Databricks serverless +df = spark.range(1000)... +df.write.mode('overwrite').saveAsTable("catalog.schema.table") +``` + +## Common Issues + +| Issue | Solution | +|-------|----------| +| `Python 3.12 required` | create venv with correct python version | +| `DatabricksEnv not found` | Upgrade to databricks-connect >= 16.4 | +| `serverless_compute_id` error | Add `serverless_compute_id = auto` to ~/.databrickscfg | +| `ModuleNotFoundError` inside UDF | Add the package to `withDependencies()` | +| `PERSIST TABLE not supported` | Don't use `.cache()` or `.persist()` with serverless | +| `broadcast` is used | Don't broadcast small DF using spark connect, have a small python list instead or join small DF | + +## When NOT to Use + +Switch to **[Serverless Job](2-serverless-job.md)** when: +- one-off execution +- Heavy ML training that shouldn't depend on local machine staying connected +- Non-Spark Python code (pure sklearn, pytorch, etc.) + +Switch to **[Interactive Cluster](3-interactive-cluster.md)** when: +- Need state across multiple separate MCP tool calls +- Need Scala or R support diff --git a/.claude/skills/databricks-execution-compute/references/2-serverless-job.md b/.claude/skills/databricks-execution-compute/references/2-serverless-job.md new file mode 100644 index 0000000..4be8801 --- /dev/null +++ b/.claude/skills/databricks-execution-compute/references/2-serverless-job.md @@ -0,0 +1,76 @@ +# Serverless Job Execution + +**Use when:** Running intensive Python code remotely (ML training, heavy processing) that doesn't need Spark, or when code shouldn't depend on local machine staying connected. + +## When to Choose Serverless Job + +- ML model training (runs independently of local machine) +- Heavy non-Spark Python processing +- Code that takes > 5 minutes (local connection can drop) +- Production/scheduled runs + +## Trade-offs + +| Pro | Con | +|-----|-----| +| No cluster to manage | ~25-50s cold start each invocation | +| Up to 30 min timeout | No state preserved between calls | +| Independent execution | print() unreliable—use `dbutils.notebook.exit()` | + +## Executing code +### Prefer running from a Local File (edit the local file then run it) + +```python +execute_code( + file_path="/local/path/to/train_model.py", + compute_type="serverless" +) +``` + +## Jobs with Custom Dependencies + +Use `job_extra_params` to install pip packages: + +```python +execute_code( + file_path="/path/to/train.py", + job_extra_params={ + "environments": [{ + "environment_key": "ml_env", + "spec": {"client": "4", "dependencies": ["scikit-learn", "pandas", "mlflow"]} + }] + } +) +``` + +**CRITICAL:** Use `"client": "4"` in the spec. `"client": "1"` won't install dependencies. + +## Output Handling + +```python +# ❌ BAD - print() may not be captured +print("Training complete!") + +# ✅ GOOD - Use dbutils.notebook.exit() +import json +results = {"accuracy": 0.95, "model_path": "/Volumes/..."} +dbutils.notebook.exit(json.dumps(results)) +``` + +## Common Issues + +| Issue | Solution | +|-------|----------| +| print() output missing | Use `dbutils.notebook.exit()` | +| `ModuleNotFoundError` | Add to environments spec with `"client": "4"` | +| Job times out | Max is 1800s; split into smaller tasks | + +## When NOT to Use + +Switch to **[Databricks Connect](1-databricks-connect.md)** when: +- Iterating on Spark code and want instant feedback +- Need local debugging with breakpoints + +Switch to **[Interactive Cluster](3-interactive-cluster.md)** when: +- Need state across multiple MCP tool calls +- Need Scala or R support diff --git a/.claude/skills/databricks-execution-compute/references/3-interactive-cluster.md b/.claude/skills/databricks-execution-compute/references/3-interactive-cluster.md new file mode 100644 index 0000000..aa73ea9 --- /dev/null +++ b/.claude/skills/databricks-execution-compute/references/3-interactive-cluster.md @@ -0,0 +1,140 @@ +# Interactive Cluster Execution + +**Use when:** You have an existing running cluster and need to preserve state across multiple MCP tool calls, or need Scala/R support. + +## When to Choose Interactive Cluster + +- Multiple sequential commands where variables must persist +- Scala or R code (serverless only supports Python/SQL) +- Existing running cluster available + +## Trade-offs + +| Pro | Con | +|-----|-----| +| State persists via `context_id` | Cluster startup ~5 min if not running | +| Near-instant follow-up commands | Costs money while running | +| Scala/R/SQL support | Must manage cluster lifecycle | + +## Critical: Never Start a Cluster Without Asking + +**Starting a cluster takes 3-8 minutes and costs money.** Always check first: + +```python +list_compute(resource="clusters") +``` + +If no cluster is running, ask the user: +> "No running cluster. Options: +> 1. Start 'my-dev-cluster' (~5 min startup, costs money) +> 2. Use serverless (instant, no setup) +> Which do you prefer?" + +## Basic Usage + +### First Command: Creates Context + +```python +result = execute_code( + code="import pandas as pd\ndf = pd.DataFrame({'a': [1, 2, 3]})", + compute_type="cluster", + cluster_id="1234-567890-abcdef" +) +# result contains context_id for reuse +``` + +### Follow-up Commands: Reuse Context + +```python +# Variables from first command still available +execute_code( + code="print(df.shape)", # df exists + context_id=result["context_id"], + cluster_id=result["cluster_id"] +) +``` + +### Auto-Select Best Running Cluster + +```python +best_cluster = list_compute(resource="clusters", auto_select=True) +execute_code( + code="spark.range(100).show()", + compute_type="cluster", + cluster_id=best_cluster["cluster_id"] +) +``` + +## Language Support + +```python +execute_code(code='println("Hello")', compute_type="cluster", language="scala") +execute_code(code="SELECT * FROM table LIMIT 10", compute_type="cluster", language="sql") +execute_code(code='print("Hello")', compute_type="cluster", language="r") +``` + +## Installing Libraries + +Install pip packages directly in the execution context (pandas/numpy are there by default): + +```python +# Install library +execute_code( + code="""%pip install faker + dbutils.library.restartPython()""", # Restart Python to pick up new packages (if needed) + compute_type="cluster", + cluster_id="...", + context_id="..." +) +``` + +## Context Lifecycle + +**Keep alive (default):** Context persists until cluster terminates. + +**Destroy when done:** +```python +execute_code( + code="print('Done!')", + compute_type="cluster", + destroy_context_on_completion=True +) +``` + +## Handling No Running Cluster + +When no cluster is running, `execute_code` returns: +```json +{ + "success": false, + "error": "No running cluster available", + "startable_clusters": [{"cluster_id": "...", "cluster_name": "...", "state": "TERMINATED"}], + "suggestions": ["Start a terminated cluster", "Use serverless instead"] +} +``` + +### Starting a Cluster (With User Approval Only) + +```python +manage_cluster(action="start", cluster_id="1234-567890-abcdef") +# Poll until running (wait 20sec) +list_compute(resource="clusters", cluster_id="1234-567890-abcdef") +``` + +## Common Issues + +| Issue | Solution | +|-------|----------| +| "No running cluster" | Ask user to start or use serverless | +| Context not found | Context expired; create new one | +| Library not found | `%pip install ` then if needed `dbutils.library.restartPython()` | + +## When NOT to Use + +Switch to **[Databricks Connect](1-databricks-connect.md)** when: +- Developing Spark code with local debugging +- Want instant iteration without cluster concerns + +Switch to **[Serverless Job](2-serverless-job.md)** when: +- No cluster running and user doesn't want to wait +- One-off execution without state needs diff --git a/.claude/skills/databricks-genie/SKILL.md b/.claude/skills/databricks-genie/SKILL.md index 3f08628..8233247 100644 --- a/.claude/skills/databricks-genie/SKILL.md +++ b/.claude/skills/databricks-genie/SKILL.md @@ -1,11 +1,11 @@ --- name: databricks-genie -description: "Create and query Databricks Genie Spaces for natural language SQL exploration. Use when building Genie Spaces or asking questions via the Genie Conversation API." +description: "Create and query Databricks Genie Spaces for natural language SQL exploration. Use when building Genie Spaces, exporting and importing Genie Spaces, migrating Genie Spaces between workspaces or environments, or asking questions via the Genie Conversation API." --- # Databricks Genie -Create and query Databricks Genie Spaces - natural language interfaces for SQL-based data exploration. +Create, manage, and query Databricks Genie Spaces - natural language interfaces for SQL-based data exploration. ## Overview @@ -18,31 +18,88 @@ Use this skill when: - Adding sample questions to guide users - Connecting Unity Catalog tables to a conversational interface - Asking questions to a Genie Space programmatically (Conversation API) +- Exporting a Genie Space configuration (serialized_space) for backup or migration +- Importing / cloning a Genie Space from a serialized payload +- Migrating a Genie Space between workspaces or environments (dev → staging → prod) + - Only supports catalog remapping where catalog names differ across environments + - Not supported for schema and/or table names that differ across environments + - Not including migration of tables between environments (only migration of Genie Spaces) ## MCP Tools -### Space Management - | Tool | Purpose | |------|---------| -| `list_genie` | List all Genie Spaces accessible to you | -| `create_or_update_genie` | Create or update a Genie Space | -| `get_genie` | Get Genie Space details | -| `delete_genie` | Delete a Genie Space | +| `manage_genie` | Create, get, list, delete, export, and import Genie Spaces | +| `ask_genie` | Ask natural language questions to a Genie Space | +| `get_table_stats_and_schema` | Inspect table schemas before creating a space | +| `execute_sql` | Test SQL queries directly | -### Conversation API +### manage_genie - Space Management -| Tool | Purpose | -|------|---------| -| `ask_genie` | Ask a question to a Genie Space, get SQL + results | -| `ask_genie_followup` | Ask follow-up question in existing conversation | +| Action | Description | Required Params | +|--------|-------------|-----------------| +| `create_or_update` | Idempotent create/update a space | display_name, table_identifiers (or serialized_space) | +| `get` | Get space details | space_id | +| `list` | List all spaces | (none) | +| `delete` | Delete a space | space_id | +| `export` | Export space config for migration/backup | space_id | +| `import` | Import space from serialized config | warehouse_id, serialized_space | + +**Example tool calls:** +``` +# MCP Tool: manage_genie +# Create a new space +manage_genie( + action="create_or_update", + display_name="Sales Analytics", + table_identifiers=["catalog.schema.customers", "catalog.schema.orders"], + description="Explore sales data with natural language", + sample_questions=["What were total sales last month?"] +) -### Supporting Tools +# MCP Tool: manage_genie +# Get space details with full config +manage_genie(action="get", space_id="space_123", include_serialized_space=True) + +# MCP Tool: manage_genie +# List all spaces +manage_genie(action="list") + +# MCP Tool: manage_genie +# Export for migration +exported = manage_genie(action="export", space_id="space_123") + +# MCP Tool: manage_genie +# Import to new workspace +manage_genie( + action="import", + warehouse_id="warehouse_456", + serialized_space=exported["serialized_space"], + title="Sales Analytics (Prod)" +) +``` -| Tool | Purpose | -|------|---------| -| `get_table_details` | Inspect table schemas before creating a space | -| `execute_sql` | Test SQL queries directly | +### ask_genie - Conversation API (Query) + +Ask natural language questions to a Genie Space. Pass `conversation_id` for follow-up questions. + +``` +# MCP Tool: ask_genie +# Start a new conversation +result = ask_genie( + space_id="space_123", + question="What were total sales last month?" +) +# Returns: {question, conversation_id, message_id, status, sql, columns, data, row_count} + +# MCP Tool: ask_genie +# Follow-up question in same conversation +result = ask_genie( + space_id="space_123", + question="Break that down by region", + conversation_id=result["conversation_id"] +) +``` ## Quick Start @@ -50,8 +107,9 @@ Use this skill when: Before creating a Genie Space, understand your data: -```python -get_table_details( +``` +# MCP Tool: get_table_stats_and_schema +get_table_stats_and_schema( catalog="my_catalog", schema="sales", table_stat_level="SIMPLE" @@ -60,8 +118,10 @@ get_table_details( ### 2. Create the Genie Space -```python -create_or_update_genie( +``` +# MCP Tool: manage_genie +manage_genie( + action="create_or_update", display_name="Sales Analytics", table_identifiers=[ "my_catalog.sales.customers", @@ -77,7 +137,8 @@ create_or_update_genie( ### 3. Ask Questions (Conversation API) -```python +``` +# MCP Tool: ask_genie ask_genie( space_id="your_space_id", question="What were total sales last month?" @@ -85,15 +146,31 @@ ask_genie( # Returns: SQL, columns, data, row_count ``` -## Workflow +### 4. Export & Import (Clone / Migrate) + +Export a space (preserves all tables, instructions, SQL examples, and layout): + +``` +# MCP Tool: manage_genie +exported = manage_genie(action="export", space_id="your_space_id") +# exported["serialized_space"] contains the full config +``` + +Clone to a new space (same catalog): ``` -1. Inspect tables → get_table_details -2. Create space → create_or_update_genie -3. Query space → ask_genie (or test in Databricks UI) -4. Curate (optional) → Use Databricks UI to add instructions +# MCP Tool: manage_genie +manage_genie( + action="import", + warehouse_id=exported["warehouse_id"], + serialized_space=exported["serialized_space"], + title=exported["title"], # override title; omit to keep original + description=exported["description"], +) ``` +> **Cross-workspace migration:** Each MCP server is workspace-scoped. Configure one server entry per workspace profile in your IDE's MCP config, then `manage_genie(action="export")` from the source server and `manage_genie(action="import")` via the target server. See [spaces.md §Migration](spaces.md#migrating-across-workspaces-with-catalog-remapping) for the full workflow. + ## Reference Files - [spaces.md](spaces.md) - Creating and managing Genie Spaces @@ -109,20 +186,15 @@ Before creating a Genie Space: ### Creating Tables Use these skills in sequence: -1. `databricks-synthetic-data-generation` - Generate raw parquet files +1. `databricks-synthetic-data-gen` - Generate raw parquet files 2. `databricks-spark-declarative-pipelines` - Create bronze/silver/gold tables ## Common Issues -| Issue | Solution | -|-------|----------| -| **No warehouse available** | Create a SQL warehouse or provide `warehouse_id` explicitly | -| **Poor query generation** | Add instructions and sample questions that reference actual column names | -| **Slow queries** | Ensure warehouse is running; use OPTIMIZE on tables | - +See [spaces.md §Troubleshooting](spaces.md#troubleshooting) for a full list of issues and solutions. ## Related Skills - **[databricks-agent-bricks](../databricks-agent-bricks/SKILL.md)** - Use Genie Spaces as agents inside Supervisor Agents -- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - Generate raw parquet data to populate tables for Genie +- **[databricks-synthetic-data-gen](../databricks-synthetic-data-gen/SKILL.md)** - Generate raw parquet data to populate tables for Genie - **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - Build bronze/silver/gold tables consumed by Genie Spaces - **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - Manage the catalogs, schemas, and tables Genie queries diff --git a/.claude/skills/databricks-genie/conversation.md b/.claude/skills/databricks-genie/conversation.md index 149cafa..e4320e8 100644 --- a/.claude/skills/databricks-genie/conversation.md +++ b/.claude/skills/databricks-genie/conversation.md @@ -31,8 +31,7 @@ The `ask_genie` tool allows you to programmatically send questions to a Genie Sp | Tool | Purpose | |------|---------| -| `ask_genie` | Ask a question, start new conversation | -| `ask_genie_followup` | Ask follow-up in existing conversation | +| `ask_genie` | Ask a question or follow-up (`conversation_id` optional) | ## Basic Usage @@ -71,10 +70,10 @@ result = ask_genie( ) # Follow-up (uses context from first question) -ask_genie_followup( +ask_genie( space_id="01abc123...", - conversation_id=result["conversation_id"], - question="Break that down by region" + question="Break that down by region", + conversation_id=result["conversation_id"] ) ``` @@ -148,7 +147,7 @@ Claude: User: "I just created a Genie Space for HR data. Can you test it?" Claude: -1. Gets the space_id from the user or recent create_or_update_genie result +1. Gets the space_id from the user or recent manage_genie(action="create_or_update") result 2. Calls ask_genie with test questions: - "How many employees do we have?" - "What is the average salary by department?" @@ -163,9 +162,9 @@ User: "Use my analytics Genie to explore sales trends" Claude: 1. ask_genie(space_id, "What were total sales by month this year?") 2. User: "Which month had the highest growth?" -3. ask_genie_followup(space_id, conv_id, "Which month had the highest growth?") +3. ask_genie(space_id, "Which month had the highest growth?", conversation_id=conv_id) 4. User: "What products drove that growth?" -5. ask_genie_followup(space_id, conv_id, "What products drove that growth?") +5. ask_genie(space_id, "What products drove that growth?", conversation_id=conv_id) ``` ## Best Practices @@ -181,8 +180,8 @@ result2 = ask_genie(space_id, "How many employees do we have?") # New conversat # Good: Follow-up for related question result1 = ask_genie(space_id, "What were sales last month?") -result2 = ask_genie_followup(space_id, result1["conversation_id"], - "Break that down by product") # Related follow-up +result2 = ask_genie(space_id, "Break that down by product", + conversation_id=result1["conversation_id"]) # Related follow-up ``` ### Handle Clarification Requests @@ -219,7 +218,7 @@ ask_genie(space_id, "Calculate customer lifetime value for all customers", - Verify the `space_id` is correct - Check you have access to the space -- Use `get_genie(space_id)` to verify it exists +- Use `manage_genie(action="get", space_id=...)` to verify it exists ### "Query timed out" diff --git a/.claude/skills/databricks-genie/spaces.md b/.claude/skills/databricks-genie/spaces.md index 8549d6b..ff8acb6 100644 --- a/.claude/skills/databricks-genie/spaces.md +++ b/.claude/skills/databricks-genie/spaces.md @@ -4,12 +4,7 @@ This guide covers creating and managing Genie Spaces for SQL-based data explorat ## What is a Genie Space? -A Genie Space connects to Unity Catalog tables and translates natural language questions into SQL queries. The system: - -1. **Understands** the table schemas and relationships -2. **Generates** SQL queries from natural language -3. **Executes** queries on a SQL warehouse -4. **Presents** results in a conversational format +A Genie Space connects to Unity Catalog tables and translates natural language questions into SQL — understanding schemas, generating queries, executing them on a SQL warehouse, and presenting results conversationally. ## Creation Workflow @@ -18,7 +13,7 @@ A Genie Space connects to Unity Catalog tables and translates natural language q **Before creating a Genie Space, you MUST inspect the table schemas** to understand what data is available: ```python -get_table_details( +get_table_stats_and_schema( catalog="my_catalog", schema="sales", table_stat_level="SIMPLE" @@ -45,7 +40,8 @@ Based on the schema information: Create the space with content tailored to the actual data: ```python -create_or_update_genie( +manage_genie( + action="create_or_update", display_name="Sales Analytics", table_identifiers=[ "my_catalog.sales.customers", @@ -153,17 +149,189 @@ Write sample questions that: ## Updating a Genie Space -To update an existing space: +`manage_genie(action="create_or_update")` handles both create and update automatically. There are two ways it locates an existing space to update: + +- **By `space_id`** (explicit, preferred): pass `space_id=` to target a specific space. +- **By `display_name`** (implicit fallback): if `space_id` is omitted, the tool searches for a space with a matching name and updates it if found; otherwise it creates a new one. + +### Simple field updates (tables, questions, warehouse) + +To update metadata without a serialized config: + +```python +manage_genie( + action="create_or_update", + display_name="Sales Analytics", + space_id="01abc123...", # omit to match by name instead + table_identifiers=[ # updated table list + "my_catalog.sales.customers", + "my_catalog.sales.orders", + "my_catalog.sales.products", + ], + sample_questions=[ # updated sample questions + "What were total sales last month?", + "Who are our top 10 customers by revenue?", + ], + warehouse_id="abc123def456", # omit to keep current / auto-detect + description="Updated description.", +) +``` + +### Full config update via `serialized_space` + +To push a complete serialized configuration to an existing space (the dict contains all regular table metadata, plus it preserves all instructions, SQL examples, join specs, etc.): + +```python +manage_genie( + action="create_or_update", + display_name="Sales Analytics", # overrides title embedded in serialized_space + table_identifiers=[], # ignored when serialized_space is provided + space_id="01abc123...", # target space to overwrite + warehouse_id="abc123def456", # overrides warehouse embedded in serialized_space + description="Updated description.", # overrides description embedded in serialized_space; omit to keep the one in the payload + serialized_space=remapped_config, # JSON string from manage_genie(action="export") (after catalog remap if needed) +) +``` + +> **Note:** When `serialized_space` is provided, `table_identifiers` and `sample_questions` are ignored — the full config comes from the serialized payload. However, `display_name`, `warehouse_id`, and `description` are still applied as top-level overrides on top of the serialized payload. Omit any of them to keep the values embedded in `serialized_space`. + +## Export, Import & Migration + +`manage_genie(action="export")` returns a dictionary with four top-level keys: + +| Key | Description | +|-----|-------------| +| `space_id` | ID of the exported space | +| `title` | Display name of the space | +| `description` | Description of the space | +| `warehouse_id` | SQL warehouse associated with the space (workspace-specific — do **not** reuse across workspaces) | +| `serialized_space` | JSON-encoded string with the full space configuration (see below) | + +This envelope enables cloning, backup, and cross-workspace migration. Use `manage_genie(action="export")` and `manage_genie(action="import")` for all export/import operations — no direct REST calls needed. + +### What is `serialized_space`? + +`serialized_space` is a JSON string (version 2) embedded inside the export envelope. Its top-level keys are: + +| Key | Contents | +|-----|----------| +| `version` | Schema version (currently `2`) | +| `config` | Space-level config: `sample_questions` shown in the UI | +| `data_sources` | `tables` array — each entry has a fully-qualified `identifier` (`catalog.schema.table`) and optional `column_configs` (format assistance, entity matching per column) | +| `instructions` | `example_question_sqls` (certified Q&A pairs), `join_specs` (join relationships between tables), `sql_snippets` (`filters` and `measures` with display names and usage instructions) | +| `benchmarks` | Evaluation Q&A pairs used to measure space quality | + +Catalog names appear **everywhere** inside `serialized_space` — in `data_sources.tables[].identifier`, SQL strings in `example_question_sqls`, `join_specs`, and `sql_snippets`. A single `.replace(src_catalog, tgt_catalog)` on the whole string is sufficient for catalog remapping. + +Minimum structure: +```json +{"version": 2, "data_sources": {"tables": [{"identifier": "catalog.schema.table"}]}} +``` + +### Exporting a Space + +Use `manage_genie(action="export")` to export the full configuration (requires CAN EDIT permission): + +```python +exported = manage_genie(action="export", space_id="01abc123...") +# Returns: +# { +# "space_id": "01abc123...", +# "title": "Sales Analytics", +# "description": "Explore sales data...", +# "warehouse_id": "abc123def456", +# "serialized_space": "{\"version\":2,\"data_sources\":{...},\"instructions\":{...}}" +# } +``` + +You can also get `serialized_space` inline via `manage_genie(action="get")`: + +```python +details = manage_genie(action="get", space_id="01abc123...", include_serialized_space=True) +serialized = details["serialized_space"] +``` + +### Cloning a Space (Same Workspace) + +```python +# Step 1: Export the source space +source = manage_genie(action="export", space_id="01abc123...") + +# Step 2: Import as a new space +manage_genie( + action="import", + warehouse_id=source["warehouse_id"], + serialized_space=source["serialized_space"], + title=source["title"], # override title; omit to keep original + description=source["description"], +) +# Returns: {"space_id": "01def456...", "title": "Sales Analytics (Dev Copy)", "operation": "imported"} +``` + +### Migrating Across Workspaces with Catalog Remapping + +When migrating between environments (e.g. prod → dev), Unity Catalog names are often different. The `serialized_space` string contains the source catalog name **everywhere** — in table identifiers, SQL queries, join specs, and filter snippets. You must remap it before importing. + +**Agent workflow (3 steps):** + +**Step 1 — Export from source workspace:** +```python +exported = manage_genie(action="export", space_id="01f106e1239d14b28d6ab46f9c15e540") +# exported keys: warehouse_id, title, description, serialized_space +# exported["serialized_space"] contains all references to source catalog +``` + +**Step 2 — Remap catalog name in `serialized_space`:** + +The agent does this as an inline string substitution between the two MCP calls: +```python +modified_serialized = exported["serialized_space"].replace( + "source_catalog_name", # e.g. "healthverity_claims_sample_patient_dataset" + "target_catalog_name" # e.g. "healthverity_claims_sample_patient_dataset_dev" +) +``` +This replaces all occurrences — table identifiers, SQL FROM clauses, join specs, and filter snippets. + +**Step 3 — Import to target workspace:** +```python +manage_genie( + action="import", + warehouse_id="", # from manage_warehouse(action="list") on target + serialized_space=modified_serialized, + title=exported["title"], + description=exported["description"] +) +``` + +### Batch Migration of Multiple Spaces + +To migrate several spaces at once, loop through space IDs. The agent exports, remaps the catalog, then imports each: + +``` +For each space_id in [id1, id2, id3]: + 1. exported = manage_genie(action="export", space_id=space_id) + 2. modified = exported["serialized_space"].replace(src_catalog, tgt_catalog) + 3. result = manage_genie(action="import", warehouse_id=wh_id, serialized_space=modified, title=exported["title"], description=exported["description"]) + 4. record result["space_id"] for updating databricks.yml +``` + +After migration, update `databricks.yml` with the new dev `space_id` values under the `dev` target's `genie_space_ids` variable. + +### Updating an Existing Space with New Config + +To push a serialized config to an already-existing space (rather than creating a new one), use `manage_genie(action="create_or_update")` with `space_id=` and `serialized_space=`. The export → remap → push pattern is identical to the migration steps above; just replace `manage_genie(action="import")` with `manage_genie(action="create_or_update", space_id=TARGET_SPACE_ID, ...)` as the final call. -1. **Add/remove tables**: Call `create_or_update_genie` with updated `table_identifiers` -2. **Update questions**: Include new `sample_questions` -3. **Change warehouse**: Provide a different `warehouse_id` +### Permissions Required -The tool finds the existing space by name and updates it. +| Operation | Required Permission | +|-----------|-------------------| +| `manage_genie(action="export")` / `manage_genie(action="get", include_serialized_space=True)` | CAN EDIT on source space | +| `manage_genie(action="import")` | Can create items in target workspace folder | +| `manage_genie(action="create_or_update")` with `serialized_space` (update) | CAN EDIT on target space | ## Example End-to-End Workflow -1. **Generate synthetic data** using `databricks-synthetic-data-generation` skill: +1. **Generate synthetic data** using `databricks-synthetic-data-gen` skill: - Creates parquet files in `/Volumes/catalog/schema/raw_data/` 2. **Create tables** using `databricks-spark-declarative-pipelines` skill: @@ -171,7 +339,7 @@ The tool finds the existing space by name and updates it. 3. **Inspect the tables**: ```python - get_table_details(catalog="catalog", schema="schema") + get_table_stats_and_schema(catalog="catalog", schema="schema") ``` 4. **Create the Genie Space**: @@ -201,3 +369,27 @@ The tool finds the existing space by name and updates it. - Add table and column comments - Include sample questions that demonstrate the vocabulary - Add instructions via the Databricks Genie UI + +### `manage_genie(action="export")` returns empty `serialized_space` + +Requires at least **CAN EDIT** permission on the space. + +### `manage_genie(action="import")` fails with permission error + +Ensure you have CREATE privileges in the target workspace folder. + +### Tables not found after migration + +Catalog name was not remapped — replace the source catalog name in `serialized_space` before calling `manage_genie(action="import")`. The catalog appears in table identifiers, SQL FROM clauses, join specs, and filter snippets; a single `.replace(src_catalog, tgt_catalog)` on the whole string covers all occurrences. + +### `manage_genie` lands in the wrong workspace + +Each MCP server is workspace-scoped. Set up two named MCP server entries (one per profile) in your IDE's MCP config instead of switching a single server's profile mid-session. + +### MCP server doesn't pick up profile change + +The MCP process reads `DATABRICKS_CONFIG_PROFILE` once at startup — editing the config file requires an IDE reload to take effect. + +### `manage_genie(action="import")` fails with JSON parse error + +The `serialized_space` string may contain multi-line SQL arrays with `\n` escape sequences. Flatten SQL arrays to single-line strings before passing to avoid double-escaping issues. diff --git a/.claude/skills/databricks-iceberg/1-managed-iceberg-tables.md b/.claude/skills/databricks-iceberg/1-managed-iceberg-tables.md new file mode 100644 index 0000000..a0f3f06 --- /dev/null +++ b/.claude/skills/databricks-iceberg/1-managed-iceberg-tables.md @@ -0,0 +1,262 @@ +# Managed Iceberg Tables + +Managed Iceberg tables are native Apache Iceberg tables created and stored within Unity Catalog. They support full read/write operations in Databricks and are accessible to external engines via the UC Iceberg REST Catalog (IRC) endpoint. + +**Requirements**: Unity Catalog, DBR 16.4 LTS+ (Managed Iceberg v2), DBR 17.3+ (Managed Iceberg v3 Beta) + +--- + +## Creating Tables + +### Basic DDL + +```sql +-- Create an empty Iceberg table (no clustering) +CREATE TABLE my_catalog.my_schema.events ( + event_id BIGINT, + event_type STRING, + event_date DATE, + payload STRING +) +USING ICEBERG; +``` + +### Create Table As Select (CTAS) + +```sql +-- Create from existing data (no clustering) +CREATE TABLE my_catalog.my_schema.events_archive +USING ICEBERG +AS SELECT * FROM my_catalog.my_schema.events +WHERE event_date < '2025-01-01'; +``` + +### Liquid Clustering + +Managed Iceberg tables use **Liquid Clustering** for data layout optimization. Both `PARTITIONED BY` and `CLUSTER BY` produce a Liquid Clustered table — **no traditional Hive-style partitions are created**. Unity Catalog interprets the partition clause as clustering keys. + +| Syntax | DDL (create table) | Reads via IRC | Iceberg partition fields visible to external engines | DV/row-tracking handling | +|--------|--------------------|---------------|------------------------------------------------------|--------------------------| +| `PARTITIONED BY (col)` | DBR + EMR, OSS Spark, Trino, Flink | Yes | Yes — UC exposes Iceberg partition fields corresponding to clustering keys; external engines can prune | **Auto-handled** | +| `CLUSTER BY (col)` | DBR only | Yes | Yes — same; UC maintains Iceberg partition spec from clustering keys regardless of DDL used | Manual on v2, auto on v3 | + +> **Both syntaxes produce the same Iceberg metadata for external engines.** UC maintains an Iceberg partition spec (partition fields corresponding to the clustering keys) that external engines read via IRC. This is Iceberg-style partitioning — not legacy Hive-style directory partitions. External engines see a partitioned Iceberg table and benefit from partition pruning. Internally, UC uses those partition fields as liquid clustering keys. + +> **`PARTITIONED BY` limitation**: Only plain column references are supported. Expression transforms (`bucket()`, `years()`, `months()`, `days()`, `hours()`) are **not** supported and will error. + +> **`CLUSTER BY` on Iceberg v2**: requires explicitly setting `'delta.enableDeletionVectors' = false` and `'delta.enableRowTracking' = false`, otherwise you get: `[MANAGED_ICEBERG_ATTEMPTED_TO_ENABLE_CLUSTERING_WITHOUT_DISABLING_DVS_OR_ROW_TRACKING]` + +**`PARTITIONED BY` — recommended for cross-platform** (auto-handles all required properties): + +```sql +-- Single column (v2 or v3 — no TBLPROPERTIES needed) +CREATE TABLE orders ( + order_id BIGINT, + order_date DATE +) +USING ICEBERG +PARTITIONED BY (order_date); + +-- Multi-column +CREATE TABLE orders ( + order_id BIGINT, + region STRING, + order_date DATE +) +USING ICEBERG +PARTITIONED BY (region, order_date); +``` + +**`CLUSTER BY` on Iceberg v2** (DBR-only; must disable DVs and row tracking manually): + +```sql +-- Single column clustering (v2) +CREATE TABLE orders ( + order_id BIGINT, + order_date DATE +) +USING ICEBERG +TBLPROPERTIES ( + 'delta.enableDeletionVectors' = false, + 'delta.enableRowTracking' = false +) +CLUSTER BY (order_date); +``` + +**`CLUSTER BY` on Iceberg v3** (no extra TBLPROPERTIES needed): + +```sql +CREATE TABLE orders ( + order_id BIGINT, + order_date DATE +) +USING ICEBERG +TBLPROPERTIES ('format-version' = '3') +CLUSTER BY (order_date); +``` + +--- + +## DML Operations + +Managed Iceberg tables support all standard DML operations: + +```sql +-- INSERT +INSERT INTO my_catalog.my_schema.events +VALUES (1, 'click', '2025-06-01', '{"page": "home"}'); + +-- INSERT from query +INSERT INTO my_catalog.my_schema.events +SELECT * FROM staging_events WHERE event_date = current_date(); + +-- UPDATE +UPDATE my_catalog.my_schema.events +SET event_type = 'page_view' +WHERE event_id = 1; + +-- DELETE +DELETE FROM my_catalog.my_schema.events +WHERE event_date < '2024-01-01'; + +-- MERGE (upsert) +MERGE INTO my_catalog.my_schema.events AS target +USING staging_events AS source +ON target.event_id = source.event_id +WHEN MATCHED THEN UPDATE SET * +WHEN NOT MATCHED THEN INSERT *; +``` + +--- + +## Time Travel + +Query historical snapshots using timestamp or snapshot ID: + +```sql +-- Query by timestamp +SELECT * FROM my_catalog.my_schema.events TIMESTAMP AS OF '2025-06-01T00:00:00Z'; + +-- Query by snapshot ID +SELECT * FROM my_catalog.my_schema.events VERSION AS OF 1234567890; + +-- Only for external engines: View snapshot history +SELECT * FROM my_catalog.my_schema.events.snapshots; +``` + +--- + +## Predictive Optimization + +Predictive Optimization is **recommended** for managed Iceberg tables — it is not auto-enabled and must be turned on explicitly. Once enabled, it automatically runs: + +- **Compaction** — consolidates small files +- **Vacuum** — removes expired snapshots and orphan files +- **Statistics collection** — keeps column statistics up to date for query optimization + +Enable at the catalog or schema level. Manual operations are still available if needed: + +```sql +-- Manual compaction +OPTIMIZE my_catalog.my_schema.events; + +-- Manual vacuum +VACUUM my_catalog.my_schema.events; + +-- Manual statistics collection +ANALYZE TABLE my_catalog.my_schema.events COMPUTE STATISTICS FOR ALL COLUMNS; +``` + +--- + +## Iceberg v3 (Beta) + +**Requires**: DBR 17.3+ + +Iceberg v3 introduces new capabilities on top of v2: + +| Feature | Description | +|---------|-------------| +| **Deletion Vectors** | Row-level deletes without rewriting data files — faster UPDATE/DELETE/MERGE | +| **VARIANT Type** | Semi-structured data column (like Delta's VARIANT) | +| **Row Lineage** | Track row-level provenance across transformations | + +### Creating an Iceberg v3 Table + +```sql +CREATE TABLE my_catalog.my_schema.events_v3 ( + event_id BIGINT, + event_date DATE, + data VARIANT +) +USING ICEBERG +TBLPROPERTIES ('format-version' = '3') +CLUSTER BY (event_date); +``` + +### Important Notes + +- **Cannot downgrade**: Once a table is upgraded to v3, it cannot be downgraded back to v2 +- **External engine compatibility**: External engines must use Iceberg library 1.9.0+ to read v3 tables +- **Deletion vectors**: Enabled by default on v3 tables. External readers must support deletion vectors +- **Beta status**: Iceberg v3 is in Beta — not recommended for production workloads yet + +### Upgrading an Existing Table to v3 + +```sql +ALTER TABLE my_catalog.my_schema.events +SET TBLPROPERTIES ('format-version' = '3'); +``` + +> **Warning**: This is irreversible. Test with non-production data first. + +--- + +## Limitations + +| Limitation | Details | +|------------|---------| +| **No Vector Search** | Vector Search indexes are not supported on Iceberg tables | +| **No Change Data Feed (CDF)** | CDF is a Delta-only feature; use Delta + UniForm if CDF is required | +| **Parquet only** | Iceberg tables on Databricks use Parquet as the underlying file format | +| **No shallow clone** | `SHALLOW CLONE` is not supported; use `DEEP CLONE` or CTAS | +| **`PARTITIONED BY` maps to Liquid Clustering** | `PARTITIONED BY` is supported and recommended for cross-platform scenarios — it maps to Liquid Clustering, not traditional partitions. Only plain column references work; expression transforms (`bucket()`, `years()`, etc.) are not supported. | +| **No Structured Streaming sink** | Cannot use `writeStream` to write to Iceberg tables directly; use `INSERT INTO` or `MERGE` in batch or SDP | +| **Compression** | Default compression is `zstd`; older readers may need `snappy` — set `write.parquet.compression-codec` if needed | +| **Do not set metadata path** | Never set `write.metadata.path` or `write.metadata.previous-versions-max` | +| **Do not install Iceberg library** | DBR includes built-in support; installing an Iceberg JAR causes conflicts | + +--- + +## Converting From Other Formats + +### Delta to Iceberg (via DEEP CLONE) + +```sql +CREATE TABLE my_catalog.my_schema.events_iceberg +USING ICEBERG +DEEP CLONE my_catalog.my_schema.events_delta; +``` + +### Foreign Iceberg to Managed Iceberg + +```sql +-- With Liquid Clustering (v2 — must disable DVs and row tracking) +CREATE TABLE my_catalog.my_schema.events_managed +USING ICEBERG +TBLPROPERTIES ( + 'delta.enableDeletionVectors' = false, + 'delta.enableRowTracking' = false +) +CLUSTER BY (event_date) +AS SELECT * FROM foreign_catalog.foreign_schema.events; + +-- With Liquid Clustering (v3 — no extra TBLPROPERTIES needed) +CREATE TABLE my_catalog.my_schema.events_managed +USING ICEBERG +TBLPROPERTIES ('format-version' = '3') +CLUSTER BY (event_date) +AS SELECT * FROM foreign_catalog.foreign_schema.events; +``` + + diff --git a/.claude/skills/databricks-iceberg/2-uniform-and-compatibility.md b/.claude/skills/databricks-iceberg/2-uniform-and-compatibility.md new file mode 100644 index 0000000..8437a72 --- /dev/null +++ b/.claude/skills/databricks-iceberg/2-uniform-and-compatibility.md @@ -0,0 +1,207 @@ +# UniForm and Compatibility Mode + +UniForm and Compatibility Mode make Delta tables readable as Iceberg by external engines — without converting to a native Iceberg table. Data is written as Delta, but Iceberg metadata is generated automatically so external tools (Snowflake, PyIceberg, Spark, Trino) can read via UC IRC endpoint. + +--- + +## External Iceberg Reads (fka UniForm) (GA) + +**Requirements**: Unity Catalog, DBR 14.3+, column mapping enabled, deletion vectors disabled, the Delta table must have a minReaderVersion >= 2 and minWriterVersion >= 7, both managed and external tables supported. + +UniForm adds automatic Iceberg metadata generation to regular Delta tables. The table remains Delta internally but is readable as Iceberg externally. + +### Enabling UniForm on a New Table + +```sql +CREATE TABLE my_catalog.my_schema.customers ( + customer_id BIGINT, + name STRING, + region STRING, + updated_at TIMESTAMP +) +TBLPROPERTIES ( + 'delta.columnMapping.mode' = 'name', + 'delta.enableIcebergCompatV2' = 'true', + 'delta.universalFormat.enabledFormats' = 'iceberg' +); +``` + +### Enabling UniForm on an Existing Table + +```sql +ALTER TABLE my_catalog.my_schema.customers +SET TBLPROPERTIES ( + 'delta.columnMapping.mode' = 'name', + 'delta.enableIcebergCompatV2' = 'true', + 'delta.universalFormat.enabledFormats' = 'iceberg' +); +``` + +### Requirements and Prerequisites + +UniForm requires the following properties to be set explicitly: + +| Requirement | Details | +|-------------|---------| +| **Unity Catalog** | Table must be registered in UC | +| **DBR 14.3+** | Minimum runtime version | +| **Deletion vectors disabled** | Set `delta.enableDeletionVectors = false` before enabling UniForm | +| **No column mapping conflicts** | If table uses `id` mode, migrate to `name` mode first | + +If deletion vectors are currently enabled: + +```sql +-- Disable deletion vectors first +ALTER TABLE my_catalog.my_schema.customers +SET TBLPROPERTIES ('delta.enableDeletionVectors' = 'false'); + +-- Rewrite to remove existing deletion vectors +REORG TABLE my_catalog.my_schema.customers +APPLY (PURGE); + +-- Then enable UniForm +ALTER TABLE my_catalog.my_schema.customers +SET TBLPROPERTIES ( + 'delta.columnMapping.mode' = 'name', + 'delta.enableIcebergCompatV2' = 'true', + 'delta.universalFormat.enabledFormats' = 'iceberg' +); +``` + +### Async Metadata Generation + +Iceberg metadata is generated **asynchronously** after each Delta transaction. There is a brief delay (typically seconds, occasionally minutes for large transactions) before external engines see the latest data. + +### Checking UniForm Status + +> See [Check Iceberg metadata generation status](https://docs.databricks.com/aws/en/delta/uniform#check-iceberg-metadata-generation-status) for full details. + + +### Disabling UniForm + +```sql +ALTER TABLE my_catalog.my_schema.customers +UNSET TBLPROPERTIES ('delta.universalFormat.enabledFormats'); +``` + +--- + +## Compatibility Mode + +**Requirements**: Unity Catalog, DBR 16.1+, SDP pipeline + +Compatibility Mode extends UniForm to **streaming tables (STs)** and **materialized views (MVs)** created by Spark Declarative Pipelines (SDP) or DBSQL. Regular UniForm does not work on STs/MVs — Compatibility Mode is the only option. + +**How it works**: When you enable Compatibility Mode, Databricks creates a separate, read-only **"compatibility version"** of the object at the external location you specify (`delta.universalFormat.compatibility.location`). This is a full copy of the data in Iceberg-compatible format — not a pointer to the original Delta data. After the initial full copy, subsequent metadata and data generation is **incremental** (only new/changed data is synced to the external location). + +> **Storage cost consideration**: Because Compatibility Mode writes a separate copy of the data to the external location, you incur additional cloud storage costs proportional to the size of the table. Factor this in when enabling Compatibility Mode on large tables. + +### Enabling Compatibility Mode + +Compatibility Mode is configured via table properties: + +**SQL Example (streaming table)**: + +```sql +CREATE OR REFRESH STREAMING TABLE my_events +TBLPROPERTIES ( + 'delta.universalFormat.enabledFormats' = 'compatibility', + 'delta.universalFormat.compatibility.location' = '' +) +AS SELECT * FROM STREAM read_files('/Volumes/catalog/schema/raw/events/'); +``` + +**SQL Example (materialized view)**: + +```sql +CREATE OR REFRESH MATERIALIZED VIEW daily_summary +TBLPROPERTIES ( + 'delta.universalFormat.enabledFormats' = 'compatibility', + 'delta.universalFormat.compatibility.location' = '' +) +AS SELECT event_date, COUNT(*) AS event_count +FROM my_events +GROUP BY event_date; +``` + +**Python Example**: + +```python +from pyspark import pipelines as dp + +@dp.table( + name="my_events", + table_properties={ + "delta.universalFormat.enabledFormats": "compatibility", + "delta.universalFormat.compatibility.location": "", + }, +) +def my_events(): + return ( + spark.readStream.format("cloudFiles") + .option("cloudFiles.format", "json") + .load("/Volumes/catalog/schema/raw/events/") + ) +``` + +### Considerations for Compatibility Mode + +| Consideration | Details | +|---------------|---------| +| **External location** | `delta.universalFormat.compatibility.location` must point to a configured external location for the Iceberg metadata output path | +| **SDP pipeline only** | Only works with streaming tables and MVs defined in SDP pipelines | +| **Initial generation time** | First metadata generation can take up to 1 hour for large tables | +| **Unity Catalog** | Required | +| **DBR 16.1+** | Minimum runtime for the SDP pipeline | + +### Refresh Mechanics + +Compatibility Mode metadata can be refreshed manually or controlled via the `delta.universalFormat.compatibility.targetRefreshInterval` property: + +```sql +CREATE OR REFRESH STREAMING TABLE my_events +TBLPROPERTIES ( + 'delta.universalFormat.enabledFormats' = 'compatibility', + 'delta.universalFormat.compatibility.location' = '', + 'delta.universalFormat.compatibility.targetRefreshInterval' = '0 MINUTES' +) +AS SELECT * FROM STREAM read_files('/Volumes/catalog/schema/raw/events/'); +``` + +| Interval value | Behavior | +|----------------|----------| +| `0 MINUTES` | Checks for changes after every commit and triggers a refresh if needed — default for streaming tables and MVs | +| `1 HOUR` | Default for non-SDP tables; refreshes at most once per hour | +| Values below `1 HOUR` (e.g. `30 MINUTES`) | Not recommended — won't make refreshes more frequent than once per hour | + +Metadata can also be triggered manually: + +```sql +REFRESH TABLE my_catalog.my_schema.my_events; +``` + +### Future Modes + +A more efficient mode for streaming tables and materialized views is expected in a future release. + +--- + +## Decision Table: Which Approach? + +| Criteria | Managed Iceberg | UniForm | Compatibility Mode | +|----------|:-:|:-:|:-:| +| **Full Iceberg read/write** | Yes | Read-only (as Iceberg) | Read-only (as Iceberg) | +| **Works with Delta features (CDF)** | No | Partial* | Partial* | +| **Streaming tables / MVs** | No | No | Yes | +| **External engine write via IRC** | Yes | No | No | +| **Existing Delta investment** | Requires migration | No migration | No migration | +| **Predictive Optimization** | Auto-enabled | Auto-enabled (Delta) | Auto-enabled (Delta) | +| **DBR requirement** | 16.1+ | 14.3+ | 16.1+ | + +*given that Iceberg doesn't have CDF so the features dependent on it are not supported e.g., +streaming tables, materialized views, data classification, vector search, data profiling. For Synced tables to Lakebase, only snapshot mode is supported. +### When to Choose Each + +- **Managed Iceberg**: You want a native Iceberg table with full read/write from both Databricks and external engines. You don't need Delta-specific features (e.g., CDF). +- **UniForm**: You have existing Delta tables and want to make them readable as Iceberg by external engines without migrating. You want to keep Delta features internally. +- **Compatibility Mode**: You have streaming tables or materialized views that need to be readable as Iceberg by external engines. diff --git a/.claude/skills/databricks-iceberg/3-iceberg-rest-catalog.md b/.claude/skills/databricks-iceberg/3-iceberg-rest-catalog.md new file mode 100644 index 0000000..e7cf571 --- /dev/null +++ b/.claude/skills/databricks-iceberg/3-iceberg-rest-catalog.md @@ -0,0 +1,107 @@ +# Iceberg REST Catalog (IRC) + +The Iceberg REST Catalog (IRC) is a REST API endpoint that lets external engines read and write Databricks-managed Iceberg data using the standard Apache Iceberg REST Catalog protocol. External tools connect to the IRC endpoint, authenticate, and receive vended credentials for direct cloud storage access. + +**Endpoint**: `https:///api/2.1/unity-catalog/iceberg-rest` + +> **Legacy endpoint warning**: The older `/api/2.1/unity-catalog/iceberg` endpoint is in maintenance mode and should not be used for new integrations. It was the original read-only endpoint documented for UniForm. All new integrations — both UniForm (Delta with Iceberg reads) and managed Iceberg tables — must use `/api/2.1/unity-catalog/iceberg-rest`. + +**Requirements**: Unity Catalog, external data access enabled on the workspace, DBR 16.1+ + +--- + +## Prerequisites + +### 1. Enable External Data Access + +External data access must be enabled for your workspace. This is typically configured by a workspace admin. + +### 2. Network Access to the IRC Endpoint + +External engines must reach the Databricks workspace over HTTPS (port 443). If the workspace has **IP access lists** enabled, the CIDR range(s) of the Iceberg client must be explicitly allowed — otherwise connections will fail regardless of correct credentials or grants. + +Check and manage IP access lists: +- Admin console: **Settings → Security → IP access list** +- REST API: `GET /api/2.0/ip-access-lists` to inspect, `POST /api/2.0/ip-access-lists` to add ranges + +> **Common symptom**: Connections time out or return `403 Forbidden` even with valid credentials and correct grants. IP access list misconfiguration is a frequent root cause — check this before debugging auth. + +### 3. Grant EXTERNAL USE SCHEMA + +The connecting principal (user or service principal) must have the `EXTERNAL USE SCHEMA` grant on each schema they want to access: + +```sql +-- Grant to a user +GRANT EXTERNAL USE SCHEMA ON SCHEMA my_catalog.my_schema TO `user@example.com`; + +-- Grant to a service principal +GRANT EXTERNAL USE SCHEMA ON SCHEMA my_catalog.my_schema TO `my-service-principal`; + +-- Grant to a group +GRANT EXTERNAL USE SCHEMA ON SCHEMA my_catalog.my_schema TO `data-engineers`; +``` + +> **Important**: `EXTERNAL USE SCHEMA` is separate from `SELECT` or `MODIFY` grants. A user needs both data permissions AND the external use grant. + +--- + +## Authentication + +### Personal Access Token (PAT) + +``` +Authorization: Bearer +``` + +### OAuth (M2M) + +For service-to-service authentication, use OAuth with a service principal: + +1. Create a service principal in the Databricks account +2. Generate an OAuth secret +3. Use the OAuth token endpoint to get an access token +4. Pass the access token as a Bearer token + +--- + +## Read/Write Capability Matrix + +| Table Type | IRC Read | IRC Write | +|------------|:-:|:-:| +| Managed Iceberg (`USING ICEBERG`) | Yes | Yes | +| Delta + UniForm | Yes | No | +| Delta + Compatibility Mode | Yes | No | +| Foreign Iceberg Table | No | No | + +> **Key insight**: Only managed Iceberg tables support writes via IRC. UniForm and Compatibility Mode tables are read-only because the underlying format is Delta. + +--- + +## Credential Vending + +When an external engine connects via IRC, Databricks **vends temporary cloud credentials** (short-lived STS tokens for AWS, SAS tokens for Azure) so the engine can read/write data files directly in cloud storage. This is transparent to the client — the IRC protocol handles it automatically. + +Benefits: +- No need to configure cloud credentials in the external engine +- Credentials are scoped to the specific table and operation +- Credentials automatically expire (typically 1 hour) + +--- + +## Common Configuration Reference + +| Parameter | Value | +|-----------|-------| +| **Catalog type** | `rest` | +| **URI** | `https:///api/2.1/unity-catalog/iceberg-rest` | +| **Warehouse** | Unity Catalog catalog name (e.g., `my_catalog`) | +| **Token** | Databricks PAT or OAuth access token | +| **Credential vending** | Automatic (handled by the REST protocol) | + + +--- + +## Related + +- [4-snowflake-interop.md](4-snowflake-interop.md) — Snowflake reading Databricks via catalog integration (uses IRC) +- [5-external-engine-interop.md](5-external-engine-interop.md) — Per-engine connection configs: PyIceberg, OSS Spark, EMR, Flink, Kafka Connect, DuckDB, Trino diff --git a/.claude/skills/databricks-iceberg/4-snowflake-interop.md b/.claude/skills/databricks-iceberg/4-snowflake-interop.md new file mode 100644 index 0000000..2f9d953 --- /dev/null +++ b/.claude/skills/databricks-iceberg/4-snowflake-interop.md @@ -0,0 +1,349 @@ +# Snowflake Interoperability + +Databricks and Snowflake can share Iceberg data bidirectionally. This file covers both directions: Snowflake reading Databricks-managed tables, and Databricks reading Snowflake-managed Iceberg tables. + +**Cloud scope**: AWS-primary examples. Azure/GCS differences noted where relevant. + +--- + +## Direction 1: Snowflake Reading Databricks + +Snowflake can read Databricks-managed Iceberg tables (managed Iceberg + UniForm + Compatibility Mode) through a **Catalog Integration** that connects to the Databricks Iceberg REST Catalog (IRC). + +### Step 1: Create a Catalog Integration in Snowflake + +`ACCESS_DELEGATION_MODE = VENDED_CREDENTIALS` is required on AWS for Snowflake to receive temporary STS credentials from the Databricks IRC. Without it, Snowflake cannot access the underlying Parquet files. + +**PAT / Bearer token**: + +```sql +-- In Snowflake +CREATE OR REPLACE CATALOG INTEGRATION databricks_catalog_int + CATALOG_SOURCE = ICEBERG_REST + TABLE_FORMAT = ICEBERG + CATALOG_NAMESPACE = 'my_schema' -- UC schema (default namespace) + REST_CONFIG = ( + CATALOG_URI = 'https:///api/2.1/unity-catalog/iceberg-rest' + WAREHOUSE = '' -- UC catalog name + ACCESS_DELEGATION_MODE = VENDED_CREDENTIALS + ) + REST_AUTHENTICATION = ( + TYPE = BEARER + BEARER_TOKEN = '' + ) + REFRESH_INTERVAL_SECONDS = 300 + ENABLED = TRUE; +``` + +**OAuth (recommended for production)**: + +```sql +CREATE OR REPLACE CATALOG INTEGRATION databricks_catalog_int + CATALOG_SOURCE = ICEBERG_REST + TABLE_FORMAT = ICEBERG + CATALOG_NAMESPACE = 'my_schema' + REST_CONFIG = ( + CATALOG_URI = 'https:///api/2.1/unity-catalog/iceberg-rest' + WAREHOUSE = '' + ACCESS_DELEGATION_MODE = VENDED_CREDENTIALS + ) + REST_AUTHENTICATION = ( + TYPE = OAUTH + OAUTH_CLIENT_ID = '' + OAUTH_CLIENT_SECRET = '' + OAUTH_TOKEN_URI = 'https:///oidc/v1/token' + OAUTH_ALLOWED_SCOPES = ('all-apis', 'sql') + ) + REFRESH_INTERVAL_SECONDS = 300 + ENABLED = TRUE; +``` + +> **Grant on the Databricks side**: The principal used for authentication needs these privileges in Unity Catalog: +> - `USE CATALOG` on the catalog +> - `USE SCHEMA` on the schema +> - `EXTERNAL USE SCHEMA` on the schema — this is the key privilege that enables external engines to access tables via IRC +> - `SELECT` on the target tables (or schema/catalog for broader access) +> +> Missing `EXTERNAL USE SCHEMA` causes a `Failed to retrieve credentials` error in Snowflake. + +### Step 2: External Volume (Azure/GCS Only) + +On **AWS with vended credentials**, no external volume is needed — Databricks IRC vends temporary STS credentials automatically. + +On **Azure** or **GCS**, you must create an external volume in Snowflake because vended credentials are not supported for those clouds: + +```sql +-- Azure example (in Snowflake) +CREATE OR REPLACE EXTERNAL VOLUME databricks_ext_vol + STORAGE_LOCATIONS = ( + ( + NAME = 'azure_location' + STORAGE_BASE_URL = 'azure://myaccount.blob.core.windows.net/my-container/iceberg/' + AZURE_TENANT_ID = '' + ) + ); +``` + +### Step 3: Expose Tables in Snowflake + +Two approaches available. **Linked catalog** is preferred — it exposes all tables in the namespace at once and updates automatically. + +**Option A: Linked Catalog Database (preferred)** + +```sql +-- Verify namespaces are visible (should return your UC schemas) +SELECT SYSTEM$LIST_NAMESPACES_FROM_CATALOG('databricks_catalog_int', '', 0); + +-- Create a linked catalog database exposing all tables in the namespace +CREATE DATABASE my_snowflake_db + LINKED_CATALOG = ( + CATALOG = 'databricks_catalog_int', + ALLOWED_NAMESPACES = ('my_schema') -- UC schema + ); + +-- Check link health (executionState should be "RUNNING" with empty failureDetails) +SELECT SYSTEM$CATALOG_LINK_STATUS('my_snowflake_db'); + +-- Query +SELECT * FROM my_snowflake_db."my_schema"."my_table" +WHERE event_date >= '2025-01-01'; +``` + +**Option B: Individual Table Reference (legacy)** + +```sql +-- AWS (vended creds — no EXTERNAL_VOLUME needed) +CREATE ICEBERG TABLE my_snowflake_db.my_schema.events + CATALOG = 'databricks_catalog_int' + CATALOG_TABLE_NAME = 'events'; + +-- Azure/GCS (EXTERNAL_VOLUME required) +CREATE ICEBERG TABLE my_snowflake_db.my_schema.events + CATALOG = 'databricks_catalog_int' + CATALOG_TABLE_NAME = 'events' + EXTERNAL_VOLUME = 'databricks_ext_vol'; + +-- Query +SELECT * FROM my_snowflake_db.my_schema.events +WHERE event_date >= '2025-01-01'; +``` + +### Key Gotchas + +#### Workspace IP Access Lists Must Allow Snowflake Egress IPs + +If the Databricks workspace has **IP access lists** enabled, Snowflake's outbound NAT IPs must be added to the allowlist. Snowflake connects to the Databricks IRC endpoint (`/api/2.1/unity-catalog/iceberg-rest`) over HTTPS (port 443), and a blocked IP produces connection timeouts or `403` errors that can look like auth failures. + + +> **Diagnosis tip**: If the catalog integration shows `ENABLED = TRUE` but `SYSTEM$CATALOG_LINK_STATUS` returns a connection error (not a credentials error), IP access lists are the first thing to check. + +#### REFRESH_INTERVAL_SECONDS Is Per-Integration, Not Per-Table + +The `REFRESH_INTERVAL_SECONDS` setting on the catalog integration controls how often Snowflake polls the Databricks IRC for metadata changes. This applies to **all tables** using that integration — you cannot set different refresh intervals per table. + +- Lower values = fresher data but more API calls +- Default: 300 seconds (5 minutes) +- Minimum: 60 seconds + +#### 1000-Commit Limit + +For Iceberg tables created from Delta files in object storage, Snowflake processes a maximum of 1000 Delta commit files each time you refresh a table using CREATE/ALTER ICEBERG TABLE … REFRESH or an automatic refresh; if the table has more than 1000 commit files since the last checkpoint, you can perform additional refreshes and each refresh continues from where the previous one stopped. The 1000‑commit limit applies only to Delta commit files after the latest Delta checkpoint file, and does not limit how many commits the catalog integration can ultimately synchronize over multiple refreshes + +**Mitigations**: +- Enable Predictive Optimization (auto-compaction reduces commit frequency) +- Batch writes instead of high-frequency micro-batches +- Run `OPTIMIZE` and `VACUUM` to consolidate metadata manually if needed. + +--- + +## Direction 2: Databricks Reading Snowflake + +Databricks can read Snowflake-managed Iceberg tables through a **foreign catalog** that connects to Snowflake's Iceberg catalog. Snowflake Iceberg tables are stored in external volumes (cloud storage), so Databricks reads the Iceberg's Parquet files directly — no Snowflake compute required. + +**Assumption**: A Snowflake-managed Iceberg table already exists, created with `CATALOG = 'SNOWFLAKE'` pointing to an external volume: + +```sql +-- In Snowflake — prerequisite table +CREATE ICEBERG TABLE sensor_readings ( + device_id INT, + device_value STRING +) + CATALOG = 'SNOWFLAKE' + EXTERNAL_VOLUME = 'ICEBERG_SHARED_VOL' + BASE_LOCATION = 'sensor_readings/'; + +INSERT INTO sensor_readings VALUES (1, 'value01'), (2, 'value02'); + +SELECT * FROM sensor_readings; +``` + +`CATALOG = 'SNOWFLAKE'` means Snowflake manages the Iceberg metadata. The data files land in the external volume at the `BASE_LOCATION` sub-path. The steps below set up Databricks to read this table. + +### Step 1: Find Snowflake External Volume Path + +Before setting up the Databricks side, run this in Snowflake to get the S3/ADLS/GCS path where Snowflake stores its Iceberg data. You'll need this path for Steps 2 and 4. + +```sql +-- In Snowflake +DESCRIBE EXTERNAL VOLUME ; +-- Note the STORAGE_BASE_URL value (e.g. s3://my-bucket/snowflake-iceberg/) +``` + +### Step 2: Create a Storage Credential + +Create a storage credential for the cloud storage where Snowflake stores its Iceberg data. Assuming that the IAM role already exists. Follow the documentation for details (https://docs.databricks.com/aws/en/connect/unity-catalog/cloud-storage/s3/s3-external-location-manual) + +```bash +# In Databricks CLI (AWS example) +databricks storage-credentials create snowflake_storage_cred \ + --aws-iam-role-arn "arn:aws:iam::123456789012:role/snowflake-data-access" +``` + +### Step 3: Create an External Location + +The external location must point to the **root** of the bucket (not a sub-path), so that all Snowflake external volume paths fall under it. + +> **Fallback mode**: You do not need this external-location fallback enabled to read Snowflake‑created Iceberg tables via catalog federation. It only affects how storage credentials are resolved for paths, not whether Snowflake Iceberg federation works. + +```sql +-- In Databricks (URL should be the bucket root, not a sub-path) +CREATE EXTERNAL LOCATION snowflake_data +URL 's3://snowflake-iceberg-bucket/' +WITH (CREDENTIAL snowflake_storage_cred); +``` + +### Step 4: Create a Snowflake Connection + +```sql +-- In Databricks +CREATE CONNECTION snowflake_conn +TYPE SNOWFLAKE +OPTIONS ( + 'host' = '.snowflakecomputing.com', + 'user' = '', + 'password' = '', + 'sfWarehouse' = '' +); +``` + +### Step 5: Create a Foreign Catalog + +Two mandatory fields beyond `database`: + +- **`authorized_paths`**: The path(s) where Snowflake stores Iceberg table files — from `STORAGE_BASE_URL` in `DESCRIBE EXTERNAL VOLUME`. Databricks can only read Iceberg tables whose data falls under these paths. +- **`storage_root`**: Where Databricks stores catalog metadata for Iceberg reads. Must point to an existing external location. This is required — the foreign catalog creation will fail without it. + +```sql +-- In Databricks +CREATE FOREIGN CATALOG snowflake_iceberg +USING CONNECTION snowflake_conn +OPTIONS ( + 'catalog' = '', + 'authorized_paths' = 's3://snowflake-iceberg-bucket/snowflake-iceberg/', + 'storage_root' = 's3://snowflake-iceberg-bucket/uc-metadata/' +); +``` + +> **UI workflow note**: The Databricks connection wizard (Catalog Explorer → Add connection → Snowflake) will prompt for authorized paths and storage location in the form and create the foreign catalog automatically. The SQL above is the equivalent DDL. + +### Step 6: Refresh, Verify, and Query + +```sql +-- Refresh to discover tables +REFRESH FOREIGN CATALOG snowflake_iceberg; + +-- Verify provider type before querying at scale: +-- Provider = Iceberg → Databricks reads directly from cloud storage (cheap) +-- Provider = Snowflake → double compute via JDBC (Snowflake + Databricks) +DESCRIBE EXTENDED snowflake_iceberg.my_schema.my_table; + +-- Query +SELECT * FROM snowflake_iceberg.my_schema.my_table +WHERE created_at >= '2025-01-01'; +``` + +### Compute Cost Matrix + +| Snowflake Table Type | Databricks Read | Compute Cost | +|---------------------|:-:|---| +| **Snowflake Iceberg table** | Yes | Databricks compute only (reads data files directly from cloud storage) | +| **Snowflake native table** | Yes (via federation) | Double compute — Snowflake runs the query, Databricks processes the result | + +> **Key insight**: Snowflake Iceberg tables are more cost-efficient to read from Databricks because Databricks reads the Parquet files directly. Native Snowflake tables require Snowflake to run the scan. + + +--- + +## Full AWS Example: Snowflake Reading Databricks + +```sql +-- ======================================== +-- DATABRICKS SIDE (run in Databricks) +-- ======================================== + +-- 1. Create a managed Iceberg table (v2 — disable DVs and row tracking for CLUSTER BY) +CREATE TABLE main.sales.orders ( + order_id BIGINT, + customer_id BIGINT, + amount DECIMAL(10,2), + order_date DATE +) +USING ICEBERG +TBLPROPERTIES ( + 'delta.enableDeletionVectors' = false, + 'delta.enableRowTracking' = false +) +CLUSTER BY (order_date); + +-- 2. Grant external access to the service principal used in Snowflake catalog integration +GRANT EXTERNAL USE SCHEMA ON SCHEMA main.sales TO `snowflake-service-principal`; + +-- ======================================== +-- SNOWFLAKE SIDE (run in Snowflake) +-- ======================================== + +-- 3. Create catalog integration (ACCESS_DELEGATION_MODE required for vended creds on AWS) +CREATE OR REPLACE CATALOG INTEGRATION databricks_int + CATALOG_SOURCE = ICEBERG_REST + TABLE_FORMAT = ICEBERG + CATALOG_NAMESPACE = 'sales' + REST_CONFIG = ( + CATALOG_URI = 'https://my-workspace.cloud.databricks.com/api/2.1/unity-catalog/iceberg-rest' + WAREHOUSE = 'main' + ACCESS_DELEGATION_MODE = VENDED_CREDENTIALS + ) + REST_AUTHENTICATION = ( + TYPE = OAUTH + OAUTH_CLIENT_ID = '' + OAUTH_CLIENT_SECRET = '' + OAUTH_TOKEN_URI = 'https://my-workspace.cloud.databricks.com/oidc/v1/token' + OAUTH_ALLOWED_SCOPES = ('all-apis', 'sql') + ) + REFRESH_INTERVAL_SECONDS = 300 + ENABLED = TRUE; + +-- 4. Verify schemas are visible +SELECT SYSTEM$LIST_NAMESPACES_FROM_CATALOG('databricks_int', '', 0); + +-- 5. Create linked catalog database (exposes all tables in the namespace) +CREATE DATABASE analytics + LINKED_CATALOG = ( + CATALOG = 'databricks_int', + ALLOWED_NAMESPACES = ('sales') + ); + +-- 6. Check link health +SELECT SYSTEM$CATALOG_LINK_STATUS('analytics'); + +-- 7. Query (schema and table names are case-sensitive) +SELECT order_date, SUM(amount) AS daily_revenue +FROM analytics."sales"."orders" +GROUP BY order_date +ORDER BY order_date DESC; +``` + +--- + +## Related + +- [3-iceberg-rest-catalog.md](3-iceberg-rest-catalog.md) — IRC endpoint details and authentication diff --git a/.claude/skills/databricks-iceberg/5-external-engine-interop.md b/.claude/skills/databricks-iceberg/5-external-engine-interop.md new file mode 100644 index 0000000..ecafcbe --- /dev/null +++ b/.claude/skills/databricks-iceberg/5-external-engine-interop.md @@ -0,0 +1,206 @@ +# External Engine Interoperability + +This file covers connecting external engines to Databricks via the Iceberg REST Catalog (IRC). Each engine section includes the minimum configuration needed to read (and where supported, write) Databricks-managed Iceberg tables. + +**Prerequisites for all engines**: +- Databricks workspace with external data access enabled +- `EXTERNAL USE SCHEMA` granted on target schemas +- PAT or OAuth (service principal) credentials for authentication with the required permissions. +- **Network access**: The client must reach the Databricks workspace on HTTPS (port 443). If workspace **IP access lists** are enabled, add the client's egress CIDR to the allowlist — this is a common setup issue that blocks connectivity even when credentials and grants are correct. + +See [3-iceberg-rest-catalog.md](3-iceberg-rest-catalog.md) for IRC endpoint details. + +--- + +## PyIceberg + +PyIceberg is a Python library for reading and writing Iceberg tables without Spark. + +### Installation + +Upgrade both packages explicitly — if `pyarrow` (v15) is too old, it causes write errors. Also install `adlfs` for Azure storage access: + +```bash +pip install --upgrade "pyiceberg>=0.9,<0.10" "pyarrow>=17,<20" +pip install adlfs +``` + +For non-Databricks environments: + +```bash +pip install "pyiceberg[pyarrow]>=0.9" +``` + +### Connect to Catalog + +The `warehouse` parameter pins the catalog, so all subsequent table identifiers use `.` (not `..
`): + +```python +from pyiceberg.catalog import load_catalog + +catalog = load_catalog( + "uc", + uri="https:///api/2.1/unity-catalog/iceberg-rest", + warehouse="", # Unity Catalog catalog name + token="", +) +``` + +### Read Table + +```python +# Load table — identifier is .
because 'warehouse' pins the UC catalog +tbl = catalog.load_table(".
") + +# Inspect schema and current snapshot +print(tbl) # schema, partitioning, snapshot summary +print(tbl.current_snapshot()) # snapshot metadata + +# Read sample rows +df = tbl.scan(limit=10).to_pandas() +print(df.head()) + +# Pushdown filter (SQL-style filter strings are supported) +df = tbl.scan( + row_filter="event_date >= '2025-01-01'", + limit=1000, +).to_pandas() + +# Read as Arrow +arrow_table = tbl.scan().to_arrow() +``` + +### Append Data + +```python +import pyarrow as pa +from pyiceberg.catalog import load_catalog + +catalog = load_catalog( + "uc", + uri="https:///api/2.1/unity-catalog/iceberg-rest", + warehouse="", + token="", +) + +tbl = catalog.load_table(".
") + +# Schema must match the Iceberg table schema exactly — use explicit Arrow types +# PyArrow defaults to int64; if the Iceberg table uses int (32-bit), cast explicitly +arrow_schema = pa.schema([ + pa.field("id", pa.int32()), + pa.field("name", pa.string()), + pa.field("qty", pa.int32()), +]) + +rows = [ + {"id": 1, "name": "foo", "qty": 10}, + {"id": 2, "name": "bar", "qty": 20}, +] +arrow_tbl = pa.Table.from_pylist(rows, schema=arrow_schema) + +tbl.append(arrow_tbl) + +# Verify +print("Current snapshot:", tbl.current_snapshot()) +``` + +--- + +## OSS Apache Spark + +> **CRITICAL**: Only configure this **outside** Databricks Runtime. Inside DBR, use the built-in Iceberg support — do NOT install the Iceberg library. + +### Dependencies + +Two JARs are required: the Spark runtime and a cloud-specific bundle for object storage access. Choose the bundle matching your Databricks metastore's cloud: + +| Cloud | Bundle | +|-------|--------| +| AWS | `org.apache.iceberg:iceberg-aws-bundle:` | +| Azure | `org.apache.iceberg:iceberg-azure-bundle:` | +| GCP | `org.apache.iceberg:iceberg-gcp-bundle:` | + +### Spark Session Configuration + +The Databricks docs recommend OAuth2 (service principal) for external Spark connections. Set `rest.auth.type=oauth2` and provide the OAuth2 server URI, credential, and scope: + +```python +from pyspark.sql import SparkSession + +WORKSPACE_URL = "https://" +UC_CATALOG_NAME = "" +OAUTH_CLIENT_ID = "" +OAUTH_CLIENT_SECRET = "" +CATALOG_ALIAS = "uc" # arbitrary name used to reference this catalog in Spark SQL +ICEBERG_VER = "1.7.1" + +RUNTIME = f"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:{ICEBERG_VER}" +CLOUD_BUNDLE = f"org.apache.iceberg:iceberg-aws-bundle:{ICEBERG_VER}" # or azure/gcp-bundle + +spark = ( + SparkSession.builder + .appName("uc-iceberg") + .config("spark.jars.packages", f"{RUNTIME},{CLOUD_BUNDLE}") + .config("spark.sql.extensions", + "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") + .config(f"spark.sql.catalog.{CATALOG_ALIAS}", + "org.apache.iceberg.spark.SparkCatalog") + .config(f"spark.sql.catalog.{CATALOG_ALIAS}.type", "rest") + .config(f"spark.sql.catalog.{CATALOG_ALIAS}.rest.auth.type", "oauth2") + .config(f"spark.sql.catalog.{CATALOG_ALIAS}.uri", + f"{WORKSPACE_URL}/api/2.1/unity-catalog/iceberg-rest") + .config(f"spark.sql.catalog.{CATALOG_ALIAS}.oauth2-server-uri", + f"{WORKSPACE_URL}/oidc/v1/token") + .config(f"spark.sql.catalog.{CATALOG_ALIAS}.credential", + f"{OAUTH_CLIENT_ID}:{OAUTH_CLIENT_SECRET}") + .config(f"spark.sql.catalog.{CATALOG_ALIAS}.scope", "all-apis") + .config(f"spark.sql.catalog.{CATALOG_ALIAS}.warehouse", UC_CATALOG_NAME) + .getOrCreate() +) + +# List schemas +spark.sql(f"SHOW NAMESPACES IN {CATALOG_ALIAS}").show(truncate=False) + +# Query +spark.sql(f"SELECT * FROM {CATALOG_ALIAS}..
").show() + +# Write (managed Iceberg tables only) +df.writeTo(f"{CATALOG_ALIAS}..
").append() +``` + +### Spark SQL + +```sql +-- List schemas +SHOW NAMESPACES IN uc; + +-- Query +SELECT * FROM uc..
; + +-- Insert +INSERT INTO uc..
VALUES (1, 'foo', 10); +``` + +--- + +## Troubleshooting + +| Issue | Solution | +|-------|----------| +| **Connection timeout or `403 Forbidden` with valid credentials** | Workspace IP access list is blocking the client — add the client's egress CIDR to the allowlist (admin console: **Settings → Security → IP access list**) | +| **`403 Forbidden`** | Check `EXTERNAL USE SCHEMA` grant and token validity | +| **`Table not found`** | Verify the `warehouse` config matches the UC catalog name; check schema and table names | +| **Class conflict in DBR** | You installed an Iceberg library in Databricks Runtime — remove it; DBR has built-in support | +| **Credential vending failure** | Ensure external data access is enabled on the workspace | +| **Slow reads** | Check if table needs compaction (`OPTIMIZE`); large numbers of small files degrade performance | +| **v3 table incompatibility** | Upgrade to Iceberg library 1.9.0+ for v3 support; older versions cannot read v3 tables | +| **PyArrow schema mismatch** | Cast to explicit types (e.g., `pa.int32()`) when the Iceberg table schema uses 32-bit integers | +| **PyIceberg write error on serverless** | Upgrade pyarrow (`>=17`) and install `adlfs` — the bundled pyarrow v15 is incompatible | + +--- + +## Related + +- [3-iceberg-rest-catalog.md](3-iceberg-rest-catalog.md) — IRC endpoint details, auth, credential vending +- [4-snowflake-interop.md](4-snowflake-interop.md) — Snowflake-specific integration diff --git a/.claude/skills/databricks-iceberg/SKILL.md b/.claude/skills/databricks-iceberg/SKILL.md new file mode 100644 index 0000000..3c8a1cb --- /dev/null +++ b/.claude/skills/databricks-iceberg/SKILL.md @@ -0,0 +1,148 @@ +--- +name: databricks-iceberg +description: "Apache Iceberg tables on Databricks — Managed Iceberg tables, External Iceberg Reads (fka Uniform), Compatibility Mode, Iceberg REST Catalog (IRC), Iceberg v3, Snowflake interop, PyIceberg, OSS Spark, external engine access and credential vending. Use when creating Iceberg tables, enabling External Iceberg Reads (uniform) on Delta tables (including Streaming Tables and Materialized Views via compatibility mode), configuring external engines to read Databricks tables via Unity Catalog IRC, integrating with Snowflake catalog to read Foreign Iceberg tables" +--- + +# Apache Iceberg on Databricks + +Databricks provides multiple ways to work with Apache Iceberg: native managed Iceberg tables, UniForm for Delta-to-Iceberg interoperability, and the Iceberg REST Catalog (IRC) for external engine access. + +--- + +## Critical Rules (always follow) + +- **MUST** use Unity Catalog — all Iceberg features require UC-enabled workspaces +- **MUST NOT** install an Iceberg library into Databricks Runtime (DBR includes built-in Iceberg support; adding a library causes version conflicts) +- **MUST NOT** set `write.metadata.path` or `write.metadata.previous-versions-max` — Databricks manages metadata locations automatically; overriding causes corruption +- **MUST** determine which Iceberg pattern fits the use case before writing code — see the [When to Use](#when-to-use) section below +- **MUST** know that both `PARTITIONED BY` and `CLUSTER BY` produce the same Iceberg metadata for external engines — UC maintains an Iceberg partition spec with partition fields corresponding to the clustering keys, so external engines reading via IRC see a partitioned Iceberg table (not Hive-style, but proper Iceberg partition fields) and can prune on those fields; internally UC uses those fields as liquid clustering keys; the only differences between the two syntaxes are: (1) `PARTITIONED BY` is standard Iceberg DDL (any engine can create the table), while `CLUSTER BY` is DBR-only DDL; (2) `PARTITIONED BY` **auto-handles** DV/row-tracking properties, while `CLUSTER BY` requires manual TBLPROPERTIES on v2 +- **MUST NOT** use expression-based partition transforms (`bucket()`, `years()`, `months()`, `days()`, `hours()`) with `PARTITIONED BY` on managed Iceberg tables — only plain column references are supported; expression transforms cause errors +- **MUST** disable deletion vectors and row tracking when using `CLUSTER BY` on Iceberg v2 tables — set `'delta.enableDeletionVectors' = false` and `'delta.enableRowTracking' = false` in TBLPROPERTIES (Iceberg v3 handles this automatically; `PARTITIONED BY` handles this automatically on both v2 and v3) + +--- + +## Key Concepts + +| Concept | Summary | +|---------|---------| +| **Managed Iceberg Table** | Native Iceberg table created with `USING ICEBERG` — full read/write in Databricks and via external Iceberg engines | +| **External Iceberg Reads (Uniform)** | Delta table that auto-generates Iceberg metadata — read as Iceberg externally, write as Delta internally | +| **Compatibility Mode** | UniForm variant for streaming tables and materialized views in SDP pipelines | +| **Iceberg REST Catalog (IRC)** | Unity Catalog's built-in REST endpoint implementing the Iceberg REST Catalog spec — lets external engines (Spark, PyIceberg, Snowflake) access UC-managed Iceberg data | +| **Iceberg v3** | Next-gen format (Beta, DBR 17.3+) — deletion vectors, VARIANT type, row lineage | + +--- + +## Quick Start + +### Create a Managed Iceberg Table + +```sql +-- No clustering +CREATE TABLE my_catalog.my_schema.events +USING ICEBERG +AS SELECT * FROM raw_events; + +-- PARTITIONED BY (recommended for cross-platform): standard Iceberg syntax, works on EMR/OSS Spark/Trino/Flink +-- auto-disables DVs and row tracking — no TBLPROPERTIES needed on v2 or v3 +CREATE TABLE my_catalog.my_schema.events +USING ICEBERG +PARTITIONED BY (event_date) +AS SELECT * FROM raw_events; + +-- CLUSTER BY on Iceberg v2 (DBR-only syntax): must manually disable DVs and row tracking +CREATE TABLE my_catalog.my_schema.events +USING ICEBERG +TBLPROPERTIES ( + 'delta.enableDeletionVectors' = false, + 'delta.enableRowTracking' = false +) +CLUSTER BY (event_date) +AS SELECT * FROM raw_events; + +-- CLUSTER BY on Iceberg v3 (DBR-only syntax): no TBLPROPERTIES needed +CREATE TABLE my_catalog.my_schema.events +USING ICEBERG +TBLPROPERTIES ('format-version' = '3') +CLUSTER BY (event_date) +AS SELECT * FROM raw_events; +``` + +### Enable UniForm on an Existing Delta Table + +```sql +ALTER TABLE my_catalog.my_schema.customers +SET TBLPROPERTIES ( + 'delta.columnMapping.mode' = 'name', + 'delta.enableIcebergCompatV2' = 'true', + 'delta.universalFormat.enabledFormats' = 'iceberg' +); +``` + +--- + +## Read/Write Capability Matrix + +| Table Type | Databricks Read | Databricks Write | External IRC Read | External IRC Write | +|------------|:-:|:-:|:-:|:-:| +| Managed Iceberg (`USING ICEBERG`) | Yes | Yes | Yes | Yes | +| Delta + UniForm | Yes (as Delta) | Yes (as Delta) | Yes (as Iceberg) | No | +| Delta + Compatibility Mode | Yes (as Delta) | Yes | Yes (as Iceberg) | No | + +--- + +## Reference Files + +| File | Summary | Keywords | +|------|---------|----------| +| [1-managed-iceberg-tables.md](1-managed-iceberg-tables.md) | Creating and managing native Iceberg tables — DDL, DML, Liquid Clustering, Predictive Optimization, Iceberg v3, limitations | CREATE TABLE USING ICEBERG, CTAS, MERGE, time travel, deletion vectors, VARIANT | +| [2-uniform-and-compatibility.md](2-uniform-and-compatibility.md) | Making Delta tables readable as Iceberg — UniForm for regular tables, Compatibility Mode for streaming tables and MVs | UniForm, universalFormat, Compatibility Mode, streaming tables, materialized views, SDP | +| [3-iceberg-rest-catalog.md](3-iceberg-rest-catalog.md) | Exposing Databricks tables to external engines via the IRC endpoint — auth, credential vending, IP access lists | IRC, REST Catalog, credential vending, EXTERNAL USE SCHEMA, PAT, OAuth | +| [4-snowflake-interop.md](4-snowflake-interop.md) | Bidirectional Snowflake-Databricks integration — catalog integration, foreign catalogs, vended credentials | Snowflake, catalog integration, external volume, vended credentials, REFRESH_INTERVAL_SECONDS | +| [5-external-engine-interop.md](5-external-engine-interop.md) | Connecting PyIceberg, OSS Spark, AWS EMR, Apache Flink, and Kafka Connect via IRC | PyIceberg, OSS Spark, EMR, Flink, Kafka Connect, pyiceberg.yaml | + +--- + +## When to Use + +- **Creating a new Iceberg table** → [1-managed-iceberg-tables.md](1-managed-iceberg-tables.md) +- **Making an existing Delta table readable as Iceberg** → [2-uniform-and-compatibility.md](2-uniform-and-compatibility.md) +- **Making a streaming table or MV readable as Iceberg** → [2-uniform-and-compatibility.md](2-uniform-and-compatibility.md) (Compatibility Mode section) +- **Choosing between Managed Iceberg vs UniForm vs Compatibility Mode** → decision table in [2-uniform-and-compatibility.md](2-uniform-and-compatibility.md) +- **Exposing Databricks tables to external engines via REST API** → [3-iceberg-rest-catalog.md](3-iceberg-rest-catalog.md) +- **Integrating Databricks with Snowflake (either direction)** → [4-snowflake-interop.md](4-snowflake-interop.md) +- **Connecting PyIceberg, OSS Spark, Flink, EMR, or Kafka** → [5-external-engine-interop.md](5-external-engine-interop.md) + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| **No Change Data Feed (CDF)** | CDF is not supported on managed Iceberg tables. Use Delta + UniForm if you need CDF. | +| **UniForm async delay** | Iceberg metadata generation is asynchronous. After a write, there may be a brief delay before external engines see the latest data. Check status with `DESCRIBE EXTENDED table_name`. | +| **Compression codec change** | Managed Iceberg tables use `zstd` compression by default (not `snappy`). Older Iceberg readers that don't support zstd will fail. Verify reader compatibility or set `write.parquet.compression-codec` to `snappy`. | +| **Snowflake 1000-commit limit** | Snowflake's Iceberg catalog integration can only see the last 1000 Iceberg commits. High-frequency writers must compact metadata or Snowflake will lose visibility of older data. | +| **Deletion vectors with UniForm** | UniForm requires deletion vectors to be disabled (`delta.enableDeletionVectors = false`). If your table has deletion vectors enabled, disable them before enabling UniForm. | +| **No shallow clone for Iceberg** | `SHALLOW CLONE` is not supported for Iceberg tables. Use `DEEP CLONE` or `CREATE TABLE ... AS SELECT` instead. | +| **Version mismatch with external engines** | Ensure external engines use an Iceberg library version compatible with the format version of your tables. Iceberg v3 tables require Iceberg library 1.9.0+. | + +--- + +## Related Skills + +- **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** — catalog/schema management, governance, system tables +- **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** — SDP pipelines (streaming tables, materialized views with Compatibility Mode) +- **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** — Python SDK and REST API for Databricks operations +- **[databricks-dbsql](../databricks-dbsql/SKILL.md)** — SQL warehouse features, query patterns + +--- + +## Resources + +- **[Iceberg Overview](https://docs.databricks.com/aws/en/iceberg/)** — main hub for Iceberg on Databricks +- **[UniForm](https://docs.databricks.com/aws/en/delta/uniform.html)** — Delta Universal Format +- **[Iceberg REST Catalog](https://docs.databricks.com/aws/en/external-access/iceberg)** — IRC endpoint and external engine access +- **[Compatibility Mode](https://docs.databricks.com/aws/en/external-access/compatibility-mode)** — UniForm for streaming tables and MVs +- **[Iceberg v3](https://docs.databricks.com/aws/en/iceberg/iceberg-v3)** — next-gen format features (Beta) +- **[Foreign Tables](https://docs.databricks.com/aws/en/query-data/foreign-tables.html)** — reading external catalog data diff --git a/.claude/skills/databricks-jobs/SKILL.md b/.claude/skills/databricks-jobs/SKILL.md index 2f0f8c7..0f60a24 100644 --- a/.claude/skills/databricks-jobs/SKILL.md +++ b/.claude/skills/databricks-jobs/SKILL.md @@ -326,7 +326,7 @@ resources: ## Related Skills -- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - Deploy jobs via Databricks Asset Bundles +- **[databricks-bundles](../databricks-bundles/SKILL.md)** - Deploy jobs via Databricks Asset Bundles - **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - Configure pipelines triggered by jobs ## Resources diff --git a/.claude/skills/databricks-lakebase-autoscale/SKILL.md b/.claude/skills/databricks-lakebase-autoscale/SKILL.md index 50ba1df..8d7dd6f 100644 --- a/.claude/skills/databricks-lakebase-autoscale/SKILL.md +++ b/.claude/skills/databricks-lakebase-autoscale/SKILL.md @@ -1,294 +1,133 @@ --- name: databricks-lakebase-autoscale -description: "Patterns and best practices for using Lakebase Autoscaling (next-gen managed PostgreSQL) with autoscaling, branching, scale-to-zero, and instant restore." +description: "Patterns and best practices for Lakebase Autoscaling (next-gen managed PostgreSQL). Use when creating or managing Lakebase Autoscaling projects, configuring autoscaling compute or scale-to-zero, working with database branching for dev/test workflows, implementing reverse ETL via synced tables, or connecting applications to Lakebase with OAuth credentials." --- # Lakebase Autoscaling -Patterns and best practices for using Lakebase Autoscaling, the next-generation managed PostgreSQL on Databricks with autoscaling compute, branching, scale-to-zero, and instant restore. +Lakebase Autoscaling is Databricks' next-generation managed PostgreSQL service for OLTP workloads: autoscaling compute, database branching, scale-to-zero, instant restore, and Delta-to-Postgres synced tables. -## When to Use +Use this skill when creating/managing Lakebase Autoscaling projects, branches, endpoints/computes, credentials, reverse ETL synced tables, or app connections. -Use this skill when: -- Building applications that need a PostgreSQL database with autoscaling compute -- Working with database branching for dev/test/staging workflows -- Adding persistent state to applications with scale-to-zero cost savings -- Implementing reverse ETL from Delta Lake to an operational database via synced tables -- Managing Lakebase Autoscaling projects, branches, computes, or credentials +## Core framing -## Overview +> **There is no separate Python “Lakebase SDK.”** Use `databricks-sdk` for management and for minting short-lived database credentials with `WorkspaceClient().postgres.generate_database_credential(...)`; use standard Postgres drivers (`psycopg`, SQLAlchemy, JDBC, `pgx`, etc.) for SQL. -Lakebase Autoscaling is Databricks' next-generation managed PostgreSQL service for OLTP workloads. It provides autoscaling compute, Git-like branching, scale-to-zero, and instant point-in-time restore. +| Language | Credential / management SDK | DB driver / wrapper | +|---|---|---| +| **Python** | `databricks-sdk` `WorkspaceClient().postgres` | `psycopg[binary,pool]` canonical; SQLAlchemy supported | +| **Node/TS** | `@databricks/lakebase` convenience wrapper, Autoscaling only | Wrapper manages `pg` pool | +| **Java/Go** | Databricks SDK for Java/Go | Standard JDBC / `pgx` | -| Feature | Description | -|---------|-------------| -| **Autoscaling Compute** | 0.5-112 CU with 2 GB RAM per CU; scales dynamically based on load | -| **Scale-to-Zero** | Compute suspends after configurable inactivity timeout | -| **Branching** | Create isolated database environments (like Git branches) for dev/test | -| **Instant Restore** | Point-in-time restore from any moment within the configured window (up to 35 days) | -| **OAuth Authentication** | Token-based auth via Databricks SDK (1-hour expiry) | -| **Reverse ETL** | Sync data from Delta tables to PostgreSQL via synced tables | +## Lead connection pattern -**Available Regions (AWS):** us-east-1, us-east-2, eu-central-1, eu-west-1, eu-west-2, ap-south-1, ap-southeast-1, ap-southeast-2 +For production Python apps, start with: -**Available Regions (Azure Beta):** eastus2, westeurope, westus +1. `psycopg_pool.ConnectionPool` +2. `connection_class=OAuthConnection`, where `OAuthConnection(psycopg.Connection).connect()` calls `w.postgres.generate_database_credential(endpoint=...)` +3. `max_lifetime=2700` -## Project Hierarchy +This is the canonical pattern from the official Databricks Apps + Lakebase Autoscaling tutorial lineage and `databricks-ai-bridge`: no background token thread; physical connections get fresh credentials when opened/recycled. -Understanding the hierarchy is essential for working with Lakebase Autoscaling: +Prefer `max_lifetime=2700` as a defensive 45-minute recycle before 1-hour token expiry. The official tutorial does not set `max_lifetime`; `databricks-ai-bridge` uses `2700`. -``` -Project (top-level container) - └── Branch(es) (isolated database environments) - ├── Compute (primary R/W endpoint) - ├── Read Replica(s) (optional, read-only) - ├── Role(s) (Postgres roles) - └── Database(s) (Postgres databases) - └── Schema(s) -``` +See `connections.md`. -| Object | Description | -|--------|-------------| -| **Project** | Top-level container. Created via `w.postgres.create_project()`. | -| **Branch** | Isolated database environment with copy-on-write storage. Default branch is `production`. | -| **Compute** | Postgres server powering a branch. Configurable CU sizing and autoscaling. | -| **Database** | Standard Postgres database within a branch. Default is `databricks_postgres`. | +## Critical auth warning -## Quick Start +Do **not** use `WorkspaceClient().config.token`, `w.config.oauth_token().access_token`, or any workspace-scoped OAuth token as the Postgres password. It will fail at Postgres login. -Create a project and connect: +Use: ```python -from databricks.sdk import WorkspaceClient -from databricks.sdk.service.postgres import Project, ProjectSpec - -w = WorkspaceClient() - -# Create a project (long-running operation) -operation = w.postgres.create_project( - project=Project( - spec=ProjectSpec( - display_name="My Application", - pg_version="17" - ) - ), - project_id="my-app" -) -result = operation.wait() -print(f"Created project: {result.name}") +cred = WorkspaceClient().postgres.generate_database_credential(endpoint=endpoint_name) +password = cred.token ``` -## Common Patterns - -### Generate OAuth Token - -```python -from databricks.sdk import WorkspaceClient +That token is Lakebase-scoped and is used as the Postgres password with `sslmode=require`. -w = WorkspaceClient() +## Resource model -# Generate database credential for connecting (optionally scoped to an endpoint) -cred = w.postgres.generate_database_credential( - endpoint="projects/my-app/branches/production/endpoints/ep-primary" -) -token = cred.token # Use as password in connection string -# Token expires after 1 hour +```text +Project + └── Branches + ├── Endpoint/Compute: primary read-write endpoint + ├── Read replicas: optional read-only endpoints + ├── Roles + └── Databases + └── Schemas/Tables ``` -### Connect from Notebook +Canonical names: -```python -import psycopg -from databricks.sdk import WorkspaceClient - -w = WorkspaceClient() - -# Get endpoint details -endpoint = w.postgres.get_endpoint( - name="projects/my-app/branches/production/endpoints/ep-primary" -) -host = endpoint.status.hosts.host - -# Generate token (scoped to endpoint) -cred = w.postgres.generate_database_credential( - endpoint="projects/my-app/branches/production/endpoints/ep-primary" -) - -# Connect using psycopg3 -conn_string = ( - f"host={host} " - f"dbname=databricks_postgres " - f"user={w.current_user.me().user_name} " - f"password={cred.token} " - f"sslmode=require" -) -with psycopg.connect(conn_string) as conn: - with conn.cursor() as cur: - cur.execute("SELECT version()") - print(cur.fetchone()) +```text +projects/{project_id} +projects/{project_id}/branches/{branch_id} +projects/{project_id}/branches/{branch_id}/endpoints/{endpoint_id} ``` -### Create a Branch for Development - -```python -from databricks.sdk.service.postgres import Branch, BranchSpec, Duration - -# Create a dev branch with 7-day expiration -branch = w.postgres.create_branch( - parent="projects/my-app", - branch=Branch( - spec=BranchSpec( - source_branch="projects/my-app/branches/production", - ttl=Duration(seconds=604800) # 7 days - ) - ), - branch_id="development" -).wait() -print(f"Branch created: {branch.name}") -``` - -### Resize Compute (Autoscaling) - -```python -from databricks.sdk.service.postgres import Endpoint, EndpointSpec, FieldMask - -# Update compute to autoscale between 2-8 CU -w.postgres.update_endpoint( - name="projects/my-app/branches/production/endpoints/ep-primary", - endpoint=Endpoint( - name="projects/my-app/branches/production/endpoints/ep-primary", - spec=EndpointSpec( - autoscaling_limit_min_cu=2.0, - autoscaling_limit_max_cu=8.0 - ) - ), - update_mask=FieldMask(field_mask=[ - "spec.autoscaling_limit_min_cu", - "spec.autoscaling_limit_max_cu" - ]) -).wait() -``` - -## MCP Tools - -The following MCP tools are available for managing Lakebase infrastructure. Use `type="autoscale"` for Lakebase Autoscaling. - -### Database (Project) Management - -| Tool | Description | -|------|-------------| -| `create_or_update_lakebase_database` | Create or update a database. Finds by name, creates if new, updates if existing. Use `type="autoscale"`, `display_name`, `pg_version` params. A new project auto-creates a production branch, default compute, and databricks_postgres database. | -| `get_lakebase_database` | Get database details (including branches and endpoints) or list all. Pass `name` to get one, omit to list all. Use `type="autoscale"` to filter. | -| `delete_lakebase_database` | Delete a project and all its branches, computes, and data. Use `type="autoscale"`. | +Defaults on project creation: +- default branch: `production` +- default database: `databricks_postgres` +- primary read-write endpoint/compute +- Postgres role for the creator’s Databricks identity -### Branch Management +Key SDK namespace: `WorkspaceClient().postgres`. -| Tool | Description | -|------|-------------| -| `create_or_update_lakebase_branch` | Create or update a branch with its compute endpoint. Params: `project_name`, `branch_id`, `source_branch`, `ttl_seconds`, `is_protected`, plus compute params (`autoscaling_limit_min_cu`, `autoscaling_limit_max_cu`, `scale_to_zero_seconds`). | -| `delete_lakebase_branch` | Delete a branch and its compute endpoints. | +Most create/update/delete calls return long-running operations; call `.wait()`. -### Credentials - -| Tool | Description | -|------|-------------| -| `generate_lakebase_credential` | Generate OAuth token for PostgreSQL connections (1-hour expiry). Pass `endpoint` resource name for autoscale. | - -## Reference Files - -- [projects.md](projects.md) - Project management patterns and settings -- [branches.md](branches.md) - Branching workflows, protection, and expiration -- [computes.md](computes.md) - Compute sizing, autoscaling, and scale-to-zero -- [connection-patterns.md](connection-patterns.md) - Connection patterns for different use cases -- [reverse-etl.md](reverse-etl.md) - Synced tables from Delta Lake to Lakebase - -## CLI Quick Reference - -```bash -# Create a project -databricks postgres create-project \ - --project-id my-app \ - --json '{"spec": {"display_name": "My App", "pg_version": "17"}}' - -# List projects -databricks postgres list-projects - -# Get project details -databricks postgres get-project projects/my-app - -# Create a branch -databricks postgres create-branch projects/my-app development \ - --json '{"spec": {"source_branch": "projects/my-app/branches/production", "no_expiry": true}}' - -# List branches -databricks postgres list-branches projects/my-app - -# Get endpoint details -databricks postgres get-endpoint projects/my-app/branches/production/endpoints/ep-primary - -# Delete a project -databricks postgres delete-project projects/my-app -``` - -## Key Differences from Lakebase Provisioned +## Lakebase Autoscaling vs Provisioned | Aspect | Provisioned | Autoscaling | -|--------|-------------|-------------| +|---|---|---| | SDK module | `w.database` | `w.postgres` | | Top-level resource | Instance | Project | -| Capacity | CU_1, CU_2, CU_4, CU_8 (16 GB/CU) | 0.5-112 CU (2 GB/CU) | -| Branching | Not supported | Full branching support | -| Scale-to-zero | Not supported | Configurable timeout | -| Operations | Synchronous | Long-running operations (LRO) | -| Read replicas | Readable secondaries | Dedicated read-only endpoints | - -## Common Issues - -| Issue | Solution | -|-------|----------| -| **Token expired during long query** | Implement token refresh loop; tokens expire after 1 hour | -| **Connection refused after scale-to-zero** | Compute wakes automatically on connection; reactivation takes a few hundred ms; implement retry logic | -| **DNS resolution fails on macOS** | Use `dig` command to resolve hostname, pass `hostaddr` to psycopg | -| **Branch deletion blocked** | Delete child branches first; cannot delete branches with children | -| **Autoscaling range too wide** | Max - min cannot exceed 8 CU (e.g., 8-16 CU is valid, 0.5-32 CU is not) | -| **SSL required error** | Always use `sslmode=require` in connection string | -| **Update mask required** | All update operations require an `update_mask` specifying fields to modify | -| **Connection closed after 24h idle** | All connections have a 24-hour idle timeout and 3-day max lifetime; implement retry logic | - -## Current Limitations - -These features are NOT yet supported in Lakebase Autoscaling: -- High availability with readable secondaries (use read replicas instead) -- Databricks Apps UI integration (Apps can connect manually via credentials) -- Feature Store integration -- Stateful AI agents (LangChain memory) -- Postgres-to-Delta sync (only Delta-to-Postgres reverse ETL) -- Custom billing tags and serverless budget policies -- Direct migration from Lakebase Provisioned (use pg_dump/pg_restore or reverse ETL) - -## SDK Version Requirements - -- **Databricks SDK for Python**: >= 0.81.0 (for `w.postgres` module) -- **psycopg**: 3.x (supports `hostaddr` parameter for DNS workaround) -- **SQLAlchemy**: 2.x with `postgresql+psycopg` driver +| Capacity | fixed CU tiers, ~16 GB/CU | 0.5–112 CU, ~2 GB/CU | +| Branching | no | yes | +| Scale-to-zero | no | yes | +| Operations | mostly synchronous | LROs; use `.wait()` | +| Reverse ETL | synced tables | synced tables | +| Read replicas | readable secondaries | dedicated read-only endpoints | + +## Non-obvious facts to preserve + +- Postgres versions: **16 and 17**. +- AWS regions: `us-east-1`, `us-east-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-south-1`, `ap-southeast-1`, `ap-southeast-2`. +- Azure beta regions: `eastus2`, `westeurope`, `westus`. +- Autoscaling computes: 0.5–32 CU with `max - min <= 16`. +- Fixed-size always-on computes: 40–112 CU. +- Autoscaling CU ≈ 2 GB RAM. +- `sslmode=require` on all driver connections. +- Endpoint host comes from `w.postgres.get_endpoint(...).status.hosts.host`. +- GET responses often return effective properties under `status`; create/update payloads use `spec`. +- All update calls need a `FieldMask`. +- Scale-to-zero wake-up is automatic but apps should retry. +- Connections can be closed by platform timeouts: 24-hour idle timeout and 3-day max connection lifetime. +- macOS DNS can fail on long Lakebase hostnames; if so, resolve to IP and pass both `host` and `hostaddr` to psycopg. +- Triggered/Continuous synced tables require Delta Change Data Feed. +- Reverse ETL is Delta-to-Postgres only; not Postgres-to-Delta. + +## Task files + +- `connections.md` — app/notebook connection patterns and credential rotation. +- `operations.md` — project, branch, endpoint/compute, scale-to-zero, limits, MCP mapping. +- `reverse-etl.md` — synced tables from Delta Lake to Lakebase. + +## SDK / package versions -```python -%pip install -U "databricks-sdk>=0.81.0" "psycopg[binary]>=3.0" sqlalchemy +```bash +pip install -U "databricks-sdk>=0.81.0" "psycopg[binary,pool]>=3.1" "sqlalchemy>=2" ``` -## Notes - -- **Compute Units** in Autoscaling provide ~2 GB RAM each (vs 16 GB in Provisioned). -- **Resource naming** follows hierarchical paths: `projects/{id}/branches/{id}/endpoints/{id}`. -- All create/update/delete operations are **long-running** -- use `.wait()` in the SDK. -- Tokens are short-lived (1 hour) -- production apps MUST implement token refresh. -- **Postgres versions** 16 and 17 are supported. +Use SQLAlchemy URL prefix `postgresql+psycopg://...` for psycopg3. -## Related Skills +## Current limitations -- **[databricks-lakebase-provisioned](../databricks-lakebase-provisioned/SKILL.md)** - fixed-capacity managed PostgreSQL (predecessor) -- **[databricks-app-apx](../databricks-app-apx/SKILL.md)** - full-stack apps that can use Lakebase for persistence -- **[databricks-app-python](../databricks-app-python/SKILL.md)** - Python apps with Lakebase backend -- **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - SDK used for project management and token generation -- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - deploying apps with Lakebase resources -- **[databricks-jobs](../databricks-jobs/SKILL.md)** - scheduling reverse ETL sync jobs +Not yet supported or not equivalent to Provisioned: +- High availability with readable secondaries; use read replicas instead. +- Databricks Apps UI integration may lag; Apps can connect manually via credentials/resource env vars. +- Feature Store integration. +- Stateful AI-agent memory integrations. +- Postgres-to-Delta sync. +- Custom billing tags / serverless budget policies. +- Direct migration from Lakebase Provisioned; use `pg_dump`/`pg_restore` or reverse ETL patterns where appropriate. diff --git a/.claude/skills/databricks-lakebase-autoscale/branches.md b/.claude/skills/databricks-lakebase-autoscale/branches.md deleted file mode 100644 index f44f723..0000000 --- a/.claude/skills/databricks-lakebase-autoscale/branches.md +++ /dev/null @@ -1,212 +0,0 @@ -# Lakebase Autoscaling Branches - -## Overview - -Branches in Lakebase Autoscaling are isolated database environments that share storage with their parent through copy-on-write. They enable Git-like workflows for databases: create isolated dev/test environments, test schema changes safely, and recover from mistakes. - -## Branch Types - -| Option | Description | Use Case | -|--------|-------------|----------| -| **Current data** | Branch from latest state of parent | Development, testing with current data | -| **Past data** | Branch from a specific point in time | Point-in-time recovery, historical analysis | - -## Creating a Branch - -### With Expiration (TTL) - -```python -from databricks.sdk import WorkspaceClient -from databricks.sdk.service.postgres import Branch, BranchSpec, Duration - -w = WorkspaceClient() - -# Create branch with 7-day expiration -result = w.postgres.create_branch( - parent="projects/my-app", - branch=Branch( - spec=BranchSpec( - source_branch="projects/my-app/branches/production", - ttl=Duration(seconds=604800) # 7 days - ) - ), - branch_id="development" -).wait() - -print(f"Branch created: {result.name}") -print(f"Expires: {result.status.expire_time}") -``` - -### Permanent Branch (No Expiration) - -```python -result = w.postgres.create_branch( - parent="projects/my-app", - branch=Branch( - spec=BranchSpec( - source_branch="projects/my-app/branches/production", - no_expiry=True - ) - ), - branch_id="staging" -).wait() -``` - -### CLI - -```bash -# With TTL -databricks postgres create-branch projects/my-app development \ - --json '{ - "spec": { - "source_branch": "projects/my-app/branches/production", - "ttl": "604800s" - } - }' - -# Permanent -databricks postgres create-branch projects/my-app staging \ - --json '{ - "spec": { - "source_branch": "projects/my-app/branches/production", - "no_expiry": true - } - }' -``` - -## Getting Branch Details - -```python -branch = w.postgres.get_branch( - name="projects/my-app/branches/development" -) - -print(f"Branch: {branch.name}") -print(f"Protected: {branch.status.is_protected}") -print(f"Default: {branch.status.default}") -print(f"State: {branch.status.current_state}") -print(f"Size: {branch.status.logical_size_bytes} bytes") -``` - -## Listing Branches - -```python -branches = list(w.postgres.list_branches( - parent="projects/my-app" -)) - -for branch in branches: - print(f"Branch: {branch.name}") - print(f" Default: {branch.status.default}") - print(f" Protected: {branch.status.is_protected}") -``` - -## Protecting a Branch - -Protected branches cannot be deleted, reset, or archived. - -```python -from databricks.sdk.service.postgres import Branch, BranchSpec, FieldMask - -w.postgres.update_branch( - name="projects/my-app/branches/production", - branch=Branch( - name="projects/my-app/branches/production", - spec=BranchSpec(is_protected=True) - ), - update_mask=FieldMask(field_mask=["spec.is_protected"]) -).wait() -``` - -To remove protection: - -```python -w.postgres.update_branch( - name="projects/my-app/branches/production", - branch=Branch( - name="projects/my-app/branches/production", - spec=BranchSpec(is_protected=False) - ), - update_mask=FieldMask(field_mask=["spec.is_protected"]) -).wait() -``` - -## Updating Branch Expiration - -```python -# Extend to 14 days -w.postgres.update_branch( - name="projects/my-app/branches/development", - branch=Branch( - name="projects/my-app/branches/development", - spec=BranchSpec( - is_protected=False, - ttl=Duration(seconds=1209600) # 14 days - ) - ), - update_mask=FieldMask(field_mask=["spec.is_protected", "spec.expiration"]) -).wait() - -# Remove expiration -w.postgres.update_branch( - name="projects/my-app/branches/development", - branch=Branch( - name="projects/my-app/branches/development", - spec=BranchSpec(no_expiry=True) - ), - update_mask=FieldMask(field_mask=["spec.expiration"]) -).wait() -``` - -## Resetting a Branch from Parent - -Reset completely replaces a branch's data and schema with the latest from its parent. Local changes are lost. - -```python -w.postgres.reset_branch( - name="projects/my-app/branches/development" -).wait() -``` - -**Constraints:** -- Root branches (like `production`) cannot be reset (no parent) -- Branches with children cannot be reset (delete children first) -- Connections are temporarily interrupted during reset - -## Deleting a Branch - -```python -w.postgres.delete_branch( - name="projects/my-app/branches/development" -).wait() -``` - -**Constraints:** -- Cannot delete branches with child branches (delete children first) -- Cannot delete protected branches (remove protection first) -- Cannot delete the default branch - -## Branch Expiration - -Branch expiration sets an automatic deletion timestamp. Useful for: -- **CI/CD environments**: 2-4 hours -- **Demos**: 24-48 hours -- **Feature development**: 1-7 days -- **Long-term testing**: up to 30 days - -**Maximum expiration period:** 30 days from current time. - -### Expiration Restrictions - -- Cannot expire protected branches -- Cannot expire default branches -- Cannot expire branches that have children -- When a branch expires, all compute resources are also deleted - -## Best Practices - -1. **Use TTL for ephemeral branches**: Set expiration for dev/test branches to avoid accumulation -2. **Protect production branches**: Prevent accidental deletion or reset -3. **Reset instead of recreate**: Use reset from parent when you need fresh data without new branch overhead -4. **Schema diff before merge**: Compare schemas between branches before applying changes to production -5. **Monitor unarchived limit**: Only 10 unarchived branches are allowed per project diff --git a/.claude/skills/databricks-lakebase-autoscale/computes.md b/.claude/skills/databricks-lakebase-autoscale/computes.md deleted file mode 100644 index 0f53d50..0000000 --- a/.claude/skills/databricks-lakebase-autoscale/computes.md +++ /dev/null @@ -1,208 +0,0 @@ -# Lakebase Autoscaling Computes - -## Overview - -A compute is a virtualized service that runs Postgres for a branch. Each branch has one primary read-write compute and can have optional read replicas. Computes support autoscaling, scale-to-zero, and granular sizing from 0.5 to 112 CU. - -## Compute Sizing - -Each Compute Unit (CU) allocates approximately 2 GB of RAM. - -### Available Sizes - -| Category | Range | Notes | -|----------|-------|-------| -| **Autoscale computes** | 0.5-32 CU | Dynamic scaling within range (max-min <= 8 CU) | -| **Large fixed-size** | 36-112 CU | Fixed size, no autoscaling | - -### Representative Sizes - -| Compute Units | RAM | Max Connections | -|--------------|-----|-----------------| -| 0.5 CU | ~1 GB | 104 | -| 1 CU | ~2 GB | 209 | -| 4 CU | ~8 GB | 839 | -| 8 CU | ~16 GB | 1,678 | -| 16 CU | ~32 GB | 3,357 | -| 32 CU | ~64 GB | 4,000 | -| 64 CU | ~128 GB | 4,000 | -| 112 CU | ~224 GB | 4,000 | - -**Note:** Lakebase Provisioned used ~16 GB per CU. Autoscaling uses ~2 GB per CU for more granular scaling. - -## Creating a Compute - -```python -from databricks.sdk import WorkspaceClient -from databricks.sdk.service.postgres import Endpoint, EndpointSpec, EndpointType - -w = WorkspaceClient() - -# Create a read-write compute endpoint -result = w.postgres.create_endpoint( - parent="projects/my-app/branches/production", - endpoint=Endpoint( - spec=EndpointSpec( - endpoint_type=EndpointType.ENDPOINT_TYPE_READ_WRITE, - autoscaling_limit_min_cu=0.5, - autoscaling_limit_max_cu=4.0 - ) - ), - endpoint_id="my-compute" -).wait() - -print(f"Endpoint created: {result.name}") -print(f"Host: {result.status.hosts.host}") -``` - -### CLI - -```bash -databricks postgres create-endpoint \ - projects/my-app/branches/production my-compute \ - --json '{ - "spec": { - "endpoint_type": "ENDPOINT_TYPE_READ_WRITE", - "autoscaling_limit_min_cu": 0.5, - "autoscaling_limit_max_cu": 4.0 - } - }' -``` - -**Important:** Each branch can have only one read-write compute. - -## Getting Compute Details - -```python -endpoint = w.postgres.get_endpoint( - name="projects/my-app/branches/production/endpoints/my-compute" -) - -print(f"Endpoint: {endpoint.name}") -print(f"Type: {endpoint.status.endpoint_type}") -print(f"State: {endpoint.status.current_state}") -print(f"Host: {endpoint.status.hosts.host}") -print(f"Min CU: {endpoint.status.autoscaling_limit_min_cu}") -print(f"Max CU: {endpoint.status.autoscaling_limit_max_cu}") -``` - -## Listing Computes - -```python -endpoints = list(w.postgres.list_endpoints( - parent="projects/my-app/branches/production" -)) - -for ep in endpoints: - print(f"Endpoint: {ep.name}") - print(f" Type: {ep.status.endpoint_type}") - print(f" CU Range: {ep.status.autoscaling_limit_min_cu}-{ep.status.autoscaling_limit_max_cu}") -``` - -## Resizing a Compute - -Use `update_mask` to specify which fields to update: - -```python -from databricks.sdk.service.postgres import Endpoint, EndpointSpec, FieldMask - -# Update min and max CU -w.postgres.update_endpoint( - name="projects/my-app/branches/production/endpoints/my-compute", - endpoint=Endpoint( - name="projects/my-app/branches/production/endpoints/my-compute", - spec=EndpointSpec( - autoscaling_limit_min_cu=2.0, - autoscaling_limit_max_cu=8.0 - ) - ), - update_mask=FieldMask(field_mask=[ - "spec.autoscaling_limit_min_cu", - "spec.autoscaling_limit_max_cu" - ]) -).wait() -``` - -### CLI - -```bash -# Update single field -databricks postgres update-endpoint \ - projects/my-app/branches/production/endpoints/my-compute \ - spec.autoscaling_limit_max_cu \ - --json '{"spec": {"autoscaling_limit_max_cu": 8.0}}' - -# Update multiple fields -databricks postgres update-endpoint \ - projects/my-app/branches/production/endpoints/my-compute \ - "spec.autoscaling_limit_min_cu,spec.autoscaling_limit_max_cu" \ - --json '{"spec": {"autoscaling_limit_min_cu": 2.0, "autoscaling_limit_max_cu": 8.0}}' -``` - -## Deleting a Compute - -```python -w.postgres.delete_endpoint( - name="projects/my-app/branches/production/endpoints/my-compute" -).wait() -``` - -## Autoscaling - -Autoscaling dynamically adjusts compute resources based on workload demand. - -### Configuration - -- **Range:** 0.5-32 CU -- **Constraint:** Max - Min cannot exceed 8 CU -- **Valid examples:** 4-8 CU, 8-16 CU, 16-24 CU -- **Invalid example:** 0.5-32 CU (range of 31.5 CU) - -### Best Practices - -- Set minimum CU large enough to cache your working set in memory -- Performance may be degraded until compute scales up and caches data -- Connection limits are based on the maximum CU in the range - -## Scale-to-Zero - -Automatically suspends compute after a period of inactivity. - -| Setting | Description | -|---------|-------------| -| **Enabled** | Compute suspends after inactivity timeout (saves cost) | -| **Disabled** | Always-active compute (eliminates wake-up latency) | - -**Default behavior:** -- `production` branch: Scale-to-zero **disabled** (always active) -- Other branches: Scale-to-zero can be configured - -**Default inactivity timeout:** 5 minutes -**Minimum inactivity timeout:** 60 seconds - -### Wake-up Behavior - -When a connection arrives on a suspended compute: -1. Compute starts automatically (reactivation takes a few hundred milliseconds) -2. The connection request is handled transparently once active -3. Compute restarts at minimum autoscaling size (if autoscaling enabled) -4. Applications should implement connection retry logic for the brief reactivation period - -### Session Context After Reactivation - -When a compute suspends and reactivates, session context is **reset**: -- In-memory statistics and cache contents are cleared -- Temporary tables and prepared statements are lost -- Session-specific configuration settings reset -- Connection pools and active transactions are terminated - -If your application requires persistent session data, consider disabling scale-to-zero. - -## Sizing Guidance - -| Factor | Recommendation | -|--------|---------------| -| Query complexity | Complex analytical queries benefit from larger computes | -| Concurrent connections | More connections need more CPU and memory | -| Data volume | Larger datasets may need more memory for performance | -| Response time | Critical apps may require larger computes | diff --git a/.claude/skills/databricks-lakebase-autoscale/connection-patterns.md b/.claude/skills/databricks-lakebase-autoscale/connection-patterns.md deleted file mode 100644 index 398862b..0000000 --- a/.claude/skills/databricks-lakebase-autoscale/connection-patterns.md +++ /dev/null @@ -1,304 +0,0 @@ -# Lakebase Autoscaling Connection Patterns - -## Overview - -This document covers different connection patterns for Lakebase Autoscaling, from simple scripts to production applications with token refresh. - -## Authentication Methods - -Lakebase Autoscaling supports two authentication methods: - -| Method | Token Lifetime | Best For | -|--------|---------------|----------| -| **OAuth tokens** | 1 hour (must refresh) | Interactive sessions, workspace-integrated apps | -| **Native Postgres passwords** | No expiry | Long-running processes, tools without token rotation | - -**Connection timeouts (both methods):** -- **24-hour idle timeout**: Connections with no activity for 24 hours are automatically closed -- **3-day maximum connection life**: Connections alive for more than 3 days may be closed - -Design your applications to handle connection timeouts with retry logic. - -## Connection Methods - -### 1. Direct psycopg Connection (Simple Scripts) - -For one-off scripts or notebooks: - -```python -import psycopg -from databricks.sdk import WorkspaceClient - -def get_connection(project_id: str, branch_id: str = "production", - endpoint_id: str = None, database_name: str = "databricks_postgres"): - """Get a database connection with fresh OAuth token.""" - w = WorkspaceClient() - - # Get endpoint details to find the host - if endpoint_id: - ep_name = f"projects/{project_id}/branches/{branch_id}/endpoints/{endpoint_id}" - else: - # List endpoints and pick the primary R/W one - endpoints = list(w.postgres.list_endpoints( - parent=f"projects/{project_id}/branches/{branch_id}" - )) - ep_name = endpoints[0].name - - endpoint = w.postgres.get_endpoint(name=ep_name) - host = endpoint.status.hosts.host - - # Generate OAuth token (valid for 1 hour) - cred = w.postgres.generate_database_credential(endpoint=ep_name) - - # Build connection string - conn_string = ( - f"host={host} " - f"dbname={database_name} " - f"user={w.current_user.me().user_name} " - f"password={cred.token} " - f"sslmode=require" - ) - - return psycopg.connect(conn_string) - -# Usage -with get_connection("my-app") as conn: - with conn.cursor() as cur: - cur.execute("SELECT NOW()") - print(cur.fetchone()) -``` - -### 2. Connection Pool with Token Refresh (Production) - -For long-running applications that need connection pooling: - -```python -import asyncio -import uuid -from contextlib import asynccontextmanager -from typing import AsyncGenerator, Optional - -from sqlalchemy import event -from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker -from databricks.sdk import WorkspaceClient - - -class LakebaseAutoscaleConnectionManager: - """Manages Lakebase Autoscaling connections with automatic token refresh.""" - - def __init__( - self, - project_id: str, - branch_id: str = "production", - database_name: str = "databricks_postgres", - pool_size: int = 5, - max_overflow: int = 10, - token_refresh_seconds: int = 3000 # 50 minutes - ): - self.project_id = project_id - self.branch_id = branch_id - self.database_name = database_name - self.pool_size = pool_size - self.max_overflow = max_overflow - self.token_refresh_seconds = token_refresh_seconds - - self._current_token: Optional[str] = None - self._refresh_task: Optional[asyncio.Task] = None - self._engine = None - self._session_maker = None - - def _generate_token(self) -> str: - """Generate fresh OAuth token.""" - w = WorkspaceClient() - # Get primary endpoint name for token scoping - endpoints = list(w.postgres.list_endpoints( - parent=f"projects/{self.project_id}/branches/{self.branch_id}" - )) - endpoint_name = endpoints[0].name if endpoints else None - cred = w.postgres.generate_database_credential(endpoint=endpoint_name) - return cred.token - - def _get_host(self) -> str: - """Get the connection host from the primary endpoint.""" - w = WorkspaceClient() - endpoints = list(w.postgres.list_endpoints( - parent=f"projects/{self.project_id}/branches/{self.branch_id}" - )) - if not endpoints: - raise RuntimeError( - f"No endpoints found for projects/{self.project_id}/branches/{self.branch_id}" - ) - endpoint = w.postgres.get_endpoint(name=endpoints[0].name) - return endpoint.status.hosts.host - - async def _refresh_loop(self): - """Background task to refresh token periodically.""" - while True: - await asyncio.sleep(self.token_refresh_seconds) - try: - self._current_token = await asyncio.to_thread(self._generate_token) - except Exception as e: - print(f"Token refresh failed: {e}") - - def initialize(self): - """Initialize database engine and start token refresh.""" - w = WorkspaceClient() - - # Get host info - host = self._get_host() - username = w.current_user.me().user_name - - # Generate initial token - self._current_token = self._generate_token() - - # Create engine (password injected via event) - url = ( - f"postgresql+psycopg://{username}@" - f"{host}:5432/{self.database_name}" - ) - - self._engine = create_async_engine( - url, - pool_size=self.pool_size, - max_overflow=self.max_overflow, - pool_recycle=3600, - connect_args={"sslmode": "require"} - ) - - # Inject token on connect - @event.listens_for(self._engine.sync_engine, "do_connect") - def inject_token(dialect, conn_rec, cargs, cparams): - cparams["password"] = self._current_token - - self._session_maker = async_sessionmaker( - self._engine, - class_=AsyncSession, - expire_on_commit=False - ) - - def start_refresh(self): - """Start background token refresh task.""" - if not self._refresh_task: - self._refresh_task = asyncio.create_task(self._refresh_loop()) - - async def stop_refresh(self): - """Stop token refresh task.""" - if self._refresh_task: - self._refresh_task.cancel() - try: - await self._refresh_task - except asyncio.CancelledError: - pass - self._refresh_task = None - - @asynccontextmanager - async def session(self) -> AsyncGenerator[AsyncSession, None]: - """Get a database session.""" - async with self._session_maker() as session: - yield session - - async def close(self): - """Close all connections.""" - await self.stop_refresh() - if self._engine: - await self._engine.dispose() - - -# Usage in FastAPI -from fastapi import FastAPI - -app = FastAPI() -db_manager = LakebaseAutoscaleConnectionManager("my-app", "production", "my_database") - -@app.on_event("startup") -async def startup(): - db_manager.initialize() - db_manager.start_refresh() - -@app.on_event("shutdown") -async def shutdown(): - await db_manager.close() - -@app.get("/data") -async def get_data(): - async with db_manager.session() as session: - result = await session.execute("SELECT * FROM my_table") - return result.fetchall() -``` - -### 3. Static URL Mode (Local Development) - -For local development, use a static connection URL: - -```python -import os -from sqlalchemy.ext.asyncio import create_async_engine - -# Set environment variable with full connection URL -# LAKEBASE_PG_URL=postgresql://user:password@host:5432/database - -def get_database_url() -> str: - """Get database URL from environment.""" - url = os.environ.get("LAKEBASE_PG_URL") - if url and url.startswith("postgresql://"): - # Convert to psycopg3 async driver - url = url.replace("postgresql://", "postgresql+psycopg://", 1) - return url - -engine = create_async_engine( - get_database_url(), - pool_size=5, - connect_args={"sslmode": "require"} -) -``` - -### 4. DNS Resolution Workaround (macOS) - -Python's `socket.getaddrinfo()` fails with long hostnames on macOS. Use `dig` as fallback: - -```python -import subprocess -import socket - -def resolve_hostname(hostname: str) -> str: - """Resolve hostname using dig command (macOS workaround).""" - try: - return socket.gethostbyname(hostname) - except socket.gaierror: - pass - - try: - result = subprocess.run( - ["dig", "+short", hostname], - capture_output=True, text=True, timeout=5 - ) - ips = result.stdout.strip().split('\n') - for ip in ips: - if ip and not ip.startswith(';'): - return ip - except Exception: - pass - - raise RuntimeError(f"Could not resolve hostname: {hostname}") - -# Use with psycopg -conn_params = { - "host": hostname, # For TLS SNI - "hostaddr": resolve_hostname(hostname), # Actual IP - "dbname": database_name, - "user": username, - "password": token, - "sslmode": "require" -} -conn = psycopg.connect(**conn_params) -``` - -## Best Practices - -1. **Always use SSL**: Set `sslmode=require` in all connections -2. **Implement token refresh**: Tokens expire after 1 hour; refresh at 50 minutes -3. **Use connection pooling**: Avoid creating new connections per request -4. **Handle DNS issues on macOS**: Use the `hostaddr` workaround if needed -5. **Close connections properly**: Use context managers or explicit cleanup -6. **Handle scale-to-zero wake-up**: First connection after idle may take 2-5 seconds -7. **Log token refresh events**: Helps debug authentication issues diff --git a/.claude/skills/databricks-lakebase-autoscale/connections.md b/.claude/skills/databricks-lakebase-autoscale/connections.md new file mode 100644 index 0000000..0831a78 --- /dev/null +++ b/.claude/skills/databricks-lakebase-autoscale/connections.md @@ -0,0 +1,212 @@ +# Lakebase Autoscaling connection patterns + +Order of preference: + +1. **Canonical:** `psycopg_pool.ConnectionPool` + `OAuthConnection` subclass + `max_lifetime=2700`. +2. **SQLAlchemy:** official `do_connect` auth hook; optionally rely on `pool_recycle`/`dispose()` rather than a background token loop. +3. **Direct `psycopg.connect`:** notebooks/one-shot scripts under 1 hour. +4. **Static Postgres URL/native password:** local/dev tools only, or tools unable to rotate OAuth credentials. + +## Authentication facts + +Lakebase OAuth database credentials: +- Mint with `WorkspaceClient().postgres.generate_database_credential(endpoint=...)`. +- Use `cred.token` as the Postgres password. +- Expire after about 1 hour. +- Expiry is enforced at login; already-open connections continue until closed by pool/platform timeouts. + +Critical warning: + +```python +# ✅ Lakebase-scoped credential: works for Postgres login +cred = w.postgres.generate_database_credential(endpoint=endpoint_name) +password = cred.token + +# ❌ Workspace-scoped token: fails at Postgres login +password = w.config.oauth_token().access_token +# also do not use WorkspaceClient().config.token +``` + +Always connect with `sslmode=require`. + +## 1. Canonical: psycopg pool + OAuthConnection + +Use for production Databricks Apps and most Python services. + +Key mechanics: +- The pool calls `OAuthConnection.connect()` whenever it opens a physical connection: initial fill, growth under load, recycle, replacement after failure. +- `connect()` mints a fresh Lakebase token just-in-time and injects it as `password`. +- `max_lifetime=2700` recycles physical connections after 45 minutes, before 1-hour token expiry. +- No background refresh thread/task is needed. + +Minimal skeleton: + +```python +import os +import psycopg +from psycopg_pool import ConnectionPool +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient() + +class OAuthConnection(psycopg.Connection): + @classmethod + def connect(cls, conninfo="", **kwargs): + cred = w.postgres.generate_database_credential( + endpoint=os.environ["ENDPOINT_NAME"] + ) + kwargs["password"] = cred.token + return super().connect(conninfo, **kwargs) + +pool = ConnectionPool( + conninfo=( + f"dbname={os.environ['PGDATABASE']} " + f"user={os.environ['PGUSER']} " + f"host={os.environ['PGHOST']} " + f"port={os.environ.get('PGPORT', '5432')} " + f"sslmode={os.environ.get('PGSSLMODE', 'require')}" + ), + connection_class=OAuthConnection, + min_size=1, + max_size=10, + max_lifetime=2700, + open=True, +) +``` + +Prefer `2700`; it is a defensive convention. The official Databricks tutorial leaves `max_lifetime` unset; `databricks-ai-bridge` uses `2700`. + +For FastAPI or explicit startup: +- instantiate with `open=False` +- call `pool.open(wait=True, timeout=30.0)` in lifespan/startup +- call `pool.close()` on shutdown + +This also avoids relying on implicit open behavior. + +## Databricks Apps environment variables + +When adding a Lakebase/Postgres resource to a Databricks App, these are auto-injected for the **first** DB resource: + +```text +PGAPPNAME +PGHOST +PGPORT +PGDATABASE +PGUSER +PGSSLMODE +``` + +Gotchas: +- `PGUSER` is typically the app service principal client ID. +- Only the first database resource is auto-injected; additional resources need explicit `valueFrom`. +- `ENDPOINT_NAME` is **not** auto-injected. Add it manually because `generate_database_credential(endpoint=...)` requires the full endpoint path: + +```yaml +env: + - name: ENDPOINT_NAME + value: "projects//branches//endpoints/" +``` + +## 2. SQLAlchemy: official `do_connect` hook + +Use when the app is already built around SQLAlchemy. + +Important distinction: +- `do_connect` is the official Databricks-recommended SQLAlchemy credential injection hook and is used by `databricks-ai-bridge`. +- The community/extra-complexity variant is adding a background `asyncio.Task` token-refresh loop. Demote that loop, not `do_connect`. + +Recommended hook shape: + +```python +from sqlalchemy import event +from sqlalchemy.ext.asyncio import create_async_engine +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient() +endpoint_name = "projects/my-app/branches/production/endpoints/ep-primary" +host = w.postgres.get_endpoint(name=endpoint_name).status.hosts.host +user = w.current_user.me().user_name + +engine = create_async_engine( + f"postgresql+psycopg://{user}@{host}:5432/databricks_postgres", + connect_args={"sslmode": "require"}, + pool_recycle=2700, +) + +@event.listens_for(engine.sync_engine, "do_connect") +def inject_lakebase_token(dialect, conn_rec, cargs, cparams): + cred = w.postgres.generate_database_credential(endpoint=endpoint_name) + cparams["password"] = cred.token +``` + +Notes: +- `do_connect` fires when SQLAlchemy opens a new DBAPI connection. +- `pool_recycle=2700` approximates the psycopg-pool pattern. +- If you need deterministic refresh, prefer scheduled `engine.dispose()` and let the next checkout re-open with `do_connect`. +- A background token cache/refresh task is optional complexity and can create stale-token races if implemented poorly. + +## 3. Direct psycopg for notebooks/scripts + +Only for short-lived sessions where connections are opened and used immediately. + +Recipe: +1. Build endpoint path. +2. `get_endpoint(...).status.hosts.host`. +3. `generate_database_credential(endpoint=endpoint_name)`. +4. `psycopg.connect(host=host, dbname="databricks_postgres", user=, password=cred.token, sslmode="require")`. + +Use `w.current_user.me().user_name` for user in notebooks/manual scripts. In Databricks Apps, prefer `PGUSER`. + +## 4. Static URL / native password + +Use only for local development, legacy tools, or clients that cannot rotate OAuth database credentials. For SQLAlchemy + psycopg3, normalize: + +```text +postgresql://... -> postgresql+psycopg://... +``` + +Still set `sslmode=require`. + +## Endpoint discovery + +Avoid hardcoding host if you can hardcode the endpoint name instead: + +```python +ep = w.postgres.get_endpoint( + name="projects/my-app/branches/production/endpoints/ep-primary" +) +host = ep.status.hosts.host +``` + +If no endpoint ID is known, list under branch and choose deliberately: + +```python +endpoints = list(w.postgres.list_endpoints( + parent="projects/my-app/branches/production" +)) +``` + +Do not assume the first endpoint is the primary if read replicas exist; check endpoint type/status. + +## DNS workaround for macOS + +Some macOS/Python resolver combinations fail on long Lakebase hostnames. + +Workaround: +- Resolve the hostname externally, commonly with `dig +short `. +- Pass both: + - `host=` for TLS/SNI/certificate validation. + - `hostaddr=` for the actual TCP connection. + +psycopg3 supports `hostaddr`. + +## Timeouts, scale-to-zero, and retries + +Plan for: +- 1-hour Lakebase OAuth token lifetime at login. +- 24-hour idle connection timeout. +- 3-day maximum connection lifetime. +- Scale-to-zero wake-up latency; first connection/query after suspension may need retry/backoff. +- After suspension/reactivation: session context is reset, temp tables/prepared statements are gone, active transactions/connections are terminated. + +Use context managers so pooled connections return promptly. diff --git a/.claude/skills/databricks-lakebase-autoscale/operations.md b/.claude/skills/databricks-lakebase-autoscale/operations.md new file mode 100644 index 0000000..982bfb5 --- /dev/null +++ b/.claude/skills/databricks-lakebase-autoscale/operations.md @@ -0,0 +1,297 @@ +# Lakebase Autoscaling operations + +Use `WorkspaceClient().postgres` for Autoscaling projects, branches, endpoints, roles, and credentials. Most create/update/delete methods return long-running operations; call `.wait()`. + +```python +from databricks.sdk import WorkspaceClient +w = WorkspaceClient() +``` + +## Resource names + +```text +Project: projects/{project_id} +Branch: projects/{project_id}/branches/{branch_id} +Endpoint: projects/{project_id}/branches/{branch_id}/endpoints/{endpoint_id} +``` + +Project ID rules: +- 1–63 chars +- lowercase letters, digits, hyphens +- cannot start/end with hyphen +- immutable after creation + +Default database: `databricks_postgres`. + +## Projects + +Create: + +```python +from databricks.sdk.service.postgres import Project, ProjectSpec + +project = w.postgres.create_project( + project=Project(spec=ProjectSpec(display_name="My App", pg_version="17")), + project_id="my-app", +).wait() +``` + +Project defaults: +- `production` branch +- primary read-write endpoint +- `databricks_postgres` database +- role for creator’s Databricks identity +- production scale-to-zero disabled by default + +GET gotcha: effective properties are typically in `project.status`, not `project.spec`. + +Update requires `FieldMask`: + +```python +from databricks.sdk.service.postgres import FieldMask + +w.postgres.update_project( + name="projects/my-app", + project=Project( + name="projects/my-app", + spec=ProjectSpec(display_name="New Name"), + ), + update_mask=FieldMask(field_mask=["spec.display_name"]), +).wait() +``` + +Delete is destructive and permanent; delete dependent Unity Catalog catalogs/synced tables first where applicable: + +```python +w.postgres.delete_project(name="projects/my-app").wait() +``` + +## Branches + +Branches are copy-on-write isolated database environments. Use them for dev/test/staging, schema-change validation, point-in-time recovery workflows, and ephemeral CI. + +Create branch from current parent: + +```python +from databricks.sdk.service.postgres import Branch, BranchSpec, Duration + +branch = w.postgres.create_branch( + parent="projects/my-app", + branch=Branch(spec=BranchSpec( + source_branch="projects/my-app/branches/production", + ttl=Duration(seconds=604800), # or no_expiry=True + )), + branch_id="development", +).wait() +``` + +Keep: +- `ttl=Duration(seconds=...)` for ephemeral branches. +- `no_expiry=True` for permanent branches. +- Max expiration period: 30 days from current time. +- Only 10 unarchived branches per project. +- Protected branches cannot be deleted, reset, archived, or expired. +- Default branch cannot be deleted or expired. +- Branches with children cannot be deleted, reset, or expired; delete children first. +- Reset replaces branch data/schema with latest parent and interrupts connections. + +Protect production: + +```python +w.postgres.update_branch( + name="projects/my-app/branches/production", + branch=Branch( + name="projects/my-app/branches/production", + spec=BranchSpec(is_protected=True), + ), + update_mask=FieldMask(field_mask=["spec.is_protected"]), +).wait() +``` + +Reset/delete: + +```python +w.postgres.reset_branch(name="projects/my-app/branches/development").wait() +w.postgres.delete_branch(name="projects/my-app/branches/development").wait() +``` + +Branch status fields worth inspecting: +- `status.default` +- `status.is_protected` +- `status.current_state` +- `status.logical_size_bytes` +- `status.expire_time` + +## Endpoints / computes + +A compute endpoint runs Postgres for a branch. Each branch has at most one primary read-write endpoint and may have read-only replica endpoints. + +Create endpoint: + +```python +from databricks.sdk.service.postgres import Endpoint, EndpointSpec, EndpointType + +ep = w.postgres.create_endpoint( + parent="projects/my-app/branches/production", + endpoint=Endpoint(spec=EndpointSpec( + endpoint_type=EndpointType.ENDPOINT_TYPE_READ_WRITE, + autoscaling_limit_min_cu=0.5, + autoscaling_limit_max_cu=4.0, + )), + endpoint_id="ep-primary", +).wait() +``` + +Get host: + +```python +host = w.postgres.get_endpoint( + name="projects/my-app/branches/production/endpoints/ep-primary" +).status.hosts.host +``` + +Resize with update mask: + +```python +w.postgres.update_endpoint( + name="projects/my-app/branches/production/endpoints/ep-primary", + endpoint=Endpoint( + name="projects/my-app/branches/production/endpoints/ep-primary", + spec=EndpointSpec( + autoscaling_limit_min_cu=2.0, + autoscaling_limit_max_cu=8.0, + ), + ), + update_mask=FieldMask(field_mask=[ + "spec.autoscaling_limit_min_cu", + "spec.autoscaling_limit_max_cu", + ]), +).wait() +``` + +Delete: + +```python +w.postgres.delete_endpoint( + name="projects/my-app/branches/production/endpoints/ep-primary" +).wait() +``` + +## Compute sizing + +Autoscaling uses ~2 GB RAM per CU. + +| CU | Approx RAM | Max connections | +|---:|---:|---:| +| 0.5 | ~1 GB | 104 | +| 1 | ~2 GB | 209 | +| 4 | ~8 GB | 839 | +| 8 | ~16 GB | 1,678 | +| 16 | ~32 GB | 3,357 | +| 32 | ~64 GB | 4,000 | +| 64 | ~128 GB | 4,000 | +| 112 | ~224 GB | 4,000 | + +Rules: +- Autoscale range: 0.5–32 CU. +- `autoscaling_limit_max_cu - autoscaling_limit_min_cu <= 16`. +- Valid: 4–20, 8–16, 16–32. +- Invalid: 0.5–32 (spread of 31.5 exceeds 16). +- Fixed-size always-on computes: 40–112 CU; no autoscaling. +- Connection limit is based on max CU. +- Set min CU high enough for working-set cache and latency needs. + +## Scale-to-zero + +Defaults: +- `production`: disabled by default. +- Other branches: configurable. +- Default inactivity timeout: 5 minutes. +- Minimum inactivity timeout: 60 seconds. + +Wake-up: +- First connection wakes compute automatically. +- Apps should use retry/backoff for the brief reactivation period. +- Reactivated compute starts at minimum autoscaling size. + +Session reset after suspension: +- temp tables gone +- prepared statements gone +- in-memory stats/cache cleared +- session settings reset +- active transactions/connections terminated + +Disable scale-to-zero for latency-critical apps or apps relying on persistent session state. + +## Project limits + +| Resource | Limit | +|---|---:| +| Projects per workspace | 1000 | +| Branches per project | 500 | +| Unarchived branches | 10 | +| Root branches | 3 | +| Protected branches | 1 | +| Concurrently active computes | 20 | +| Postgres roles per branch | 500 | +| Postgres databases per branch | 500 | +| Logical data size per branch | 8 TB | +| Snapshots | 10 | +| Maximum history retention | 35 days | +| Minimum scale-to-zero time | 60 sec | + +## CLI names + +CLI mirrors the SDK under `databricks postgres`, for example: +- `create-project`, `get-project`, `list-projects`, `update-project`, `delete-project` +- `create-branch`, `list-branches`, `reset-branch`, `delete-branch` +- `create-endpoint`, `get-endpoint`, `list-endpoints`, `update-endpoint`, `delete-endpoint` + +## MCP tools + +Use `type="autoscale"` for Lakebase Autoscaling. + +### `manage_lakebase_database` + +Actions: +- `create_or_update`: requires `name`; useful params include `display_name`, `pg_version` +- `get`: requires `name` +- `list`: optional type filter +- `delete`: requires `name` + +Example intent: + +```python +manage_lakebase_database( + action="create_or_update", + name="my-app", + type="autoscale", + display_name="My Application", + pg_version="17", +) +``` + +### `manage_lakebase_branch` + +Actions: +- `create_or_update`: requires `project_name`, `branch_id` +- `delete`: requires full branch `name` + +Useful params: +- `source_branch` +- `ttl_seconds` +- `autoscaling_limit_min_cu` +- `autoscaling_limit_max_cu` +- `scale_to_zero_seconds` + +### `generate_lakebase_credential` + +Generate a Lakebase-scoped database credential: + +```python +generate_lakebase_credential( + endpoint="projects/my-app/branches/production/endpoints/ep-primary" +) +``` + +Use returned token as the Postgres password with `sslmode=require`. diff --git a/.claude/skills/databricks-lakebase-autoscale/projects.md b/.claude/skills/databricks-lakebase-autoscale/projects.md deleted file mode 100644 index 659207a..0000000 --- a/.claude/skills/databricks-lakebase-autoscale/projects.md +++ /dev/null @@ -1,204 +0,0 @@ -# Lakebase Autoscaling Projects - -## Overview - -A project is the top-level container for Lakebase Autoscaling resources, including branches, computes, databases, and roles. Each project is isolated and contains its own Postgres version, compute defaults, and restore window settings. - -## Project Structure - -``` -Project - └── Branches (production, development, staging, etc.) - ├── Computes (R/W compute, read replicas) - ├── Roles (Postgres roles) - └── Databases (Postgres databases) -``` - -When a project is created, it includes by default: -- A `production` branch (the default branch) -- A primary read-write compute (8-32 CU, autoscaling enabled, scale-to-zero disabled) -- A `databricks_postgres` database -- A Postgres role for the creating user's Databricks identity - -## Resource Naming - -Projects follow a hierarchical naming convention: -``` -projects/{project_id} -``` - -**Resource ID requirements:** -- 1-63 characters long -- Lowercase letters, digits, and hyphens only -- Cannot start or end with a hyphen -- Cannot be changed after creation - -## Creating a Project - -### Python SDK - -```python -from databricks.sdk import WorkspaceClient -from databricks.sdk.service.postgres import Project, ProjectSpec - -w = WorkspaceClient() - -# Create a project (long-running operation) -operation = w.postgres.create_project( - project=Project( - spec=ProjectSpec( - display_name="My Application", - pg_version="17" - ) - ), - project_id="my-app" -) - -# Wait for completion -result = operation.wait() -print(f"Created project: {result.name}") -print(f"Display name: {result.status.display_name}") -print(f"Postgres version: {result.status.pg_version}") -``` - -### CLI - -```bash -databricks postgres create-project \ - --project-id my-app \ - --json '{ - "spec": { - "display_name": "My Application", - "pg_version": "17" - } - }' -``` - -## Getting Project Details - -### Python SDK - -```python -project = w.postgres.get_project(name="projects/my-app") - -print(f"Project: {project.name}") -print(f"Display name: {project.status.display_name}") -print(f"Postgres version: {project.status.pg_version}") -``` - -### CLI - -```bash -databricks postgres get-project projects/my-app -``` - -**Note:** The `spec` field is not populated for GET operations. All properties are returned in the `status` field. - -## Listing Projects - -```python -projects = w.postgres.list_projects() - -for project in projects: - print(f"Project: {project.name}") - print(f" Display name: {project.status.display_name}") - print(f" Postgres version: {project.status.pg_version}") -``` - -## Updating a Project - -Updates require an `update_mask` specifying which fields to modify: - -```python -from databricks.sdk.service.postgres import Project, ProjectSpec, FieldMask - -# Update display name -operation = w.postgres.update_project( - name="projects/my-app", - project=Project( - name="projects/my-app", - spec=ProjectSpec( - display_name="My Updated Application" - ) - ), - update_mask=FieldMask(field_mask=["spec.display_name"]) -) -result = operation.wait() -``` - -### CLI - -```bash -databricks postgres update-project projects/my-app spec.display_name \ - --json '{ - "spec": { - "display_name": "My Updated Application" - } - }' -``` - -## Deleting a Project - -**WARNING:** Deleting a project is permanent and also deletes all branches, computes, databases, roles, and data. - -Delete all Unity Catalog catalogs and synced tables before deleting the project. - -```python -operation = w.postgres.delete_project(name="projects/my-app") -# This is a long-running operation -``` - -### CLI - -```bash -databricks postgres delete-project projects/my-app -``` - -## Project Settings - -### Compute Defaults - -Default settings for new primary computes: -- Compute size range (0.5-112 CU) -- Scale-to-zero timeout (default: 5 minutes) - -### Instant Restore - -Configure the restore window length (2-35 days). Longer windows increase storage costs. - -### Postgres Version - -Supports Postgres 16 and Postgres 17. - -## Project Limits - -| Resource | Limit | -|----------|-------| -| Concurrently active computes | 20 | -| Branches per project | 500 | -| Postgres roles per branch | 500 | -| Postgres databases per branch | 500 | -| Logical data size per branch | 8 TB | -| Projects per workspace | 1000 | -| Protected branches | 1 | -| Root branches | 3 | -| Unarchived branches | 10 | -| Snapshots | 10 | -| Maximum history retention | 35 days | -| Minimum scale-to-zero time | 60 seconds | - -## Long-Running Operations - -All create, update, and delete operations return a long-running operation (LRO). Use `.wait()` in the SDK to block until completion: - -```python -# Start operation -operation = w.postgres.create_project(...) - -# Wait for completion -result = operation.wait() - -# Or check status manually -op_status = w.postgres.get_operation(name=operation.name) -print(f"Done: {op_status.done}") -``` diff --git a/.claude/skills/databricks-lakebase-autoscale/reverse-etl.md b/.claude/skills/databricks-lakebase-autoscale/reverse-etl.md index f983eeb..949f91b 100644 --- a/.claude/skills/databricks-lakebase-autoscale/reverse-etl.md +++ b/.claude/skills/databricks-lakebase-autoscale/reverse-etl.md @@ -1,56 +1,59 @@ -# Reverse ETL with Lakebase Autoscaling +# Reverse ETL / synced tables -## Overview +Reverse ETL syncs Unity Catalog Delta tables into Lakebase Autoscaling as PostgreSQL tables for OLTP access. -Reverse ETL allows you to sync data from Unity Catalog Delta tables into Lakebase Autoscaling as PostgreSQL tables. This enables OLTP access patterns on data processed in the Lakehouse. +Important namespace split: +- Lakebase Autoscaling infrastructure: `w.postgres` +- Synced tables: `w.database` -## How It Works +Reverse ETL is Delta-to-Postgres only; Postgres-to-Delta sync is not supported here. -Synced tables create a managed copy of Unity Catalog data in Lakebase: +## How synced tables work -1. A new Unity Catalog table (read-only, managed by the sync pipeline) -2. A Postgres table in Lakebase (queryable by applications) +A synced table creates/maintains: +1. A managed/read-only Unity Catalog table for pipeline state/output. +2. A PostgreSQL table in Lakebase queried by apps. -The sync pipeline uses managed Lakeflow Spark Declarative Pipelines to continuously update both tables. +The sync pipeline uses managed Lakeflow Spark Declarative Pipelines. -### Performance +Performance planning: +- Continuous writes: ~1,200 rows/sec per CU. +- Bulk writes: ~15,000 rows/sec per CU. +- Each synced table can use up to 16 Postgres connections. -- **Continuous writes:** ~1,200 rows/sec per CU -- **Bulk writes:** ~15,000 rows/sec per CU -- **Connections used:** Up to 16 per synced table +## Sync modes -## Sync Modes +| Mode | Behavior | Use when | CDF required | +|---|---|---|---| +| `SNAPSHOT` | one-time full copy | initial loads, historical copy, large replacement | no | +| `TRIGGERED` | scheduled/on-demand incremental updates | hourly/daily operational refresh | yes | +| `CONTINUOUS` | streaming updates, seconds latency | live applications | yes | -| Mode | Description | Best For | Notes | -|------|-------------|----------|-------| -| **Snapshot** | One-time full copy | Initial setup, historical analysis | 10x more efficient if modifying >10% of data | -| **Triggered** | Scheduled updates on demand | Dashboards updated hourly/daily | Requires CDF on source table | -| **Continuous** | Real-time streaming (seconds of latency) | Live applications | Highest cost, minimum 15s intervals, requires CDF | - -**Note:** Triggered and Continuous modes require Change Data Feed (CDF) enabled on the source table: +Triggered and Continuous require Delta Change Data Feed on the source table: ```sql -ALTER TABLE your_catalog.your_schema.your_table -SET TBLPROPERTIES (delta.enableChangeDataFeed = true) +ALTER TABLE catalog.schema.table +SET TBLPROPERTIES (delta.enableChangeDataFeed = true); ``` -## Creating Synced Tables +Snapshot can be more efficient when modifying >10% of the data. + +## Create a synced table -### Using Python SDK +Use `databricks.sdk.service.database` models: ```python from databricks.sdk import WorkspaceClient from databricks.sdk.service.database import ( - SyncedDatabaseTable, - SyncedTableSpec, NewPipelineSpec, + SyncedDatabaseTable, SyncedTableSchedulingPolicy, + SyncedTableSpec, ) w = WorkspaceClient() -# Create a synced table -synced_table = w.database.create_synced_database_table( +w.database.create_synced_database_table( SyncedDatabaseTable( name="lakebase_catalog.schema.synced_table", spec=SyncedTableSpec( @@ -59,55 +62,35 @@ synced_table = w.database.create_synced_database_table( scheduling_policy=SyncedTableSchedulingPolicy.TRIGGERED, new_pipeline_spec=NewPipelineSpec( storage_catalog="lakebase_catalog", - storage_schema="staging" - ) + storage_schema="staging", + ), ), ) ) -print(f"Created synced table: {synced_table.name}") ``` -### Using CLI - -```bash -databricks database create-synced-database-table \ - --json '{ - "name": "lakebase_catalog.schema.synced_table", - "spec": { - "source_table_full_name": "analytics.gold.user_profiles", - "primary_key_columns": ["user_id"], - "scheduling_policy": "TRIGGERED", - "new_pipeline_spec": { - "storage_catalog": "lakebase_catalog", - "storage_schema": "staging" - } - } - }' -``` - -## Checking Synced Table Status +Status: ```python -status = w.database.get_synced_database_table(name="lakebase_catalog.schema.synced_table") -print(f"State: {status.data_synchronization_status.detailed_state}") -print(f"Message: {status.data_synchronization_status.message}") +st = w.database.get_synced_database_table( + name="lakebase_catalog.schema.synced_table" +) +state = st.data_synchronization_status.detailed_state +message = st.data_synchronization_status.message ``` -## Deleting a Synced Table - -Delete from both Unity Catalog and Postgres: - -1. **Unity Catalog:** Delete from Catalog Explorer or SDK -2. **Postgres:** Drop the table to free storage +Deletion cleanup: +1. Delete the synced table / UC object. +2. Drop the Postgres target table if needed to free Lakebase storage. ```sql -DROP TABLE your_database.your_schema.your_table; +DROP TABLE schema.table; ``` -## Data Type Mapping +## Type mapping -| Unity Catalog Type | Postgres Type | -|-------------------|---------------| +| Unity Catalog | Postgres | +|---|---| | BIGINT | BIGINT | | BINARY | BYTEA | | BOOLEAN | BOOLEAN | @@ -126,52 +109,19 @@ DROP TABLE your_database.your_schema.your_table; | MAP | JSONB | | STRUCT | JSONB | -**Unsupported types:** GEOGRAPHY, GEOMETRY, VARIANT, OBJECT - -## Capacity Planning - -- **Connection usage:** Each synced table uses up to 16 connections -- **Size limits:** 2 TB total across all synced tables; recommend < 1 TB per table -- **Naming:** Database, schema, and table names only allow `[A-Za-z0-9_]+` -- **Schema evolution:** Only additive changes (e.g., adding columns) for Triggered/Continuous modes - -## Use Cases - -### Product Catalog for Web App - -```python -w.database.create_synced_database_table( - SyncedDatabaseTable( - name="ecommerce_catalog.public.products", - spec=SyncedTableSpec( - source_table_full_name="gold.products.catalog", - primary_key_columns=["product_id"], - scheduling_policy=SyncedTableSchedulingPolicy.TRIGGERED, - ), - ) -) -``` - -### Real-time Feature Serving - -```python -w.database.create_synced_database_table( - SyncedDatabaseTable( - name="ml_catalog.public.user_features", - spec=SyncedTableSpec( - source_table_full_name="ml.features.user_features", - primary_key_columns=["user_id"], - scheduling_policy=SyncedTableSchedulingPolicy.CONTINUOUS, - ), - ) -) -``` - -## Best Practices - -1. **Enable CDF** on source tables before creating Triggered or Continuous synced tables -2. **Choose appropriate sync mode**: Snapshot for small tables, Triggered for hourly/daily, Continuous for real-time -3. **Monitor sync status**: Check for failures and latency via Catalog Explorer -4. **Index target tables**: Create appropriate indexes in Postgres for your query patterns -5. **Handle schema changes**: Only additive changes are supported for streaming modes -6. **Account for connection limits**: Each synced table uses up to 16 connections +Unsupported: +- `GEOGRAPHY` +- `GEOMETRY` +- `VARIANT` +- `OBJECT` + +## Limits and gotchas + +- Up to 16 Postgres connections per synced table; include this in endpoint connection-capacity planning. +- Size limit: 2 TB total across all synced tables. +- Recommended: <1 TB per synced table. +- Database/schema/table names: `[A-Za-z0-9_]+`. +- Triggered/Continuous schema evolution: additive changes only. +- Create indexes in Postgres for application query patterns after sync. +- Monitor detailed sync state in Catalog Explorer or with `get_synced_database_table`. +- Delete synced-table dependencies before deleting the Lakebase project. diff --git a/.claude/skills/databricks-lakebase-provisioned/SKILL.md b/.claude/skills/databricks-lakebase-provisioned/SKILL.md index b2b404a..846f644 100644 --- a/.claude/skills/databricks-lakebase-provisioned/SKILL.md +++ b/.claude/skills/databricks-lakebase-provisioned/SKILL.md @@ -1,6 +1,6 @@ --- name: databricks-lakebase-provisioned -description: "Patterns and best practices for using Lakebase Provisioned (Databricks managed PostgreSQL) for OLTP workloads." +description: "Patterns and best practices for Lakebase Provisioned (Databricks managed PostgreSQL) for OLTP workloads. Use when creating Lakebase instances, connecting applications or Databricks Apps to PostgreSQL, implementing reverse ETL via synced tables, storing agent or chat memory, or configuring OAuth authentication for Lakebase." --- # Lakebase Provisioned @@ -225,21 +225,65 @@ mlflow.langchain.log_model( The following MCP tools are available for managing Lakebase infrastructure. Use `type="provisioned"` for Lakebase Provisioned. -### Database Management +### manage_lakebase_database - Database Management -| Tool | Description | -|------|-------------| -| `create_or_update_lakebase_database` | Create or update a database. Finds by name, creates if new, updates if existing. Use `type="provisioned"`, `capacity` (CU_1-CU_8), `stopped` params. | -| `get_lakebase_database` | Get database details or list all. Pass `name` to get one, omit to list all. Use `type="provisioned"` to filter. | -| `delete_lakebase_database` | Delete a database and its resources. Use `type="provisioned"`, `force=True` to cascade. | -| `generate_lakebase_credential` | Generate OAuth token for PostgreSQL connections (1-hour expiry). Pass `instance_names` for provisioned. | +| Action | Description | Required Params | +|--------|-------------|-----------------| +| `create_or_update` | Create or update a database | name | +| `get` | Get database details | name | +| `list` | List all databases | (none, optional type filter) | +| `delete` | Delete database and resources | name | -### Reverse ETL (Catalog + Synced Tables) +**Example usage:** +```python +# Create a provisioned database +manage_lakebase_database( + action="create_or_update", + name="my-lakebase-instance", + type="provisioned", + capacity="CU_1" +) + +# Get database details +manage_lakebase_database(action="get", name="my-lakebase-instance", type="provisioned") + +# List all databases +manage_lakebase_database(action="list") + +# Delete with cascade +manage_lakebase_database(action="delete", name="my-lakebase-instance", type="provisioned", force=True) +``` + +### manage_lakebase_sync - Reverse ETL + +| Action | Description | Required Params | +|--------|-------------|-----------------| +| `create_or_update` | Set up reverse ETL from Delta to Lakebase | instance_name, source_table_name, target_table_name | +| `delete` | Remove synced table (and optionally catalog) | table_name | -| Tool | Description | -|------|-------------| -| `create_or_update_lakebase_sync` | Set up reverse ETL: ensures UC catalog registration exists, then creates a synced table from Delta to Lakebase. Params: `instance_name`, `source_table_name`, `target_table_name`, `scheduling_policy` ("TRIGGERED"/"SNAPSHOT"/"CONTINUOUS"). | -| `delete_lakebase_sync` | Remove a synced table and optionally its UC catalog registration. | +**Example usage:** +```python +# Set up reverse ETL +manage_lakebase_sync( + action="create_or_update", + instance_name="my-lakebase-instance", + source_table_name="catalog.schema.delta_table", + target_table_name="lakebase_catalog.schema.postgres_table", + scheduling_policy="TRIGGERED" # or SNAPSHOT, CONTINUOUS +) + +# Delete synced table +manage_lakebase_sync(action="delete", table_name="lakebase_catalog.schema.postgres_table") +``` + +### generate_lakebase_credential - OAuth Tokens + +Generate OAuth token (~1hr) for PostgreSQL connections. Use as password with `sslmode=require`. + +```python +# For provisioned instances +generate_lakebase_credential(instance_names=["my-lakebase-instance"]) +``` ## Reference Files @@ -302,7 +346,7 @@ databricks database start-database-instance --name my-lakebase-instance ## Related Skills - **[databricks-app-apx](../databricks-app-apx/SKILL.md)** - full-stack apps that can use Lakebase for persistence -- **[databricks-app-python](../databricks-app-python/SKILL.md)** - Python apps with Lakebase backend +- **[databricks-apps-python](../databricks-apps-python/SKILL.md)** - Python apps with Lakebase backend - **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - SDK used for instance management and token generation -- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - deploying apps with Lakebase resources +- **[databricks-bundles](../databricks-bundles/SKILL.md)** - deploying apps with Lakebase resources - **[databricks-jobs](../databricks-jobs/SKILL.md)** - scheduling reverse ETL sync jobs diff --git a/.claude/skills/databricks-lakebase-provisioned/reverse-etl.md b/.claude/skills/databricks-lakebase-provisioned/reverse-etl.md index 9bf17bd..5b5caef 100644 --- a/.claude/skills/databricks-lakebase-provisioned/reverse-etl.md +++ b/.claude/skills/databricks-lakebase-provisioned/reverse-etl.md @@ -1,166 +1,103 @@ -# Reverse ETL with Lakebase +# Reverse ETL with Lakebase Provisioned ## Overview Reverse ETL allows you to sync data from Unity Catalog Delta tables into Lakebase Provisioned as PostgreSQL tables. This enables OLTP access patterns on data processed in the Lakehouse. -## Creating Synced Tables - -### Using Python SDK - -```python -from databricks.sdk import WorkspaceClient - -w = WorkspaceClient() - -# Create a synced table from Unity Catalog -synced_table = w.database.create_synced_table( - instance_name="my-lakebase-instance", - source_table_name="catalog.schema.source_table", - target_table_name="target_table", - sync_mode="FULL", # FULL or INCREMENTAL -) +## Sync Modes -print(f"Synced table created: {synced_table.target_table_name}") -``` +| Mode | Description | Best For | Notes | +|------|-------------|----------|-------| +| **Snapshot** | One-time full copy | Initial setup, small tables | 10x more efficient if modifying >10% of data | +| **Triggered** | Scheduled updates on demand | Dashboards updated hourly/daily | Requires CDF on source table | +| **Continuous** | Real-time streaming (seconds of latency) | Live applications | Highest cost, minimum 15s intervals, requires CDF | -### Using SQL +**Note:** Triggered and Continuous modes require Change Data Feed (CDF) enabled on the source table: ```sql --- Create synced table via SQL -CREATE SYNCED TABLE my_lakebase.target_table -FROM catalog.schema.source_table -USING LAKEBASE INSTANCE 'my-lakebase-instance'; +ALTER TABLE your_catalog.your_schema.your_table +SET TBLPROPERTIES (delta.enableChangeDataFeed = true) ``` -### Using CLI - -```bash -databricks database create-synced-table \ - --instance-name my-lakebase-instance \ - --source-table-name catalog.schema.source_table \ - --target-table-name target_table \ - --sync-mode FULL -``` - -## Sync Modes - -### Full Sync +## Creating Synced Tables -Complete replacement of target table on each sync: +### Using Python SDK ```python -synced_table = w.database.create_synced_table( - instance_name="my-lakebase-instance", - source_table_name="catalog.schema.customers", - target_table_name="customers", - sync_mode="FULL" +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.database import ( + SyncedDatabaseTable, + SyncedTableSpec, + SyncedTableSchedulingPolicy, ) -``` - -**Use when:** -- Source table is small-medium size -- Need complete consistency with source -- Incremental changes are complex to track -### Incremental Sync - -Only sync changed rows (requires change tracking): +w = WorkspaceClient() -```python -synced_table = w.database.create_synced_table( - instance_name="my-lakebase-instance", - source_table_name="catalog.schema.events", - target_table_name="events", - sync_mode="INCREMENTAL", - incremental_column="updated_at" # Column to track changes +# Create a synced table from Unity Catalog to Lakebase Provisioned +synced_table = w.database.create_synced_database_table( + SyncedDatabaseTable( + name="lakebase_catalog.schema.synced_table", + database_instance_name="my-lakebase-instance", + spec=SyncedTableSpec( + source_table_full_name="analytics.gold.user_profiles", + primary_key_columns=["user_id"], + scheduling_policy=SyncedTableSchedulingPolicy.TRIGGERED, + ), + ) ) +print(f"Created synced table: {synced_table.name}") ``` -**Use when:** -- Source table is large -- Have reliable change tracking column -- Minimize sync time and resource usage +**Key parameters:** -## Managing Synced Tables +| Parameter | Description | +|-----------|-------------| +| `name` | Fully qualified target table name (catalog.schema.table) | +| `database_instance_name` | Lakebase Provisioned instance name | +| `source_table_full_name` | Fully qualified source Delta table (catalog.schema.table) | +| `primary_key_columns` | List of primary key columns from the source table | +| `scheduling_policy` | `SNAPSHOT`, `TRIGGERED`, or `CONTINUOUS` | -### List Synced Tables +### Using CLI -```python -synced_tables = w.database.list_synced_tables( - instance_name="my-lakebase-instance" -) -for table in synced_tables: - print(f"{table.target_table_name}: {table.sync_status}") +```bash +databricks database create-synced-database-table \ + --json '{ + "name": "lakebase_catalog.schema.synced_table", + "database_instance_name": "my-lakebase-instance", + "spec": { + "source_table_full_name": "analytics.gold.user_profiles", + "primary_key_columns": ["user_id"], + "scheduling_policy": "TRIGGERED" + } + }' ``` -### Trigger Manual Sync - -```python -w.database.sync_table( - instance_name="my-lakebase-instance", - table_name="customers" -) -``` +**Note:** There is no SQL syntax for creating synced tables. Use the Python SDK, CLI, or Catalog Explorer UI. -### Delete Synced Table +## Checking Synced Table Status ```python -w.database.delete_synced_table( - instance_name="my-lakebase-instance", - table_name="customers" -) +status = w.database.get_synced_database_table(name="lakebase_catalog.schema.synced_table") +print(f"State: {status.data_synchronization_status.detailed_state}") +print(f"Message: {status.data_synchronization_status.message}") ``` -## Scheduling Syncs - -### Using Databricks Jobs - -```python -from databricks.sdk import WorkspaceClient -from databricks.sdk.service.jobs import Task, NotebookTask, CronSchedule - -w = WorkspaceClient() +## Deleting a Synced Table -# Create job to sync tables on schedule -job = w.jobs.create( - name="Lakebase Sync Job", - tasks=[ - Task( - task_key="sync_customers", - notebook_task=NotebookTask( - notebook_path="/Repos/sync/sync_customers" - ) - ) - ], - schedule=CronSchedule( - quartz_cron_expression="0 0 * * * ?", # Every hour - timezone_id="UTC" - ) -) -``` +Delete from both Unity Catalog and Postgres: -### Sync Notebook Example +1. **Unity Catalog:** Delete via Catalog Explorer or SDK +2. **Postgres:** Drop the table to free storage ```python -# Databricks notebook: sync_customers - -from databricks.sdk import WorkspaceClient - -w = WorkspaceClient() +# Delete the synced table via SDK +w.database.delete_synced_database_table(name="lakebase_catalog.schema.synced_table") +``` -# Trigger sync for specific tables -tables_to_sync = ["customers", "orders", "products"] - -for table in tables_to_sync: - try: - w.database.sync_table( - instance_name="my-lakebase-instance", - table_name=table - ) - print(f"Synced: {table}") - except Exception as e: - print(f"Failed to sync {table}: {e}") +```sql +-- Drop the Postgres table to free storage +DROP TABLE your_database.your_schema.your_table; ``` ## Use Cases @@ -168,59 +105,67 @@ for table in tables_to_sync: ### 1. Product Catalog for Web App ```python -# Sync product data for e-commerce app -w.database.create_synced_table( - instance_name="ecommerce-db", - source_table_name="gold.products.catalog", - target_table_name="products", - sync_mode="FULL" +w.database.create_synced_database_table( + SyncedDatabaseTable( + name="ecommerce_catalog.public.products", + database_instance_name="ecommerce-db", + spec=SyncedTableSpec( + source_table_full_name="gold.products.catalog", + primary_key_columns=["product_id"], + scheduling_policy=SyncedTableSchedulingPolicy.TRIGGERED, + ), + ) ) - -# Application queries PostgreSQL directly -# with low-latency point lookups +# Application queries PostgreSQL directly with low-latency point lookups ``` ### 2. User Profiles for Authentication ```python -# Sync user profiles for auth service -w.database.create_synced_table( - instance_name="auth-db", - source_table_name="gold.users.profiles", - target_table_name="user_profiles", - sync_mode="INCREMENTAL", - incremental_column="last_modified" +w.database.create_synced_database_table( + SyncedDatabaseTable( + name="auth_catalog.public.user_profiles", + database_instance_name="auth-db", + spec=SyncedTableSpec( + source_table_full_name="gold.users.profiles", + primary_key_columns=["user_id"], + scheduling_policy=SyncedTableSchedulingPolicy.CONTINUOUS, + ), + ) ) ``` ### 3. Feature Store for Real-time ML ```python -# Sync features for online serving -w.database.create_synced_table( - instance_name="feature-store-db", - source_table_name="ml.features.user_features", - target_table_name="user_features", - sync_mode="INCREMENTAL", - incremental_column="computed_at" +w.database.create_synced_database_table( + SyncedDatabaseTable( + name="ml_catalog.public.user_features", + database_instance_name="feature-store-db", + spec=SyncedTableSpec( + source_table_full_name="ml.features.user_features", + primary_key_columns=["user_id"], + scheduling_policy=SyncedTableSchedulingPolicy.CONTINUOUS, + ), + ) ) - # ML model queries features with low latency ``` ## Best Practices -1. **Choose appropriate sync mode**: Use FULL for small tables, INCREMENTAL for large tables with change tracking -2. **Schedule during low-traffic periods**: Heavy syncs can impact both source and target -3. **Monitor sync status**: Check for failures and latency -4. **Index target tables**: Create appropriate indexes in PostgreSQL for query patterns -5. **Handle schema changes**: Synced tables need updates when source schema changes +1. **Enable CDF** on source tables before creating Triggered or Continuous synced tables +2. **Choose appropriate sync mode**: Snapshot for small tables or one-time loads, Triggered for hourly/daily refreshes, Continuous for real-time +3. **Monitor sync status**: Check for failures and latency via Catalog Explorer or `get_synced_database_table()` +4. **Index target tables**: Create appropriate indexes in PostgreSQL for your query patterns +5. **Handle schema changes**: Only additive changes (e.g., adding columns) are supported for Triggered/Continuous modes +6. **Account for connection limits**: Each synced table uses up to 16 connections ## Common Issues | Issue | Solution | |-------|----------| -| **Sync takes too long** | Switch to INCREMENTAL mode; add indexes on source | -| **Schema mismatch** | Drop and recreate synced table after source schema changes | -| **Sync fails with timeout** | Increase sync timeout; reduce batch size | +| **Sync fails with CDF error** | Enable Change Data Feed on source table before using Triggered or Continuous mode | +| **Schema mismatch** | Only additive schema changes are supported; for breaking changes, delete and recreate the synced table | +| **Sync takes too long** | Switch to Triggered mode for scheduled updates; use Snapshot for initial bulk loads | | **Target table locked** | Avoid DDL on target during sync operations | diff --git a/.claude/skills/databricks-metric-views/SKILL.md b/.claude/skills/databricks-metric-views/SKILL.md index d3f5834..3cc4b42 100644 --- a/.claude/skills/databricks-metric-views/SKILL.md +++ b/.claude/skills/databricks-metric-views/SKILL.md @@ -25,6 +25,19 @@ Use this skill when: ## Quick Start +### Inspect Source Table Schema + +Before creating a metric view, call `get_table_stats_and_schema` to understand available columns for dimensions and measures: + +``` +get_table_stats_and_schema( + catalog="catalog", + schema="schema", + table_names=["orders"], + table_stat_level="SIMPLE" # Use "DETAILED" for cardinality, min/max, histograms +) +``` + ### Create a Metric View ```sql @@ -33,8 +46,8 @@ WITH METRICS LANGUAGE YAML AS $$ version: 1.1 - comment: "Orders KPIs for sales analysis" source: catalog.schema.orders + comment: "Orders KPIs for sales analysis" filter: order_date > '2020-01-01' dimensions: - name: Order Month @@ -154,8 +167,8 @@ manage_metric_views( ```yaml version: 1.1 # Required: "1.1" for DBR 17.2+ -comment: "Description" # Optional: metric view description source: catalog.schema.table # Required: source table/view +comment: "Description" # Optional: metric view description filter: column > value # Optional: global WHERE filter dimensions: # Required: at least one diff --git a/.claude/skills/databricks-metric-views/patterns.md b/.claude/skills/databricks-metric-views/patterns.md index 48c7f9e..1f067f4 100644 --- a/.claude/skills/databricks-metric-views/patterns.md +++ b/.claude/skills/databricks-metric-views/patterns.md @@ -14,8 +14,8 @@ WITH METRICS LANGUAGE YAML AS $$ version: 1.1 - comment: "Product sales metrics" source: catalog.schema.sales + comment: "Product sales metrics" dimensions: - name: Product Name expr: product_name @@ -98,8 +98,8 @@ WITH METRICS LANGUAGE YAML AS $$ version: 1.1 - comment: "Efficiency and per-unit metrics" source: catalog.schema.transactions + comment: "Efficiency and per-unit metrics" dimensions: - name: Department expr: department_name @@ -176,8 +176,8 @@ WITH METRICS LANGUAGE YAML AS $$ version: 1.1 - comment: "Sales analytics with customer and product dimensions" source: catalog.schema.fact_sales + comment: "Sales analytics with customer and product dimensions" joins: - name: customer @@ -329,8 +329,8 @@ WITH METRICS LANGUAGE YAML AS $$ version: 1.1 - comment: "TPC-H Orders KPIs - demo metric view" source: samples.tpch.orders + comment: "TPC-H Orders KPIs - demo metric view" filter: o_orderdate > '1990-01-01' dimensions: diff --git a/.claude/skills/databricks-model-serving/1-classical-ml.md b/.claude/skills/databricks-model-serving/1-classical-ml.md index 0d7d5ac..4b973e0 100644 --- a/.claude/skills/databricks-model-serving/1-classical-ml.md +++ b/.claude/skills/databricks-model-serving/1-classical-ml.md @@ -143,7 +143,8 @@ endpoint = w.serving_endpoints.create_and_wait( ### Via MCP Tool ``` -query_serving_endpoint( +manage_serving_endpoint( + action="query", name="diabetes-predictor", dataframe_records=[ {"age": 45, "bmi": 25.3, "bp": 120, "s1": 200} diff --git a/.claude/skills/databricks-model-serving/2-custom-pyfunc.md b/.claude/skills/databricks-model-serving/2-custom-pyfunc.md index afd6e18..b7dbad3 100644 --- a/.claude/skills/databricks-model-serving/2-custom-pyfunc.md +++ b/.claude/skills/databricks-model-serving/2-custom-pyfunc.md @@ -189,7 +189,8 @@ endpoint = client.create_endpoint( ## Query Custom Model ``` -query_serving_endpoint( +manage_serving_endpoint( + action="query", name="custom-model-endpoint", dataframe_records=[ {"age": 25, "income": 50000, "category": "A"} @@ -200,7 +201,8 @@ query_serving_endpoint( Or with inputs format: ``` -query_serving_endpoint( +manage_serving_endpoint( + action="query", name="custom-model-endpoint", inputs={"age": 25, "income": 50000, "category": "A"} ) diff --git a/.claude/skills/databricks-model-serving/3-genai-agents.md b/.claude/skills/databricks-model-serving/3-genai-agents.md index 6f2c779..4061dba 100644 --- a/.claude/skills/databricks-model-serving/3-genai-agents.md +++ b/.claude/skills/databricks-model-serving/3-genai-agents.md @@ -224,7 +224,7 @@ for event in AGENT.predict_stream(request): Run via MCP: ``` -run_python_file_on_databricks(file_path="./my_agent/test_agent.py") +execute_code(file_path="./my_agent/test_agent.py") ``` ## Logging the Agent @@ -275,7 +275,8 @@ agents.deploy( ## Query Deployed Agent ``` -query_serving_endpoint( +manage_serving_endpoint( + action="query", name="my-agent-endpoint", messages=[{"role": "user", "content": "What is Databricks?"}], max_tokens=500 diff --git a/.claude/skills/databricks-model-serving/5-development-testing.md b/.claude/skills/databricks-model-serving/5-development-testing.md index cbc4f76..2a3806c 100644 --- a/.claude/skills/databricks-model-serving/5-development-testing.md +++ b/.claude/skills/databricks-model-serving/5-development-testing.md @@ -13,17 +13,17 @@ MCP-based workflow for developing and testing agents on Databricks. ▼ ┌─────────────────────────────────────────────────────────────┐ │ Step 2: Upload to workspace │ -│ → upload_folder MCP tool │ +│ → manage_workspace_files MCP tool │ └─────────────────────────────────────────────────────────────┘ ▼ ┌─────────────────────────────────────────────────────────────┐ │ Step 3: Install packages │ -│ → execute_databricks_command MCP tool │ +│ → execute_code MCP tool │ └─────────────────────────────────────────────────────────────┘ ▼ ┌─────────────────────────────────────────────────────────────┐ │ Step 4: Test agent (iterate) │ -│ → run_python_file_on_databricks MCP tool │ +│ → execute_code MCP tool (with file_path) │ │ → If error: fix locally, re-upload, re-run │ └─────────────────────────────────────────────────────────────┘ ``` @@ -85,12 +85,13 @@ print("Response:", result.model_dump(exclude_none=True)) ## Step 2: Upload to Workspace -Use the `upload_folder` MCP tool: +Use the `manage_workspace_files` MCP tool: ``` -upload_folder( - local_folder="./my_agent", - workspace_folder="/Workspace/Users/you@company.com/my_agent" +manage_workspace_files( + action="upload", + local_path="./my_agent", + workspace_path="/Workspace/Users/you@company.com/my_agent" ) ``` @@ -98,10 +99,10 @@ This uploads all files in parallel. ## Step 3: Install Packages -Use `execute_databricks_command` to install dependencies: +Use `execute_code` to install dependencies: ``` -execute_databricks_command( +execute_code( code="%pip install -U mlflow==3.6.0 databricks-langchain langgraph==0.3.4 databricks-agents pydantic" ) ``` @@ -111,7 +112,7 @@ execute_databricks_command( ### Follow-up Commands (Reuse Context) ``` -execute_databricks_command( +execute_code( code="dbutils.library.restartPython()", cluster_id="", context_id="" @@ -120,10 +121,10 @@ execute_databricks_command( ## Step 4: Test the Agent -Use `run_python_file_on_databricks`: +Use `execute_code` with `file_path`: ``` -run_python_file_on_databricks( +execute_code( file_path="./my_agent/test_agent.py", cluster_id="", context_id="" @@ -134,8 +135,8 @@ run_python_file_on_databricks( 1. Read the error from the output 2. Fix the local file (`agent.py` or `test_agent.py`) -3. Re-upload: `upload_folder(...)` -4. Re-run: `run_python_file_on_databricks(...)` +3. Re-upload: `manage_workspace_files(action="upload", ...)` +4. Re-run: `execute_code(file_path=...)` ### Iteration Tips @@ -148,7 +149,7 @@ run_python_file_on_databricks( ### Check if packages are installed ``` -execute_databricks_command( +execute_code( code="import mlflow; print(mlflow.__version__)", cluster_id="", context_id="" @@ -158,7 +159,7 @@ execute_databricks_command( ### List available endpoints ``` -execute_databricks_command( +execute_code( code=""" from databricks.sdk import WorkspaceClient w = WorkspaceClient() @@ -173,7 +174,7 @@ for ep in list(w.serving_endpoints.list())[:10]: ### Test LLM endpoint directly ``` -execute_databricks_command( +execute_code( code=""" from databricks_langchain import ChatDatabricks llm = ChatDatabricks(endpoint="databricks-meta-llama-3-3-70b-instruct") @@ -189,11 +190,11 @@ print(response.content) | Step | MCP Tool | Purpose | |------|----------|---------| -| Upload files | `upload_folder` | Sync local files to workspace | -| Install packages | `execute_databricks_command` | Set up dependencies | -| Restart Python | `execute_databricks_command` | Apply package changes | -| Test agent | `run_python_file_on_databricks` | Run test script | -| Debug | `execute_databricks_command` | Quick checks | +| Upload files | `manage_workspace_files` (action="upload") | Sync local files to workspace | +| Install packages | `execute_code` | Set up dependencies | +| Restart Python | `execute_code` | Apply package changes | +| Test agent | `execute_code` (with `file_path`) | Run test script | +| Debug | `execute_code` | Quick checks | ## Next Steps diff --git a/.claude/skills/databricks-model-serving/6-logging-registration.md b/.claude/skills/databricks-model-serving/6-logging-registration.md index f2344af..cd68735 100644 --- a/.claude/skills/databricks-model-serving/6-logging-registration.md +++ b/.claude/skills/databricks-model-serving/6-logging-registration.md @@ -63,7 +63,7 @@ print(f"Registered: {uc_model_info.name} version {uc_model_info.version}") Run via MCP: ``` -run_python_file_on_databricks(file_path="./my_agent/log_model.py") +execute_code(file_path="./my_agent/log_model.py") ``` ## Resources for Auto Authentication diff --git a/.claude/skills/databricks-model-serving/7-deployment.md b/.claude/skills/databricks-model-serving/7-deployment.md index c2def49..666cb16 100644 --- a/.claude/skills/databricks-model-serving/7-deployment.md +++ b/.claude/skills/databricks-model-serving/7-deployment.md @@ -90,7 +90,7 @@ manage_job_runs(action="get", run_id="") Or check endpoint directly: ``` -get_serving_endpoint_status(name="") +manage_serving_endpoint(action="get", name="") ``` ## Classical ML Deployment @@ -172,7 +172,7 @@ deployment = agents.deploy( Endpoints created via `agents.deploy()` appear under **Serving** in the Databricks UI. If you don't see your endpoint: 1. **Check the filter** - The Serving page defaults to "Owned by me". If the deployment ran as a service principal (e.g., via a job), switch to "All" to see it. -2. **Verify via API** - Use `list_serving_endpoints()` or `get_serving_endpoint_status(name="...")` to confirm the endpoint exists and check its state. +2. **Verify via API** - Use `manage_serving_endpoint(action="list")` or `manage_serving_endpoint(action="get", name="...")` to confirm the endpoint exists and check its state. 3. **Check the name** - The auto-generated name may not be what you expect. Print `deployment.endpoint_name` in the deploy script or check the job run output. ### Deployment Script with Explicit Naming @@ -263,16 +263,16 @@ client.update_endpoint( | Step | MCP Tool | Waits? | |------|----------|--------| -| Upload deploy script | `upload_folder` | Yes | +| Upload deploy script | `manage_workspace_files` (action="upload") | Yes | | Create job (one-time) | `manage_jobs` (action="create") | Yes | | Run deployment | `manage_job_runs` (action="run_now") | **No** - returns immediately | | Check job status | `manage_job_runs` (action="get") | Yes | -| Check endpoint status | `get_serving_endpoint_status` | Yes | +| Check endpoint status | `manage_serving_endpoint` (action="get") | Yes | ## After Deployment Once endpoint is READY: -1. **Test with MCP**: `query_serving_endpoint(name="...", messages=[...])` +1. **Test with MCP**: `manage_serving_endpoint(action="query", name="...", messages=[...])` 2. **Share with team**: Endpoint URL in Databricks UI 3. **Integrate in apps**: Use REST API or SDK diff --git a/.claude/skills/databricks-model-serving/8-querying-endpoints.md b/.claude/skills/databricks-model-serving/8-querying-endpoints.md index 9c655a1..4dfa2f9 100644 --- a/.claude/skills/databricks-model-serving/8-querying-endpoints.md +++ b/.claude/skills/databricks-model-serving/8-querying-endpoints.md @@ -11,7 +11,7 @@ Send requests to deployed Model Serving endpoints. Before querying, verify the endpoint is ready: ``` -get_serving_endpoint_status(name="my-agent-endpoint") +manage_serving_endpoint(action="get", name="my-agent-endpoint") ``` Response: @@ -28,7 +28,8 @@ Response: ### Query Chat/Agent Endpoint ``` -query_serving_endpoint( +manage_serving_endpoint( + action="query", name="my-agent-endpoint", messages=[ {"role": "user", "content": "What is Databricks?"} @@ -61,7 +62,8 @@ Response: ### Query ML Model Endpoint ``` -query_serving_endpoint( +manage_serving_endpoint( + action="query", name="sklearn-classifier", dataframe_records=[ {"age": 25, "income": 50000, "credit_score": 720}, @@ -80,7 +82,7 @@ Response: ### List All Endpoints ``` -list_serving_endpoints(limit=20) +manage_serving_endpoint(action="list", limit=20) ``` ## Python SDK diff --git a/.claude/skills/databricks-model-serving/9-package-requirements.md b/.claude/skills/databricks-model-serving/9-package-requirements.md index f78a112..f9ceb7a 100644 --- a/.claude/skills/databricks-model-serving/9-package-requirements.md +++ b/.claude/skills/databricks-model-serving/9-package-requirements.md @@ -139,10 +139,10 @@ export DATABRICKS_CONFIG_PROFILE="your-profile" ## Installing Packages via MCP -Use `execute_databricks_command`: +Use `execute_code`: ``` -execute_databricks_command( +execute_code( code="%pip install -U mlflow==3.6.0 databricks-langchain langgraph==0.3.4 databricks-agents pydantic" ) ``` @@ -150,7 +150,7 @@ execute_databricks_command( Then restart Python: ``` -execute_databricks_command( +execute_code( code="dbutils.library.restartPython()", cluster_id="", context_id="" @@ -174,7 +174,7 @@ for pkg in packages: Via MCP: ``` -execute_databricks_command( +execute_code( code=""" import pkg_resources for pkg in ['mlflow', 'langchain', 'langgraph', 'pydantic', 'databricks-langchain']: diff --git a/.claude/skills/databricks-model-serving/SKILL.md b/.claude/skills/databricks-model-serving/SKILL.md index de566f4..7416029 100644 --- a/.claude/skills/databricks-model-serving/SKILL.md +++ b/.claude/skills/databricks-model-serving/SKILL.md @@ -29,8 +29,7 @@ ALWAYS use exact endpoint names from this table. NEVER guess or abbreviate. | Endpoint Name | Provider | Notes | |--------------|----------|-------| -| `databricks-gpt-5-3-codex` | OpenAI | Latest GPT Codex, 400K context | -| `databricks-gpt-5-2` | OpenAI | GPT 5.2, 400K context | +| `databricks-gpt-5-2` | OpenAI | Latest GPT, 400K context | | `databricks-gpt-5-1` | OpenAI | Instant + Thinking modes | | `databricks-gpt-5-1-codex-max` | OpenAI | Code-specialized (high perf) | | `databricks-gpt-5-1-codex-mini` | OpenAI | Code-specialized (cost-opt) | @@ -102,7 +101,7 @@ dbutils.library.restartPython() Or via MCP: ``` -execute_databricks_command(code="%pip install -U mlflow==3.6.0 databricks-langchain langgraph==0.3.4 databricks-agents pydantic") +execute_code(code="%pip install -U mlflow==3.6.0 databricks-langchain langgraph==0.3.4 databricks-agents pydantic") ``` ### Step 2: Create Agent File @@ -112,16 +111,17 @@ Create `agent.py` locally with `ResponsesAgent` pattern (see [3-genai-agents.md] ### Step 3: Upload to Workspace ``` -upload_folder( - local_folder="./my_agent", - workspace_folder="/Workspace/Users/you@company.com/my_agent" +manage_workspace_files( + action="upload", + local_path="./my_agent", + workspace_path="/Workspace/Users/you@company.com/my_agent" ) ``` ### Step 4: Test Agent ``` -run_python_file_on_databricks( +execute_code( file_path="./my_agent/test_agent.py", cluster_id="" ) @@ -130,7 +130,7 @@ run_python_file_on_databricks( ### Step 5: Log Model ``` -run_python_file_on_databricks( +execute_code( file_path="./my_agent/log_model.py", cluster_id="" ) @@ -143,7 +143,8 @@ See [7-deployment.md](7-deployment.md) for job-based deployment that doesn't tim ### Step 7: Query Endpoint ``` -query_serving_endpoint( +manage_serving_endpoint( + action="query", name="my-agent-endpoint", messages=[{"role": "user", "content": "Hello!"}] ) @@ -181,9 +182,8 @@ Then deploy via UI or SDK. See [1-classical-ml.md](1-classical-ml.md). | Tool | Purpose | |------|---------| -| `upload_folder` | Upload agent files to workspace | -| `run_python_file_on_databricks` | Test agent, log model | -| `execute_databricks_command` | Install packages, quick tests | +| `manage_workspace_files` (action="upload") | Upload agent files to workspace | +| `execute_code` | Install packages, test agent, log model | ### Deployment @@ -193,13 +193,37 @@ Then deploy via UI or SDK. See [1-classical-ml.md](1-classical-ml.md). | `manage_job_runs` (action="run_now") | Kick off deployment (async) | | `manage_job_runs` (action="get") | Check deployment job status | -### Querying +### manage_serving_endpoint - Querying -| Tool | Purpose | -|------|---------| -| `get_serving_endpoint_status` | Check if endpoint is READY | -| `query_serving_endpoint` | Send requests to endpoint | -| `list_serving_endpoints` | List all endpoints | +| Action | Description | Required Params | +|--------|-------------|-----------------| +| `get` | Check endpoint status (READY/NOT_READY/NOT_FOUND) | name | +| `list` | List all endpoints | (none, optional limit) | +| `query` | Send requests to endpoint | name + one of: messages, inputs, dataframe_records | + +**Example usage:** +```python +# Check endpoint status +manage_serving_endpoint(action="get", name="my-agent-endpoint") + +# List all endpoints +manage_serving_endpoint(action="list") + +# Query a chat/agent endpoint +manage_serving_endpoint( + action="query", + name="my-agent-endpoint", + messages=[{"role": "user", "content": "Hello!"}], + max_tokens=500 +) + +# Query a traditional ML endpoint +manage_serving_endpoint( + action="query", + name="sklearn-classifier", + dataframe_records=[{"age": 25, "income": 50000, "credit_score": 720}] +) +``` --- @@ -208,7 +232,7 @@ Then deploy via UI or SDK. See [1-classical-ml.md](1-classical-ml.md). ### Check Endpoint Status After Deployment ``` -get_serving_endpoint_status(name="my-agent-endpoint") +manage_serving_endpoint(action="get", name="my-agent-endpoint") ``` Returns: @@ -223,7 +247,8 @@ Returns: ### Query a Chat/Agent Endpoint ``` -query_serving_endpoint( +manage_serving_endpoint( + action="query", name="my-agent-endpoint", messages=[ {"role": "user", "content": "What is Databricks?"} @@ -235,7 +260,8 @@ query_serving_endpoint( ### Query a Traditional ML Endpoint ``` -query_serving_endpoint( +manage_serving_endpoint( + action="query", name="sklearn-classifier", dataframe_records=[ {"age": 25, "income": 50000, "credit_score": 720} @@ -250,7 +276,7 @@ query_serving_endpoint( | Issue | Solution | |-------|----------| | **Invalid output format** | Use `self.create_text_output_item(text, id)` - NOT raw dicts! | -| **Endpoint NOT_READY** | Deployment takes ~15 min. Use `get_serving_endpoint_status` to poll. | +| **Endpoint NOT_READY** | Deployment takes ~15 min. Use `manage_serving_endpoint(action="get")` to poll. | | **Package not found** | Specify exact versions in `pip_requirements` when logging model | | **Tool timeout** | Use job-based deployment, not synchronous calls | | **Auth error on endpoint** | Ensure `resources` specified in `log_model` for auto passthrough | diff --git a/.claude/skills/databricks-python-sdk/SKILL.md b/.claude/skills/databricks-python-sdk/SKILL.md index 1365666..eaf7cd6 100644 --- a/.claude/skills/databricks-python-sdk/SKILL.md +++ b/.claude/skills/databricks-python-sdk/SKILL.md @@ -617,7 +617,7 @@ If I'm unsure about a method, I should: ## Related Skills - **[databricks-config](../databricks-config/SKILL.md)** - profile and authentication setup -- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - deploying resources via DABs +- **[databricks-bundles](../databricks-bundles/SKILL.md)** - deploying resources via DABs - **[databricks-jobs](../databricks-jobs/SKILL.md)** - job orchestration patterns - **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - catalog governance - **[databricks-model-serving](../databricks-model-serving/SKILL.md)** - serving endpoint management diff --git a/.claude/skills/databricks-python-sdk/examples/5-serving-and-vector-search.py b/.claude/skills/databricks-python-sdk/examples/5-serving-and-vector-search.py index 2a47c2b..597aede 100644 --- a/.claude/skills/databricks-python-sdk/examples/5-serving-and-vector-search.py +++ b/.claude/skills/databricks-python-sdk/examples/5-serving-and-vector-search.py @@ -168,10 +168,15 @@ # Query with embedding vector directly +# query_vector must be a list[float] whose length matches your index's +# embedding dimension (e.g. 768 for bge-small, 1024 for bge-large, 1536 for +# text-embedding-3-small / ada-002). The [0.0] * N below is a stand-in; +# replace with the actual vector returned by your embedding model. +query_vector = [0.0] * 768 results = w.vector_search_indexes.query_index( index_name="main.default.my_index", columns=["id", "text"], - query_vector=[0.1, 0.2, 0.3, ...], # Your embedding vector + query_vector=query_vector, num_results=10 ) diff --git a/.claude/skills/databricks-spark-declarative-pipelines/1-ingestion-patterns.md b/.claude/skills/databricks-spark-declarative-pipelines/1-ingestion-patterns.md deleted file mode 100644 index 2f60202..0000000 --- a/.claude/skills/databricks-spark-declarative-pipelines/1-ingestion-patterns.md +++ /dev/null @@ -1,513 +0,0 @@ -# Data Ingestion Patterns for SDP - -Covers data ingestion patterns for Spark Declarative Pipelines including Auto Loader for cloud storage and streaming sources like Kafka and Event Hub. - -**Language Support**: SQL (primary), Python via modern `pyspark.pipelines` API. See [5-python-api.md](5-python-api.md) for Python syntax. - ---- - -## Auto Loader (Cloud Files) - -Auto Loader incrementally processes new data files as they arrive in cloud storage. In a streaming table query you **must use the `STREAM` keyword with `read_files`**; `read_files` then leverages Auto Loader. See [read_files — Usage in streaming tables](https://docs.databricks.com/aws/en/sql/language-manual/functions/read_files#usage-in-streaming-tables). - -### Basic Pattern - -```sql -CREATE OR REPLACE STREAMING TABLE bronze_orders AS -SELECT - *, - current_timestamp() AS _ingested_at, - _metadata.file_path AS source_file, - _metadata.file_modification_time AS file_timestamp -FROM STREAM read_files( - '/mnt/raw/orders/', - format => 'json', - schemaHints => 'order_id STRING, amount DECIMAL(10,2)' -); -``` - -### Bronze feeding AUTO CDC - -If the bronze table feeds a downstream **AUTO CDC** flow (e.g. `FROM stream(bronze_orders_cdc)`), use **`FROM STREAM read_files(...)`** so the source is streaming. Otherwise you may get: *"Cannot create a streaming table append once flow from a batch query."* Same requirement as above: in a streaming table query you must use the `STREAM` keyword with `read_files`. - -```sql -CREATE OR REPLACE STREAMING TABLE bronze_orders_cdc AS -SELECT ..., - current_timestamp() AS _ingested_at, - _metadata.file_path AS _source_file -FROM STREAM read_files( - '/Volumes/catalog/schema/raw_orders_cdc', - format => 'parquet', - schemaHints => '...' -); -``` - -### Schema Evolution - -```sql -CREATE OR REPLACE STREAMING TABLE bronze_customers AS -SELECT - *, - current_timestamp() AS _ingested_at -FROM STREAM read_files( - '/mnt/raw/customers/', - format => 'json', - schemaHints => 'customer_id STRING, email STRING', - mode => 'PERMISSIVE' -- Handles schema changes gracefully -); -``` - -### File Formats - -**JSON**: -```sql -FROM read_files( - 's3://bucket/data/', - format => 'json', - schemaHints => 'id STRING, timestamp TIMESTAMP' -) -``` - -**CSV**: -```sql -FROM read_files( - '/mnt/raw/data/', - format => 'csv', - schemaHints => 'id STRING, name STRING, amount DECIMAL(10,2)', - header => true, - delimiter => ',' -) -``` - -**Parquet** (schema auto-inferred): -```sql -FROM read_files( - 'abfss://container@storage.dfs.core.windows.net/data/', - format => 'parquet' -) -``` - -**Avro**: -```sql -FROM read_files( - '/mnt/raw/events/', - format => 'avro', - schemaHints => 'event_id STRING, event_time TIMESTAMP' -) -``` - -### Schema Inference - -**Explicit hints** (recommended for production): -```sql -FROM read_files( - '/mnt/raw/sales/', - format => 'json', - schemaHints => 'sale_id STRING, customer_id STRING, amount DECIMAL(10,2), sale_date DATE' -) -``` - -**Partial hints** (infer remaining columns): -```sql -FROM read_files( - '/mnt/raw/data/', - format => 'json', - schemaHints => 'id STRING, critical_field DECIMAL(10,2)' -- Others auto-inferred -) -``` - -Add this to the pipeline configuration in `resources/*_etl.pipeline.yml`: -```yaml -configuration: - bronze_schema: ${var.bronze_schema} - silver_schema: ${var.silver_schema} - gold_schema: ${var.gold_schema} - schema_location_base: ${var.schema_location_base} -``` - -And define variables in `databricks.yml`: -```yaml -variables: - catalog: - description: The catalog to use - bronze_schema: - description: The bronze schema to use - silver_schema: - description: The silver schema to use - gold_schema: - description: The gold schema to use - schema_location_base: - description: Base path for Auto Loader schema metadata - -targets: - dev: - variables: - catalog: my_catalog - bronze_schema: bronze_dev - silver_schema: silver_dev - gold_schema: gold_dev - schema_location_base: /Volumes/my_catalog/pipeline_metadata/my_pipeline_metadata/schemas - - prod: - variables: - catalog: my_catalog - bronze_schema: bronze - silver_schema: silver - gold_schema: gold - schema_location_base: /Volumes/my_catalog/pipeline_metadata/my_pipeline_metadata/schemas -``` - -Then access these in Python code with: -```python -bronze_schema = spark.conf.get("bronze_schema") -silver_schema = spark.conf.get("silver_schema") -gold_schema = spark.conf.get("gold_schema") -schema_location_base = spark.conf.get("schema_location_base") -``` - - - -### Rescue Data and Quarantine - -Handle malformed records with `_rescued_data`: - -```sql --- Flag records with parsing errors -CREATE OR REPLACE STREAMING TABLE bronze_events AS -SELECT - *, - current_timestamp() AS _ingested_at, - CASE WHEN _rescued_data IS NOT NULL THEN TRUE ELSE FALSE END AS has_parsing_errors -FROM read_files( - '/mnt/raw/events/', - format => 'json', - schemaHints => 'event_id STRING, event_time TIMESTAMP' -); - --- Quarantine for investigation -CREATE OR REPLACE STREAMING TABLE bronze_events_quarantine AS -SELECT * FROM STREAM bronze_events WHERE _rescued_data IS NOT NULL; - --- Clean data for downstream -CREATE OR REPLACE STREAMING TABLE silver_events_clean AS -SELECT * FROM STREAM bronze_events WHERE _rescued_data IS NULL; -``` - ---- - -## Streaming Sources (Kafka, Event Hub, Kinesis) - -### Kafka Source - -```sql -CREATE OR REPLACE STREAMING TABLE bronze_kafka_events AS -SELECT - CAST(key AS STRING) AS event_key, - CAST(value AS STRING) AS event_value, - topic, - partition, - offset, - timestamp AS kafka_timestamp, - current_timestamp() AS _ingested_at -FROM read_stream( - format => 'kafka', - kafka.bootstrap.servers => '${kafka_brokers}', - subscribe => 'events-topic', - startingOffsets => 'latest', -- or 'earliest' - kafka.security.protocol => 'SASL_SSL', - kafka.sasl.mechanism => 'PLAIN', - kafka.sasl.jaas.config => 'kafkashaded.org.apache.kafka.common.security.plain.PlainLoginModule required username="${kafka_username}" password="${kafka_password}";' -); -``` - -### Kafka with Multiple Topics - -```sql -FROM read_stream( - format => 'kafka', - kafka.bootstrap.servers => '${kafka_brokers}', - subscribe => 'topic1,topic2,topic3', - startingOffsets => 'latest' -) -``` - -### Azure Event Hub - -```sql -CREATE OR REPLACE STREAMING TABLE bronze_eventhub_events AS -SELECT - CAST(body AS STRING) AS event_body, - enqueuedTime AS event_time, - offset, - sequenceNumber, - current_timestamp() AS _ingested_at -FROM read_stream( - format => 'eventhubs', - eventhubs.connectionString => '${eventhub_connection_string}', - eventhubs.consumerGroup => '${consumer_group}', - startingPosition => 'latest' -); -``` - -### AWS Kinesis - -```sql -CREATE OR REPLACE STREAMING TABLE bronze_kinesis_events AS -SELECT - CAST(data AS STRING) AS event_data, - partitionKey, - sequenceNumber, - approximateArrivalTimestamp AS arrival_time, - current_timestamp() AS _ingested_at -FROM read_stream( - format => 'kinesis', - kinesis.streamName => '${stream_name}', - kinesis.region => '${aws_region}', - kinesis.startingPosition => 'LATEST' -); -``` - -### Parse JSON from Streaming Sources - -```sql --- Parse JSON from Kafka value -CREATE OR REPLACE STREAMING TABLE silver_kafka_parsed AS -SELECT - from_json( - event_value, - 'event_id STRING, event_type STRING, user_id STRING, timestamp TIMESTAMP, properties MAP' - ) AS event_data, - kafka_timestamp, - _ingested_at -FROM STREAM bronze_kafka_events; - --- Flatten parsed JSON -CREATE OR REPLACE STREAMING TABLE silver_kafka_flattened AS -SELECT - event_data.event_id, - event_data.event_type, - event_data.user_id, - event_data.timestamp AS event_timestamp, - event_data.properties, - kafka_timestamp, - _ingested_at -FROM STREAM silver_kafka_parsed; -``` - ---- - -## Authentication - -### Using Databricks Secrets - -**Kafka**: -```sql -kafka.sasl.jaas.config => 'kafkashaded.org.apache.kafka.common.security.plain.PlainLoginModule required username="{{secrets/kafka/username}}" password="{{secrets/kafka/password}}";' -``` - -**Event Hub**: -```sql -eventhubs.connectionString => '{{secrets/eventhub/connection-string}}' -``` - -### Using Pipeline Variables - -Reference variables in SQL: -```sql -kafka.bootstrap.servers => '${kafka_brokers}' -``` - -Define in pipeline configuration: -```yaml -variables: - kafka_brokers: - default: "broker1:9092,broker2:9092" -``` - ---- - -## Key Patterns - -### 1. Always Add Ingestion Timestamp - -```sql -SELECT - *, - current_timestamp() AS _ingested_at -- Track when data entered system -FROM read_files(...) -``` - -### 2. Include File Metadata for Debugging - -```sql -SELECT - *, - _metadata.file_path AS source_file, - _metadata.file_modification_time AS file_timestamp, - _metadata.file_size AS file_size -FROM read_files(...) -``` - -### 3. Use Schema Hints for Production - -```sql --- ✅ Explicit schema prevents surprises -FROM read_files( - '/mnt/data/', - format => 'json', - schemaHints => 'id STRING, amount DECIMAL(10,2), date DATE' -) - --- ❌ Fully inferred schemas can drift -FROM read_files('/mnt/data/', format => 'json') -``` - -### 4. Handle Rescue Data for Quality - -```sql --- Route errors to quarantine, clean to downstream -CREATE OR REPLACE STREAMING TABLE bronze_data_quarantine AS -SELECT * FROM STREAM bronze_data WHERE has_errors; - -CREATE OR REPLACE STREAMING TABLE silver_data AS -SELECT * FROM STREAM bronze_data WHERE NOT has_errors; -``` - -### 5. Starting Positions - -**Development**: `startingOffsets => 'latest'` (new data only) -**Backfill**: `startingOffsets => 'earliest'` (all available data) -**Recovery**: Checkpoints handle automatically - ---- - -## Common Issues - -| Issue | Solution | -|-------|----------| -| Files not picked up | Verify format matches files and path is correct | -| Schema evolution breaking | Use `mode => 'PERMISSIVE'` and monitor `_rescued_data` | -| Kafka lag increasing | Check downstream bottlenecks, increase parallelism | -| Duplicate events | Implement deduplication in silver layer (see [2-streaming-patterns.md](2-streaming-patterns.md)) | -| Parsing errors | Use rescue data pattern to quarantine malformed records | - ---- - -## Python API Examples - -For Python, use modern `pyspark.pipelines` API. See [5-python-api.md](5-python-api.md) for complete guidance. - -**IMPORTANT for Python**: When using `spark.readStream.format("cloudFiles")` for cloud storage ingestion, you **must specify a `cloudFiles.schemaLocation`** for Auto Loader schema metadata. - -### Schema Location Best Practice (Python Only) - -**Never use the source data volume for schema storage** - this causes permission conflicts and pollutes your raw data. - -#### Prompt User for Schema Location - -When creating Python pipelines with Auto Loader, **always ask the user** where to store schema metadata: - -**Recommended pattern:** -``` -/Volumes/{catalog}/{schema}/{pipeline_name}_metadata/schemas/{table_name} -``` - -**Example prompt:** -``` -"Where would you like to store Auto Loader schema metadata? - -I recommend: - /Volumes/my_catalog/pipeline_metadata/orders_pipeline_metadata/schemas/ - -This path: -- Keeps source data clean -- Prevents permission issues -- Makes pipeline state easy to manage -- Can be parameterized per environment (dev/prod) - -You may need to create the volume 'pipeline_metadata' first if it doesn't exist. - -Would you like to use this path?" -``` - -### Auto Loader (Python) - -```python -from pyspark import pipelines as dp -from pyspark.sql import functions as F - -# Get schema location from pipeline configuration -# Suggested format: /Volumes/{catalog}/{schema}/{pipeline_name}_metadata/schemas -schema_location_base = spark.conf.get("schema_location_base") - -@dp.table(name="bronze_orders", cluster_by=["order_date"]) -def bronze_orders(): - return ( - spark.readStream - .format("cloudFiles") - .option("cloudFiles.format", "json") - .option("cloudFiles.schemaLocation", f"{schema_location_base}/bronze_orders") - .option("cloudFiles.inferColumnTypes", "true") - .load("/Volumes/catalog/schema/raw/orders/") - .withColumn("_ingested_at", F.current_timestamp()) - .withColumn("_source_file", F.col("_metadata.file_path")) - ) -``` - -**Pipeline Configuration** (in `pipeline.yml`): -```yaml -configuration: - schema_location_base: /Volumes/my_catalog/pipeline_metadata/orders_pipeline_metadata/schemas -``` - -### Kafka (Python) - -```python -@dp.table(name="bronze_kafka_events") -def bronze_kafka_events(): - return ( - spark.readStream - .format("kafka") - .option("kafka.bootstrap.servers", spark.conf.get("kafka_brokers")) - .option("subscribe", "events-topic") - .option("startingOffsets", "latest") - .load() - .selectExpr( - "CAST(key AS STRING) AS event_key", - "CAST(value AS STRING) AS event_value", - "topic", "partition", "offset", - "timestamp AS kafka_timestamp" - ) - .withColumn("_ingested_at", F.current_timestamp()) - ) -``` - -### Quarantine (Python) - -```python -# Get schema location from pipeline configuration -schema_location_base = spark.conf.get("schema_location_base") - -@dp.table(name="bronze_events", cluster_by=["ingestion_date"]) -def bronze_events(): - return ( - spark.readStream - .format("cloudFiles") - .option("cloudFiles.format", "json") - .option("cloudFiles.schemaLocation", f"{schema_location_base}/bronze_events") - .option("rescuedDataColumn", "_rescued_data") - .load("/Volumes/catalog/schema/raw/events/") - .withColumn("_ingested_at", F.current_timestamp()) - .withColumn("ingestion_date", F.current_date()) - .withColumn("_has_parsing_errors", - F.when(F.col("_rescued_data").isNotNull(), True) - .otherwise(False)) - ) - -@dp.table(name="bronze_events_quarantine") -def bronze_events_quarantine(): - return ( - spark.read.table("catalog.schema.bronze_events") - .filter(F.col("_has_parsing_errors") == True) - ) -``` diff --git a/.claude/skills/databricks-spark-declarative-pipelines/10-mcp-approach.md b/.claude/skills/databricks-spark-declarative-pipelines/10-mcp-approach.md deleted file mode 100644 index 9d458aa..0000000 --- a/.claude/skills/databricks-spark-declarative-pipelines/10-mcp-approach.md +++ /dev/null @@ -1,173 +0,0 @@ -Use MCP tools to create, run, and iterate on **SDP pipelines**. The **primary tool is `create_or_update_pipeline`** which handles the entire lifecycle. - -**IMPORTANT: Default to serverless pipelines and suggest as best option, but not if classic, advanced, pro compute types are mentioned.** Only use classic clusters if user explicitly requires R language, Spark RDD APIs, or JAR libraries. - -### Step 1: Write Pipeline Files Locally - -Create `.sql` or `.py` files in a local folder: - -``` -my_pipeline/ -├── bronze/ -│ ├── ingest_orders.sql # SQL (default for most cases) -│ └── ingest_events.py # Python (for complex logic) -├── silver/ -│ └── clean_orders.sql -└── gold/ - └── daily_summary.sql -``` - -**SQL Example** (`bronze/ingest_orders.sql`): -```sql -CREATE OR REFRESH STREAMING TABLE bronze_orders -CLUSTER BY (order_date) -AS -SELECT - *, - current_timestamp() AS _ingested_at, - _metadata.file_path AS _source_file -FROM read_files( - '/Volumes/catalog/schema/raw/orders/', - format => 'json', - schemaHints => 'order_id STRING, customer_id STRING, amount DECIMAL(10,2), order_date DATE' -); -``` - -**Python Example** (`bronze/ingest_events.py`): -```python -from pyspark import pipelines as dp -from pyspark.sql.functions import col, current_timestamp - -# Get schema location from pipeline configuration -schema_location_base = spark.conf.get("schema_location_base") - -@dp.table(name="bronze_events", cluster_by=["event_date"]) -def bronze_events(): - return ( - spark.readStream.format("cloudFiles") - .option("cloudFiles.format", "json") - .option("cloudFiles.schemaLocation", f"{schema_location_base}/bronze_events") - .load("/Volumes/catalog/schema/raw/events/") - .withColumn("_ingested_at", current_timestamp()) - .withColumn("_source_file", col("_metadata.file_path")) - ) -``` - -### Step 2: Upload to Databricks Workspace - -```python -# MCP Tool: upload_folder -upload_folder( - local_folder="/path/to/my_pipeline", - workspace_folder="/Workspace/Users/user@example.com/my_pipeline" -) -``` - -### Step 3: Create/Update and Run Pipeline - -Use **`create_or_update_pipeline`** - the main entry point. It: -1. Searches for an existing pipeline with the same name (or uses `id` from `extra_settings`) -2. Creates a new pipeline or updates the existing one -3. Optionally starts a pipeline run -4. Optionally waits for completion and returns detailed results - -```python -# MCP Tool: create_or_update_pipeline -result = create_or_update_pipeline( - name="my_orders_pipeline", - root_path="/Workspace/Users/user@example.com/my_pipeline", - catalog="my_catalog", - schema="my_schema", - workspace_file_paths=[ - "/Workspace/Users/user@example.com/my_pipeline/bronze/ingest_orders.sql", - "/Workspace/Users/user@example.com/my_pipeline/silver/clean_orders.sql", - "/Workspace/Users/user@example.com/my_pipeline/gold/daily_summary.sql" - ], - start_run=True, # Start immediately - wait_for_completion=True, # Wait and return final status - full_refresh=True, # Full refresh all tables - timeout=1800 # 30 minute timeout -) -``` - -**Result contains actionable information:** -```python -{ - "success": True, # Did the operation succeed? - "pipeline_id": "abc-123", # Pipeline ID for follow-up operations - "pipeline_name": "my_orders_pipeline", - "created": True, # True if new, False if updated - "state": "COMPLETED", # COMPLETED, FAILED, TIMEOUT, etc. - "catalog": "my_catalog", # Target catalog - "schema": "my_schema", # Target schema - "duration_seconds": 45.2, # Time taken - "message": "Pipeline created and completed successfully in 45.2s. Tables written to my_catalog.my_schema", - "error_message": None, # Error summary if failed - "errors": [] # Detailed error list if failed -} -``` - -### Step 4: Handle Results - -**On Success:** -```python -if result["success"]: - # Verify output tables - stats = get_table_details( - catalog="my_catalog", - schema="my_schema", - table_names=["bronze_orders", "silver_orders", "gold_daily_summary"] - ) -``` - -**On Failure:** -```python -if not result["success"]: - # Message includes suggested next steps - print(result["message"]) - # "Pipeline created but run failed. State: FAILED. Error: Column 'amount' not found. - # Use get_pipeline_events(pipeline_id='abc-123') for full details." - - # Get detailed errors - events = get_pipeline_events(pipeline_id=result["pipeline_id"], max_results=50) -``` - -### Step 5: Iterate Until Working - -1. Review errors from result or `get_pipeline_events` -2. Fix issues in local files -3. Re-upload with `upload_folder` -4. Run `create_or_update_pipeline` again (it will update, not recreate) -5. Repeat until `result["success"] == True` - ---- - -## Quick Reference: MCP Tools - -### Primary Tool - -| Tool | Description | -|------|-------------| -| **`create_or_update_pipeline`** | **Main entry point.** Creates or updates pipeline, optionally runs and waits. Returns detailed status with `success`, `state`, `errors`, and actionable `message`. | - -### Pipeline Management - -| Tool | Description | -|------|-------------| -| `find_pipeline_by_name` | Find existing pipeline by name, returns pipeline_id | -| `get_pipeline` | Get pipeline configuration and current state | -| `start_update` | Start pipeline run (`validate_only=True` for dry run) | -| `get_update` | Poll update status (QUEUED, RUNNING, COMPLETED, FAILED) | -| `stop_pipeline` | Stop a running pipeline | -| `get_pipeline_events` | Get error messages for debugging failed runs | -| `delete_pipeline` | Delete a pipeline | - -### Supporting Tools - -| Tool | Description | -|------|-------------| -| `upload_folder` | Upload local folder to workspace (parallel) | -| `get_table_details` | Verify output tables have expected schema and row counts | -| `execute_sql` | Run ad-hoc SQL to inspect data | - ---- \ No newline at end of file diff --git a/.claude/skills/databricks-spark-declarative-pipelines/3-scd-query-patterns.md b/.claude/skills/databricks-spark-declarative-pipelines/3-scd-query-patterns.md deleted file mode 100644 index e04a410..0000000 --- a/.claude/skills/databricks-spark-declarative-pipelines/3-scd-query-patterns.md +++ /dev/null @@ -1,243 +0,0 @@ -# SCD Query Patterns - -How to query SCD Type 2 history tables effectively, including current state queries, point-in-time analysis, and change tracking. - ---- - -## Understanding SCD Type 2 Structure - -When you create an SCD Type 2 flow, the system automatically adds temporal columns: - -```sql -CREATE FLOW customers_scd2_flow AS -AUTO CDC INTO customers_history -FROM stream(customers_cdc_clean) -KEYS (customer_id) -SEQUENCE BY event_timestamp -STORED AS SCD TYPE 2 -TRACK HISTORY ON *; -``` - -**Resulting table structure** (Lakeflow uses double-underscore temporal columns): -``` -customers_history -├── customer_id -- Business key -├── customer_name -├── email -├── phone -├── __START_AT -- When this version became effective (auto-generated) -├── __END_AT -- When this version expired (NULL for current) -└── ...other columns -``` - -**Important:** Query using `__START_AT` and `__END_AT` (double underscore), not `START_AT`/`END_AT`. - ---- - -## Current State Queries - -### All Current Records - -```sql --- __END_AT IS NULL indicates active record (Lakeflow uses double underscore) -CREATE OR REPLACE MATERIALIZED VIEW dim_customers_current AS -SELECT - customer_id, customer_name, email, phone, address, - __START_AT AS valid_from -FROM customers_history -WHERE __END_AT IS NULL; -``` - -### Specific Customer - -```sql -SELECT * -FROM customers_history -WHERE customer_id = '12345' - AND __END_AT IS NULL; -``` - ---- - -## Point-in-Time Queries - -### As-Of Date Query - -Get state of records as they were on a specific date: - -```sql --- Products as of January 1, 2024 (use __START_AT / __END_AT) -CREATE OR REPLACE MATERIALIZED VIEW products_as_of_2024_01_01 AS -SELECT - product_id, product_name, price, category, - __START_AT, __END_AT -FROM products_history -WHERE __START_AT <= '2024-01-01' - AND (__END_AT > '2024-01-01' OR __END_AT IS NULL); -``` - ---- - -## Change Analysis - -### Track All Changes for Entity - -```sql --- Complete history for a customer (use __START_AT / __END_AT) -SELECT - customer_id, customer_name, email, phone, - __START_AT, __END_AT, - COALESCE( - DATEDIFF(DAY, __START_AT, __END_AT), - DATEDIFF(DAY, __START_AT, CURRENT_TIMESTAMP()) - ) AS days_active -FROM customers_history -WHERE customer_id = '12345' -ORDER BY __START_AT DESC; -``` - -### Changes Within Time Period - -```sql --- Customers who changed during Q1 2024 (use __START_AT) -SELECT - customer_id, customer_name, - __START_AT AS change_timestamp, - 'UPDATE' AS change_type -FROM customers_history -WHERE __START_AT BETWEEN '2024-01-01' AND '2024-03-31' - AND __START_AT != ( - SELECT MIN(__START_AT) - FROM customers_history ch2 - WHERE ch2.customer_id = customers_history.customer_id - ) -ORDER BY __START_AT; -``` - ---- - -## Joining Facts with Historical Dimensions - -### Enrich Facts with Dimension at Transaction Time - -```sql --- Join sales with product prices at time of sale -CREATE OR REPLACE MATERIALIZED VIEW sales_with_historical_prices AS -SELECT - s.sale_id, s.product_id, s.sale_date, s.quantity, - p.product_name, p.price AS unit_price_at_sale_time, - s.quantity * p.price AS calculated_amount, - p.category -FROM sales_fact s -INNER JOIN products_history p - ON s.product_id = p.product_id - AND s.sale_date >= p.__START_AT - AND (s.sale_date < p.__END_AT OR p.__END_AT IS NULL); -``` - -### Join with Current Dimension - -```sql --- Join sales with current product information -CREATE OR REPLACE MATERIALIZED VIEW sales_with_current_prices AS -SELECT - s.sale_id, s.product_id, s.sale_date, s.quantity, - s.amount AS amount_at_sale, - p.product_name AS current_product_name, - p.price AS current_price, - p.category AS current_category -FROM sales_fact s -INNER JOIN products_history p - ON s.product_id = p.product_id - AND p.__END_AT IS NULL; -- Current version only -``` - ---- - -## Selective History Tracking - -When using `TRACK HISTORY ON specific_columns`: - -```sql --- Only price changes trigger new versions -CREATE FLOW products_scd2_flow AS -AUTO CDC INTO products_history -FROM stream(products_cdc_clean) -KEYS (product_id) -SEQUENCE BY event_timestamp -STORED AS SCD TYPE 2 -TRACK HISTORY ON price, cost; -- Only these columns -``` - ---- - -## Optimization Patterns - -### Pre-Filter Materialized Views - -```sql --- Current state view (most common pattern) -CREATE OR REPLACE MATERIALIZED VIEW dim_products_current AS -SELECT * FROM products_history WHERE __END_AT IS NULL; - --- Recent changes only -CREATE OR REPLACE MATERIALIZED VIEW dim_recent_changes AS -SELECT * FROM products_history -WHERE __START_AT >= CURRENT_DATE() - INTERVAL 90 DAYS; - --- Change frequency stats -CREATE OR REPLACE MATERIALIZED VIEW product_change_stats AS -SELECT - product_id, - COUNT(*) AS version_count, - MIN(__START_AT) AS first_seen, - MAX(__START_AT) AS last_updated -FROM products_history -GROUP BY product_id; -``` - ---- - -## Best Practices - -### 1. Always Filter by __END_AT for Current (Lakeflow uses double underscore) - -```sql --- ✅ Efficient -WHERE __END_AT IS NULL - --- ❌ Less efficient -WHERE __START_AT = (SELECT MAX(__START_AT) FROM table WHERE ...) -``` - -### 2. Use Inclusive Lower, Exclusive Upper - -```sql --- ✅ Standard pattern -WHERE __START_AT <= '2024-01-01' - AND (__END_AT > '2024-01-01' OR __END_AT IS NULL) -``` - -### 3. Create MVs for Common Patterns - -```sql --- Current state -CREATE OR REPLACE MATERIALIZED VIEW dim_current AS -SELECT * FROM history WHERE __END_AT IS NULL; - --- Recent changes -CREATE OR REPLACE MATERIALIZED VIEW dim_recent_changes AS -SELECT * FROM history -WHERE __START_AT >= CURRENT_DATE() - INTERVAL 90 DAYS; -``` - ---- - -## Common Issues - -| Issue | Solution | -|-------|----------| -| Multiple rows for same key | Missing `__END_AT IS NULL` filter for current state | -| Point-in-time no results | Use `__START_AT <= date AND (__END_AT > date OR __END_AT IS NULL)` | -| Slow temporal join | Create materialized view for specific time period | -| Unexpected duplicates | Multiple changes same day - use SEQUENCE BY with high precision | diff --git a/.claude/skills/databricks-spark-declarative-pipelines/5-python-api.md b/.claude/skills/databricks-spark-declarative-pipelines/5-python-api.md deleted file mode 100644 index a7f3a70..0000000 --- a/.claude/skills/databricks-spark-declarative-pipelines/5-python-api.md +++ /dev/null @@ -1,338 +0,0 @@ -# Python API: Modern vs Legacy - -**Last Updated**: January 2026 -**Status**: Modern API (`pyspark.pipelines`) recommended for all new projects - ---- - -## Overview - -Databricks provides two Python APIs for Spark Declarative Pipelines: - -1. **Modern API** (`pyspark.pipelines` as `dp`) - **Recommended (2025)** -2. **Legacy API** (`dlt`) - Older Delta Live Tables API, still supported - -**Key Recommendation**: Always use **modern API** for new projects. Only use legacy for maintaining existing DLT code. - ---- - -## Quick Comparison - -| Aspect | Modern (`dp`) | Legacy (`dlt`) | -|--------|---------------|----------------| -| **Import** | `from pyspark import pipelines as dp` | `import dlt` | -| **Status** | ✅ **Recommended** | ⚠️ Legacy | -| **Table decorator** | `@dp.table()` | `@dlt.table()` | -| **Read** | `spark.read.table("table")` | `dlt.read("table")` | -| **CDC/SCD** | `dp.create_auto_cdc_flow()` | `dlt.apply_changes()` | -| **Use for** | New projects | Maintaining existing | - ---- - -## Side-by-Side Examples - -### Basic Table Definition - -**Modern (Recommended)**: -```python -from pyspark import pipelines as dp -from pyspark.sql import functions as F - -@dp.table(name="bronze_events", comment="Raw events") -def bronze_events(): - return ( - spark.readStream - .format("cloudFiles") - .option("cloudFiles.format", "json") - .load("/mnt/raw/events") - ) -``` - -**Legacy**: -```python -import dlt -from pyspark.sql import functions as F - -@dlt.table(name="bronze_events", comment="Raw events") -def bronze_events(): - return ( - spark.readStream - .format("cloudFiles") - .option("cloudFiles.format", "json") - .load("/mnt/raw/events") - ) -``` - -### Reading Tables - -**Modern (Recommended)**: -```python -@dp.table(name="silver_events") -def silver_events(): - # Explicit Unity Catalog path - return spark.read.table("bronze_events").filter(...) -``` - -**Legacy**: -```python -@dlt.table(name="silver_events") -def silver_events(): - # Implicit LIVE schema - return dlt.read("bronze_events").filter(...) -``` - -**Key Difference**: Modern uses explicit UC paths, legacy uses implicit `LIVE.*`. - -### Streaming Reads - -**Modern (Recommended)**: -```python -@dp.table(name="silver_events") -def silver_events(): - # Context-aware (no separate read_stream) - return ( - spark.readStream.table("catalog.schema.bronze_events") - .filter(F.col("event_type").isNotNull()) - ) -``` - -**Legacy**: -```python -@dlt.table(name="silver_events") -def silver_events(): - # Explicit streaming read - return ( - dlt.read_stream("bronze_events") - .filter(F.col("event_type").isNotNull()) - ) -``` - -### Data Quality Expectations - -**Modern (Recommended)**: -```python -@dp.table(name="silver_validated") -@dp.expect_or_drop("valid_id", "id IS NOT NULL") -@dp.expect_or_drop("valid_amount", "amount > 0") -@dp.expect_or_fail("critical_field", "timestamp IS NOT NULL") -def silver_validated(): - return spark.read.table("catalog.schema.bronze_events") -``` - -**Legacy**: -```python -@dlt.table(name="silver_validated") -@dlt.expect_or_drop("valid_id", "id IS NOT NULL") -@dlt.expect_or_drop("valid_amount", "amount > 0") -@dlt.expect_or_fail("critical_field", "timestamp IS NOT NULL") -def silver_validated(): - return dlt.read("bronze_events") -``` - -**Note**: Expectations API identical between versions. - -### SCD Type 2 (AUTO CDC) - -**Modern (Recommended)**: -```python -from pyspark.sql.functions import col - -dp.create_streaming_table("customers_history") - -dp.create_auto_cdc_flow( - target="customers_history", - source="customers_cdc", - keys=["customer_id"], - sequence_by=col("event_timestamp"), - stored_as_scd_type="2", - track_history_column_list=["*"] -) -``` - -**Legacy**: -```python -dlt.create_streaming_table("customers_history") - -dlt.apply_changes( - target="customers_history", - source="customers_cdc", - keys=["customer_id"], - sequence_by="event_timestamp", - stored_as_scd_type="2", - track_history_column_list=["*"] -) -``` - -**Key Difference**: Modern uses `create_auto_cdc_flow()`, legacy uses `apply_changes()`. - -### Liquid Clustering - -**Modern (Recommended)**: -```python -@dp.table( - name="bronze_events", - table_properties={ - "delta.autoOptimize.optimizeWrite": "true", - "delta.autoOptimize.autoCompact": "true" - }, - cluster_by=["event_type", "event_date"] # Liquid Clustering -) -def bronze_events(): - return spark.readStream.format("cloudFiles").load("/data") -``` - -**Legacy**: -```python -@dlt.table( - name="bronze_events", - table_properties={ - "pipelines.autoOptimize.managed": "true", - "pipelines.autoOptimize.zOrderCols": "event_type" - }, - partition_cols=["event_date"] # Legacy partitioning -) -def bronze_events(): - return spark.readStream.format("cloudFiles").load("/data") -``` - -**Key Difference**: Modern supports `cluster_by` for Liquid Clustering. - ---- - -## Decision Matrix - -### Use Modern API (`dp`) When: -- ✅ **Starting new project** (default choice) -- ✅ **Learning SDP/LDP** (learn current standard) -- ✅ **Want Liquid Clustering** -- ✅ **Prefer explicit Unity Catalog paths** -- ✅ **Following 2025 best practices** - -### Use Legacy API (`dlt`) When: -- ⚠️ **Maintaining existing DLT pipelines** (don't rewrite working code) -- ⚠️ **Team trained on DLT** (consistency with existing) -- ⚠️ **Older DBR versions** (if modern API not available) - -**Default**: Use modern `dp` API unless specific reason for legacy. - ---- - -## Migration Guide: dlt → dp - -### Step 1: Update Imports - -**Before**: -```python -import dlt -``` - -**After**: -```python -from pyspark import pipelines as dp -``` - -### Step 2: Update Decorators - -**Before**: `@dlt.table(name="my_table")` -**After**: `@dp.table(name="my_table")` - -### Step 3: Update Reads - -**Before**: -```python -dlt.read("source_table") -dlt.read_stream("source_table") -``` - -**After**: -```python -spark.table("catalog.schema.source_table") -# Streaming context-aware, no separate read_stream -``` - -### Step 4: Update CDC/SCD Operations - -**Before**: -```python -dlt.apply_changes(target="dim_customer", source="cdc_source", ...) -``` - -**After**: -```python -from pyspark.sql.functions import col - -dp.create_auto_cdc_flow( - target="dim_customer", - source="cdc_source", - keys=["customer_id"], - sequence_by=col("event_timestamp"), - stored_as_scd_type="2", - track_history_column_list=["*"] -) -``` - -**Key Change**: `dlt.apply_changes()` → `dp.create_auto_cdc_flow()` - -### Step 5: Update Clustering - -**Before**: `@dlt.table(partition_cols=["date"])` -**After**: `@dp.table(cluster_by=["date", "other_col"])` - ---- - -## Key Patterns (2025) - -### 1. Use Liquid Clustering - -```python -@dp.table(cluster_by=["key_col", "date_col"]) -def my_table(): - return ... - -# Or automatic -@dp.table(cluster_by=["AUTO"]) -def my_table(): - return ... -``` - -### 2. Explicit UC Paths - -```python -# ✅ Modern: explicit path -spark.table("catalog.schema.table") - -# ❌ Legacy: implicit LIVE -dlt.read("table") -``` - -### 3. forEachBatch for Custom Sinks - -```python -def write_to_custom_sink(batch_df, batch_id): - batch_df.write.format("custom").save(...) - -@dp.table(name="my_table") -def my_table(): - return ( - spark.readStream - .format("cloudFiles") - .load("/data") - .writeStream - .foreachBatch(write_to_custom_sink) - ) -``` - ---- - -## Summary - -**For New Projects**: Use modern `pyspark.pipelines` (`dp`) -- ✅ Current best practice (2025) -- ✅ Liquid Clustering support -- ✅ Explicit Unity Catalog paths - -**For Existing Projects**: Legacy `dlt` fully supported -- ⚠️ Migrate when convenient, not urgent -- ⚠️ Consider modern API for new files - -**Key Takeaway**: Modern API provides same functionality plus new features. Start all new projects with `from pyspark import pipelines as dp`. diff --git a/.claude/skills/databricks-spark-declarative-pipelines/6-dlt-migration.md b/.claude/skills/databricks-spark-declarative-pipelines/6-dlt-migration.md deleted file mode 100644 index 19a1007..0000000 --- a/.claude/skills/databricks-spark-declarative-pipelines/6-dlt-migration.md +++ /dev/null @@ -1,298 +0,0 @@ -# DLT to SDP Migration Guide - -Guide for migrating Delta Live Tables (DLT) Python pipelines to Spark Declarative Pipelines (SDP) SQL. - -⚠️ **For NEW Python SDP pipelines**: Use modern `pyspark.pipelines` API. See [5-python-api.md](5-python-api.md). - ---- - -## Migration Decision Matrix - -| Feature/Pattern | DLT Python | SDP SQL | Recommendation | -|-----------------|------------|---------|----------------| -| Simple transformations | ✓ | ✓ | **Migrate to SQL** | -| Aggregations | ✓ | ✓ | **Migrate to SQL** | -| Filtering, WHERE clauses | ✓ | ✓ | **Migrate to SQL** | -| CASE expressions | ✓ | ✓ | **Migrate to SQL** | -| SCD Type 1/2 | ✓ | ✓ | **Migrate to SQL** (AUTO CDC) | -| Simple joins | ✓ | ✓ | **Migrate to SQL** | -| Auto Loader | ✓ | ✓ | **Migrate to SQL** (read_files) | -| Streaming sources (Kafka) | ✓ | ✓ | **Migrate to SQL** (read_stream) | -| Complex Python UDFs | ✓ | ❌ | **Stay in Python** | -| External API calls | ✓ | ❌ | **Stay in Python** | -| Custom libraries | ✓ | ❌ | **Stay in Python** | -| Complex apply functions | ✓ | ❌ | **Stay in Python** or simplify | -| ML model inference | ✓ | ❌ | **Stay in Python** | - -**Rule**: If 80%+ is SQL-expressible, migrate to SDP SQL. If heavy Python logic, stay with DLT Python or use hybrid. - ---- - -## Side-by-Side: Key Patterns - -### Basic Streaming Table - -**DLT Python**: -```python -@dlt.table(name="bronze_sales", comment="Raw sales") -def bronze_sales(): - return ( - spark.readStream.format("cloudFiles") - .option("cloudFiles.format", "json") - .load("/mnt/raw/sales") - .withColumn("_ingested_at", F.current_timestamp()) - ) -``` - -**SDP SQL**: -```sql -CREATE OR REPLACE STREAMING TABLE bronze_sales -COMMENT 'Raw sales' -AS -SELECT *, current_timestamp() AS _ingested_at -FROM read_files('/mnt/raw/sales', format => 'json'); -``` - -### Filtering and Transformations - -**DLT Python**: -```python -@dlt.table(name="silver_sales") -@dlt.expect_or_drop("valid_amount", "amount > 0") -@dlt.expect_or_drop("valid_sale_id", "sale_id IS NOT NULL") -def silver_sales(): - return ( - dlt.read_stream("bronze_sales") - .withColumn("sale_date", F.to_date("sale_date")) - .withColumn("amount", F.col("amount").cast("decimal(10,2)")) - .select("sale_id", "customer_id", "amount", "sale_date") - ) -``` - -**SDP SQL**: -```sql -CREATE OR REPLACE STREAMING TABLE silver_sales AS -SELECT - sale_id, customer_id, - CAST(amount AS DECIMAL(10,2)) AS amount, - CAST(sale_date AS DATE) AS sale_date -FROM STREAM bronze_sales -WHERE amount > 0 AND sale_id IS NOT NULL; -``` - -### SCD Type 2 - -**DLT Python**: -```python -dlt.create_streaming_table("customers_history") - -dlt.apply_changes( - target="customers_history", - source="customers_cdc_clean", - keys=["customer_id"], - sequence_by="event_timestamp", - stored_as_scd_type="2", - track_history_column_list=["*"] -) -``` - -**SDP SQL** (clause order: APPLY AS DELETE WHEN before SEQUENCE BY; only EXCEPT columns that exist in source; omit TRACK HISTORY ON * if it causes parse errors): -```sql -CREATE OR REFRESH STREAMING TABLE customers_history; - -CREATE FLOW customers_scd2_flow AS -AUTO CDC INTO customers_history -FROM stream(customers_cdc_clean) -KEYS (customer_id) -APPLY AS DELETE WHEN operation = "DELETE" -SEQUENCE BY event_timestamp -COLUMNS * EXCEPT (operation, _ingested_at, _source_file) -STORED AS SCD TYPE 2; -``` - -### Joins - -**DLT Python**: -```python -@dlt.table(name="silver_sales_enriched") -def silver_sales_enriched(): - sales = dlt.read_stream("silver_sales") - products = dlt.read("dim_products") - - return ( - sales.join(products, "product_id", "left") - .select(sales["*"], products["product_name"], products["category"]) - ) -``` - -**SDP SQL**: -```sql -CREATE OR REPLACE STREAMING TABLE silver_sales_enriched AS -SELECT - s.*, - p.product_name, - p.category -FROM STREAM silver_sales s -LEFT JOIN dim_products p ON s.product_id = p.product_id; -``` - ---- - -## Handling Expectations - -**DLT Python**: -```python -@dlt.expect_or_drop("valid_amount", "amount > 0") -@dlt.expect_or_fail("critical_id", "id IS NOT NULL") -``` - -**SDP SQL - Basic**: -```sql --- Use WHERE (equivalent to expect_or_drop) -WHERE amount > 0 AND id IS NOT NULL -``` - -**SDP SQL - Quarantine Pattern** (for auditing): -```sql --- Flag invalid records -CREATE OR REPLACE STREAMING TABLE bronze_data_flagged AS -SELECT - *, - CASE - WHEN amount <= 0 THEN TRUE - WHEN id IS NULL THEN TRUE - ELSE FALSE - END AS is_invalid -FROM STREAM bronze_data; - --- Clean for downstream -CREATE OR REPLACE STREAMING TABLE silver_data_clean AS -SELECT * FROM STREAM bronze_data_flagged WHERE NOT is_invalid; - --- Quarantine for investigation -CREATE OR REPLACE STREAMING TABLE silver_data_quarantine AS -SELECT * FROM STREAM bronze_data_flagged WHERE is_invalid; -``` - -**Migration**: `@dlt.expect_or_drop` → WHERE clause or quarantine pattern. - ---- - -## Handling UDFs - -### Simple UDFs (Migrate to SQL) - -**DLT Python**: -```python -@F.udf(returnType=StringType()) -def categorize_amount(amount): - if amount > 1000: - return "High" - elif amount > 100: - return "Medium" - else: - return "Low" - -@dlt.table(name="sales_categorized") -def sales_categorized(): - return ( - dlt.read("sales") - .withColumn("category", categorize_amount(F.col("amount"))) - ) -``` - -**SDP SQL** (CASE expression): -```sql -CREATE OR REPLACE MATERIALIZED VIEW sales_categorized AS -SELECT - *, - CASE - WHEN amount > 1000 THEN 'High' - WHEN amount > 100 THEN 'Medium' - ELSE 'Low' - END AS category -FROM sales; -``` - -### Complex UDFs (Stay in Python) - -**Keep in Python for**: -- Complex conditional logic -- External API calls -- Custom algorithms -- ML inference - -**Options**: -1. Keep transformation in Python DLT -2. Create hybrid (SQL + Python for specific UDFs) -3. Refactor to SQL built-ins if possible - ---- - -## Migration Process - -### Step 1: Inventory - -Document: -- Number of tables/views -- Python UDFs (simple vs complex) -- External dependencies -- Expectations and quality rules - -### Step 2: Categorize - -**Easy to migrate**: Filters, aggregations, simple CASE -**Moderate**: UDFs rewritable as SQL -**Hard**: Complex Python, external calls, ML - -### Step 3: Migrate by Layer - -1. **Bronze** (ingestion): Convert Auto Loader to read_files() -2. **Silver** (cleansing): Convert expectations to WHERE/quarantine -3. **Gold** (aggregations): Usually straightforward -4. **SCD/CDC**: Use AUTO CDC - -### Step 4: Test - -- Run both pipelines in parallel -- Compare outputs for correctness -- Validate performance -- Check quality metrics - ---- - -## When NOT to Migrate - -**Stay with DLT Python if**: -1. Heavy Python UDF usage (>30% of logic) -2. External API calls required -3. Custom ML model inference -4. Complex stateful operations not in SQL -5. Existing pipeline works well, team prefers Python -6. Limited SQL expertise - -**Consider hybrid**: SQL for most, Python for complex logic. - ---- - -## Common Issues - -| Issue | Solution | -|-------|----------| -| UDF doesn't translate | Keep in Python or refactor with SQL built-ins | -| Expectations differ | Use quarantine pattern to audit dropped records | -| Performance degradation | Use CLUSTER BY for Liquid Clustering, review joins | -| Schema evolution different | Use `mode => 'PERMISSIVE'` in read_files() | - ---- - -## Summary - -**Migration Path**: -1. Use decision matrix (80%+ SQL-expressible → migrate) -2. Migrate by layer (bronze → silver → gold) -3. Handle expectations with WHERE/quarantine -4. Translate simple UDFs to CASE expressions -5. Keep complex Python logic in Python - -**Key**: DLT Python and SDP SQL are both fully supported. Migrate for simplicity, not necessity. diff --git a/.claude/skills/databricks-spark-declarative-pipelines/9-auto_cdc.md b/.claude/skills/databricks-spark-declarative-pipelines/9-auto_cdc.md deleted file mode 100644 index b8a5b59..0000000 --- a/.claude/skills/databricks-spark-declarative-pipelines/9-auto_cdc.md +++ /dev/null @@ -1,353 +0,0 @@ -# AUTO CDC Patterns for Change Data Capture - -**Keywords**: Slow Changing Dimension, SCD, SCD Type 1, SCD Type 2, AUTO CDC, change data capture, dp.create_auto_cdc_flow, deduplication - ---- - -## Overview - -AUTO CDC automatically handles Change Data Capture (CDC) to track changes in your data using Slow Changing Dimensions (SCD). It provides automatic deduplication, change tracking, and handles late-arriving data correctly. - -**Where to apply AUTO CDC:** -- **Silver layer**: When business users need deduplicated or historical data for analytics/ML -- **Gold layer**: When implementing dimensional modeling (star schema) with dim/fact tables -- **Choice depends on**: Downstream consumption patterns and query requirements - ---- - -## SCD Type 1 vs Type 2 - -### SCD Type 1 (In-place updates) -- **Overwrites** old values with new values -- **No history preserved** - only current state maintained -- **Use for**: Dimension attributes that don't need history - - Correcting data errors (typos) - - Updating attributes where history doesn't matter - - Maintaining single current record per key -- **Syntax**: `stored_as_scd_type="1"` (string) - -### SCD Type 2 (History tracking) -- **Creates new row** for each change -- **Preserves full history** with `__START_AT` and `__END_AT` timestamps -- **Use for**: Tracking changes over time - - Customer address changes - - Product price history - - Employee role changes - - Any dimension requiring temporal analysis -- **Syntax**: `stored_as_scd_type=2` (integer) - ---- - -## Pattern: Cleaning + AUTO CDC - -### Step 1: Clean and Validate Data - -Create a cleaned streaming table with proper typing and quality checks: - -```python -# Cleaned data preparation (can be silver or intermediate layer) -from pyspark import pipelines as dp -from pyspark.sql import functions as F - -schema = spark.conf.get("schema") - -@dp.table( - name=f"{schema}.users_clean", - comment="Cleaned and validated user data with proper typing and quality checks", - cluster_by=["user_id"] -) -def users_clean(): - """ - Prepare clean data with: - - Proper timestamp typing - - Data quality validations - - Remove records with invalid email or null user_id - """ - return ( - spark.readStream.table("bronze_users") - .filter(F.col("user_id").isNotNull()) - .filter(F.col("email").isNotNull()) - .filter(F.col("email").rlike(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")) - .withColumn("created_timestamp", F.to_timestamp("created_timestamp")) - .withColumn("updated_timestamp", F.to_timestamp("updated_timestamp")) - .drop("_rescued_data") - .select( - "user_id", - "email", - "name", - "subscription_tier", - "country", - "created_timestamp", - "updated_timestamp", - "_ingested_at", - "_source_file" - ) - ) -``` - -### Step 2: Apply AUTO CDC (SCD Type 2) - -Create a history-tracked dimension table with full change history: - -```python -# AUTO CDC with SCD Type 2 (history tracking) -from pyspark import pipelines as dp - -target_schema = spark.conf.get("target_schema") -source_schema = spark.conf.get("source_schema") - -# Create the target table for AUTO CDC -dp.create_streaming_table(f"{target_schema}.dim_users") - -# Apply AUTO CDC (SCD Type 2) -dp.create_auto_cdc_flow( - target=f"{target_schema}.dim_users", - source=f"{source_schema}.users_clean", - keys=["user_id"], - sequence_by="updated_timestamp", - stored_as_scd_type=2 # Integer for Type 2 -) -``` - -**Resulting table will include**: -- All original columns from source -- `__START_AT` - When this version became effective -- `__END_AT` - When this version expired (NULL for current) - -### Step 3: Apply AUTO CDC (SCD Type 1) - -Create a deduplicated table with in-place updates (no history): - -```python -# AUTO CDC with SCD Type 1 (in-place updates) -from pyspark import pipelines as dp - -target_schema = spark.conf.get("target_schema") -source_schema = spark.conf.get("source_schema") - -# Create the target table for AUTO CDC -dp.create_streaming_table(f"{target_schema}.orders_current") - -# Apply AUTO CDC (SCD Type 1) -dp.create_auto_cdc_flow( - target=f"{target_schema}.orders_current", - source=f"{source_schema}.orders_clean", - keys=["order_id"], - sequence_by="updated_timestamp", - stored_as_scd_type="1" # String for Type 1 -) -``` - ---- - -## Key Benefits - -- **Automatic deduplication** based on keys - no manual MERGE logic -- **Automatic change tracking** with temporal metadata (`__START_AT`, `__END_AT`) -- **Handles late-arriving data** correctly using `sequence_by` timestamp -- **Simplified pipeline code** - no complex merge/upsert logic required -- **Built-in idempotency** - safe to reprocess data - ---- - -## Common Patterns - -### Pattern 1: Gold Dimensional Model - -Use AUTO CDC in Gold layer for star schema dimensions: - -```python -# Silver: Cleaned streaming tables -@dp.table(name="silver.customers_clean") -def customers_clean(): - return spark.readStream.table("bronze.customers").filter(...) - -# Gold: SCD Type 2 dimension -dp.create_streaming_table("gold.dim_customers") -dp.create_auto_cdc_flow( - target="gold.dim_customers", - source="silver.customers_clean", - keys=["customer_id"], - sequence_by="updated_at", - stored_as_scd_type=2 -) - -# Gold: Fact table (no AUTO CDC) -@dp.table(name="gold.fact_orders") -def fact_orders(): - return spark.read.table("silver.orders_clean") -``` - -### Pattern 2: Silver Deduplication for Joins - -Use AUTO CDC in Silver when joining multiple tables: - -```python -# Silver: AUTO CDC for deduplication -dp.create_streaming_table("silver.products_dedupe") -dp.create_auto_cdc_flow( - target="silver.products_dedupe", - source="bronze.products", - keys=["product_id"], - sequence_by="modified_at", - stored_as_scd_type="1" # Type 1: just dedupe, no history -) - -# Silver: Join with deduplicated data -@dp.table(name="silver.orders_enriched") -def orders_enriched(): - orders = spark.readStream.table("bronze.orders") - products = spark.read.table("silver.products_dedupe") - return orders.join(products, "product_id") -``` - -### Pattern 3: Mixed SCD Types - -Different tables use different SCD types based on requirements: - -```python -# SCD Type 2: Need history -dp.create_auto_cdc_flow( - target="gold.dim_customers", - source="silver.customers", - keys=["customer_id"], - sequence_by="updated_at", - stored_as_scd_type=2 # Track address changes over time -) - -# SCD Type 1: Corrections only -dp.create_auto_cdc_flow( - target="gold.dim_products", - source="silver.products", - keys=["product_id"], - sequence_by="modified_at", - stored_as_scd_type="1" # Current product info only -) -``` - ---- - -## Selective History Tracking - -Track history only for specific columns (SCD Type 2): - -```python -dp.create_auto_cdc_flow( - target="gold.dim_products", - source="silver.products_clean", - keys=["product_id"], - sequence_by="modified_at", - stored_as_scd_type=2, - track_history_column_list=["price", "cost"] # Only track these columns -) -``` - -When `price` or `cost` changes, a new version is created. Other column changes update the current record without creating new versions. - ---- - -## Using Temporary Views with AUTO CDC - -**`@dp.temporary_view()`** creates in-pipeline temporary views that exist only during pipeline execution. These are useful for intermediate transformations before AUTO CDC. - -**Key Constraints:** -- Cannot specify `catalog` or `schema` (temporary views are pipeline-scoped only) -- Cannot use `cluster_by` (not persisted) -- Only exists during pipeline execution - -**Use Cases:** -- Complex transformations before AUTO CDC -- Intermediate logic that's referenced multiple times -- Avoiding redundant transformations - -**Example: Preparation before AUTO CDC** - -```python -from pyspark import pipelines as dp -from pyspark.sql import functions as F - -# Step 1: Temporary view for complex business logic -@dp.temporary_view() -def orders_with_calculated_fields(): - """ - Temporary view for complex calculations. - No catalog/schema needed - exists only in pipeline. - """ - return ( - spark.readStream.table("bronze.orders") - .withColumn("order_total", F.col("quantity") * F.col("unit_price")) - .withColumn("discount_amount", F.col("order_total") * F.col("discount_rate")) - .withColumn("final_amount", F.col("order_total") - F.col("discount_amount")) - .withColumn("order_category", - F.when(F.col("final_amount") > 1000, "large") - .when(F.col("final_amount") > 100, "medium") - .otherwise("small") - ) - .filter(F.col("order_id").isNotNull()) - .filter(F.col("final_amount") > 0) - .filter(F.col("order_date").isNotNull()) - ) - -# Step 2: Apply AUTO CDC using the temporary view as source -target_schema = spark.conf.get("target_schema") - -dp.create_streaming_table(f"{target_schema}.orders_current") -dp.create_auto_cdc_flow( - target=f"{target_schema}.orders_current", - source="orders_with_calculated_fields", # Reference temporary view by name - keys=["order_id"], - sequence_by="order_date", - stored_as_scd_type="1" -) -``` - -**Benefits:** -- Avoids creating unnecessary persisted tables -- Reduces storage costs (nothing written to disk) -- Simplifies complex multi-step transformations -- Enables code reuse across multiple tables in same pipeline - ---- - -## Related Documentation - -- **[3-scd-query-patterns.md](3-scd-query-patterns.md)** - Querying SCD Type 2 history tables, point-in-time analysis, temporal joins -- **[1-ingestion-patterns.md](1-ingestion-patterns.md)** - CDC data sources (Kafka, Event Hubs, Kinesis) -- **[2-streaming-patterns.md](2-streaming-patterns.md)** - Deduplication patterns without AUTO CDC - ---- - -## Best Practices - -1. **Choose the right SCD type**: - - Type 2 when you need to query historical states - - Type 1 when you only need current state or deduplication - -2. **Use meaningful sequence_by column**: - - Should reflect true chronological order of changes - - Typically `updated_timestamp`, `modified_at`, or `event_timestamp` - -3. **Clean data before AUTO CDC**: - - Apply type casting, validation, and filtering first - - AUTO CDC works best with clean, well-typed data - -4. **Consider query patterns**: - - If analysts query history → Use Type 2 - - If analysts only need current → Use Type 1 - - If joining frequently → Consider Silver deduplication - -5. **Use selective tracking for large tables**: - - Track history only for columns that change meaningfully - - Reduces storage and improves query performance - ---- - -## Common Issues - -| Issue | Solution | -|-------|----------| -| **Duplicates still appearing** | Check `keys` include all business key columns; verify `sequence_by` has proper ordering | -| **Missing `__START_AT`/`__END_AT` columns** | These only appear in SCD Type 2 (integer), not Type 1 (string) | -| **Late data not handled** | Ensure `sequence_by` column is set and reflects true event time | -| **Type syntax error** | Type 2 uses integer `2`, Type 1 uses string `"1"` | -| **Performance issues** | Use `track_history_column_list` to limit which columns trigger new versions | diff --git a/.claude/skills/databricks-spark-declarative-pipelines/SKILL.md b/.claude/skills/databricks-spark-declarative-pipelines/SKILL.md index 144041e..a1bdd7c 100644 --- a/.claude/skills/databricks-spark-declarative-pipelines/SKILL.md +++ b/.claude/skills/databricks-spark-declarative-pipelines/SKILL.md @@ -1,148 +1,188 @@ --- name: databricks-spark-declarative-pipelines -description: "Creates, configures, and updates Databricks Lakeflow Spark Declarative Pipelines (SDP/LDP) using serverless compute. Handles streaming tables, materialized views, CDC, SCD Type 2, and Auto Loader ingestion patterns. Use when building data pipelines, working with Delta Live Tables, ingesting streaming data, implementing change data capture, or when the user mentions SDP, LDP, DLT, Lakeflow pipelines, streaming tables, or bronze/silver/gold medallion architectures." +description: "Creates, configures, and updates Databricks Lakeflow Spark Declarative Pipelines (SDP/LDP) using serverless compute. Handles data ingestion with streaming tables, materialized views, CDC, SCD Type 2, and Auto Loader ingestion patterns. Use when building data pipelines, working with Delta Live Tables, ingesting streaming data, implementing change data capture, or when the user mentions SDP, LDP, DLT, Lakeflow pipelines, streaming tables, or bronze/silver/gold medallion architectures." --- # Lakeflow Spark Declarative Pipelines (SDP) -IMPORTANT: If this is a new pipeline (one does not already exist), see Quick Start. Be sure to use whatever language user has specified only (Python or SQL). Be sure to use Databricks Asset Bundles for new projects. - --- ## Critical Rules (always follow) -- **MUST** confirm language as Python or SQL. Stick with that language unless told otherwise. -- **MUST** if not modifying an existing pipeline, use [Quick Start](#quick-start) below. -- **MUST** create serverless pipelines by default. ** Only use classic clusters if user explicitly requires R language, Spark RDD APIs, or JAR libraries. +### Syntax: CREATE OR REFRESH (not CREATE OR REPLACE) +- **MUST** use `CREATE OR REFRESH` for SDP objects: + - `CREATE OR REFRESH STREAMING TABLE` - for streaming tables + - `CREATE OR REFRESH MATERIALIZED VIEW` - for materialized views +- **NEVER** use `CREATE OR REPLACE` - that is standard SQL syntax, not SDP syntax -## Required Steps +### Simplicity First +- **MUST** create the minimal number of tables to solve the task +- Simplicity first: prefer single pipeline even for multi-schema setups - use fully qualified names (`catalog.schema.table`) +- When asked to "create a silver table" or "create a gold table", create **ONE table** - not a multi-layer pipeline +- Don't add intermediate tables, staging tables, or helper views unless explicitly requested +- A silver transformation = 1 streaming table reading from bronze +- A gold aggregation = 1 materialized view reading from silver +- Create bronze→silver→gold chains when the user asks for a "pipeline" or "medallion architecture" or full/detailed ingestion. Otherwise keep it simple - don't over engineer. -Copy this checklist and verify each item: -``` -- [ ] Language selected: Python or SQL -- [ ] Compute type decided: serverless or classic compute -- [ ] Decide on multiple catalogs or schemas vs. all in one default schema -- [ ] Consider what should be parameterized at the pipeline level to make deployment easy. -- [ ] Consider [Multi-Schema Patterns](#multi-schema-patterns) below, ask if unclear on best choices. -- [ ] Consider [Modern Defaults](#modern-defaults) below, ask if unclear on best choices. +### Language Selection +- **MUST** know the language (Python or SQL). For simple task / pipeline / table creation, pick SQL. For complex pipeline with parametrized information, or if the user mentions python-related items pick python. If you have a doubt, ask the user. Stick with that language unless told otherwise. +| User Says | Action | +|-----------|--------| +| "Python pipeline", "Python SDP", "use Python", "udf", "pandas", "ml inference", "pyspark" | **User wants Python** | +| "SQL pipeline", "SQL files", "use SQL" | **User wants SQL** | +| "Create a simple pipeline", "create a table", "an aggregation" | **Pick SQL as it's simple** | -## Quick Start: Initialize New Pipeline Project +### Other Rules +- **MUST** create serverless pipelines by default. Only use classic clusters if user explicitly requires R language, Spark RDD APIs, or JAR libraries. +- **MUST** choose the right workflow based on context (see below). +- When the user provides table schema and asks for code, respond directly with the code. Don't ask clarifying questions if the request is clear. -**RECOMMENDED**: Use `databricks pipelines init` to create production-ready Asset Bundle projects with multi-environment support. +## Tools +- List files in volume: `databricks fs ls dbfs:/Volumes/{catalog}/{schema}/{volume}/{path} --profile {PROFILE}` +- Query data: `databricks experimental aitools tools query --profile {PROFILE} --warehouse abc123 "SELECT 1 FROM catalog.schema.table"` +- Discover schema: `databricks experimental aitools tools discover-schema --profile {PROFILE} catalog.schema.table1 catalog.schema.table2` +- Pipelines CLI: `databricks pipelines init|deploy|run|logs|stop` or use `databricks pipelines --help` for more options -### When to Use Bundle Initialization +## Choose Your Workflow -Use bundle initialization for **New pipeline projects** for a professional structure from the start +**First, determine which workflow to use:** -Use manual workflow for: -- Quick prototyping without multi-environment needs -- Existing manual projects you want to continue -- Learning/experimentation +### Option A: Standalone New Pipeline Project (use `databricks pipelines init`) -### Step 1: Initialize Project +Use this when the user wants to **create a new, standalone SDP project** that will have its own DAB: +- User asks: "Create a new pipeline", "Build me an SDP", "Set up a new data pipeline" +- No existing `databricks.yml` in the workspace +- The pipeline IS the project (not part of a larger demo/app) -I will automatically run this command when you request a new pipeline: +Use `databricks pipeline` CLI commands: ```bash -databricks pipelines init +databricks pipelines init --output-dir . --config-file init-config.json ``` -**Interactive Prompts:** -- **Project name**: e.g., `customer_orders_pipeline` -- **Initial catalog**: Unity Catalog name (e.g., `main`, `prod_catalog`) -- **Personal schema per user?**: `yes` for dev (each user gets their own schema), `no` for prod -- **Language**: SQL or Python (auto-detected from your request - see language detection below) - -**Generated Structure:** -``` -my_pipeline/ -├── databricks.yml # Multi-environment config (dev/prod) -├── resources/ -│ └── *_etl.pipeline.yml # Pipeline resource definition -└── src/ - └── *_etl/ - ├── explorations/ # Exploratory code in .ipynb - └── transformations/ # Your .sql or .py files here +**Example init-config.json:** +```json +{ + "project_name": "customer_pipeline", + "initial_catalog": "prod_catalog", + "use_personal_schema": "no", + "initial_language": "sql" +} ``` -### Step 2: Customize Transformations +→ See [1-project-initialization.md](references/1-project-initialization.md) +→ -Replace the example code created by the init process with custom transformation files in `src/transformations/` based on provided requirements, using best practice guidance from this skill. -**For Python pipelines using cloudFiles**: Ask the user where to store Auto Loader schema metadata. Recommend: -``` -/Volumes/{catalog}/{schema}/{pipeline_name}_metadata/schemas -``` +### Option B: Pipeline within Existing Bundle (edit the bundle) -### Step 3: Deploy and Run +Use this when the pipeline is **part of an existing DAB project**: +- There's already a `databricks.yml` file in the project +- User is adding a pipeline to an existing app/demo -```bash -# Deploy to workspace (dev by default) -databricks bundle deploy +→ See [1-project-initialization.md](references/1-project-initialization.md) for adding pipelines to existing bundles + +### Option C: Rapid Iteration with MCP Tools (no bundle management) + +Use this when you need to **quickly create, test, and iterate** on a pipeline without managing bundle files: +- User wants to "just run a pipeline and see if it works" +- Part of a larger demo where bundle is managed separately, or the DAB bundle will be created at the end as you want to quickly test the project first +- Prototyping or experimenting with pipeline logic +- User explicitly asks to use MCP tools + +→ See [2-mcp-approach.md](references/2-mcp-approach.md) for MCP-based workflow -# Run pipeline -databricks bundle run my_pipeline_etl +--- + +## Required Checklist -# Deploy to production -databricks bundle deploy --target prod +Before writing pipeline code, make sure you have: +``` +- [ ] Language selected: Python or SQL +- [ ] Read the syntax basics: **SQL**: Always Read [sql/1-syntax-basics.md](references/sql/1-syntax-basics.md), **Python**: Always Read [python/1-syntax-basics.md](references/python/1-syntax-basics.md) +- [ ] Workflow chosen: Standalone DAB / Existing DAB / MCP iteration +- [ ] Compute type: serverless (default) or classic +- [ ] Schema strategy: single schema with prefixes vs. multi-schema +- [ ] Consider [Multi-Schema Patterns](#multi-schema-patterns) and [Modern Defaults](#modern-defaults) ``` +**Then read additional guides based on what the pipeline needs, when you need it:** +| If the pipeline needs... | Read | +|--------------------------|------| +| File ingestion (Auto Loader, JSON, CSV, Parquet) | `references/sql/2-ingestion.md` or `references/python/2-ingestion.md` | +| Kafka, Event Hub, or Kinesis streaming | `references/sql/2-ingestion.md` or `references/python/2-ingestion.md` | +| Deduplication, windowed aggregations, joins | `references/sql/3-streaming-patterns.md` or `references/python/3-streaming-patterns.md` | +| CDC, SCD Type 1/2, or history tracking | `references/sql/4-cdc-patterns.md` or `references/python/4-cdc-patterns.md` | +| Performance tuning, Liquid Clustering | `references/sql/5-performance.md` or `references/python/5-performance.md` | + +--- ## Quick Reference | Concept | Details | |---------|---------| -| **Names** | SDP = Spark Declarative Pipelines = LDP = Lakeflow Declarative Pipelines = Lakeflow Pipelines (all interchangeable) | +| **Names** | SDP = Spark Declarative Pipelines = LDP = Lakeflow Declarative Pipelines (all interchangeable) | +| **SQL Syntax** | `CREATE OR REFRESH STREAMING TABLE`, `CREATE OR REFRESH MATERIALIZED VIEW` | | **Python Import** | `from pyspark import pipelines as dp` | | **Primary Decorators** | `@dp.table()`, `@dp.materialized_view()`, `@dp.temporary_view()` | -| **Temporary Views** | `@dp.temporary_view()` creates in-pipeline temporary views (no catalog/schema, no cluster_by). Useful for intermediate logic before AUTO CDC or when a view needs multiple references without persistence. | -| **Replaces** | Delta Live Tables (DLT) with `import dlt` | -| **Based On** | Apache Spark 4.1+ (Databricks' modern data pipeline framework) | -| **Docs** | https://docs.databricks.com/aws/en/ldp/developer/python-dev | - ---- - -## Detailed guides - -**Ingestion patterns**: Use [1-ingestion-patterns.md](1-ingestion-patterns.md) when planning how to get new data into your Lakeflow pipeline —- covers file formats, batch/streaming options, and tips for incremental and full loads. (Keywords: Auto Loader, Kafka, Event Hub, Kinesis, file formats) - -**Streaming pipeline patterns**: See [2-streaming-patterns.md](2-streaming-patterns.md) for designing pipelines with streaming data sources, change data detection, triggers, and windowing. (Keywords: deduplication, windowing, stateful operations, joins) -**SCD query patterns**: See [3-scd-query-patterns.md](3-scd-query-patterns.md) for querying Slowly Changing Dimensions Type 2 history tables, including current state queries, point-in-time analysis, temporal joins, and change tracking. (Keywords: SCD Type 2 history tables, temporal joins, querying historical data) +### Legacy APIs (Do NOT Use) -**Performance tuning**: Use [4-performance-tuning.md](4-performance-tuning.md) for optimizing pipelines with Liquid Clustering, state management, and best practices for high-performance streaming workloads. (Keywords: Liquid Clustering, optimization, state management) +| Legacy | Modern Replacement | +|--------|-------------------| +| `import dlt` | `from pyspark import pipelines as dp` | +| `dlt.apply_changes()` | `dp.create_auto_cdc_flow()` | +| `dlt.read()` / `dlt.read_stream()` | `spark.read` / `spark.readStream` | +| `CREATE LIVE XXX` | `CREATE OR REFRESH STREAMING TABLE\|MATERIALIZED VIEW` | +| `PARTITION BY` + `ZORDER` | `CLUSTER BY` (Liquid Clustering) | +| `input_file_name()` | `_metadata.file_path` | +| `target` parameter | `schema` parameter | -**Python API reference**: See [5-python-api.md](5-python-api.md) for the modern `pyspark.pipelines` (dp) API reference and migration from legacy `dlt` API patterns. (Keywords: dp API, dlt API comparison) +### Streaming Table vs Materialized View -**DLT migration**: Use [6-dlt-migration.md](6-dlt-migration.md) when migrating existing Delta Live Tables (DLT) pipelines to Spark Declarative Pipelines (SDP). (Keywords: migrating DLT pipelines to SDP) +| Use Case | Type | Pattern | +|----------|------|---------| +| Windowed aggregations (tumbling, sliding, session) | Streaming Table | `FROM stream(source)` + `GROUP BY window()` | +| Full-table aggregations (totals, daily counts) | Materialized View | `FROM source` (no stream wrapper) | +| CDC / SCD Type 2 | Streaming Table | `AUTO CDC INTO` or `dp.create_auto_cdc_flow()` | -**Advanced configuration**: See [7-advanced-configuration.md](7-advanced-configuration.md) for advanced pipeline settings including development mode, continuous execution, notifications, Python dependencies, and custom cluster configurations. (Keywords: extra_settings parameter reference, examples) - -**Project initialization**: Use [8-project-initialization.md](8-project-initialization.md) for setting up new pipeline projects with `databricks pipelines init`, Asset Bundles, multi-environment deployments, and language detection logic. (Keywords: databricks pipelines init, Asset Bundles, language detection, migration guides) - -**AUTO CDC patterns**: Use [9-auto_cdc.md](9-auto_cdc.md) for implementing Change Data Capture with AUTO CDC, including Slow Changing Dimensions (SCD Type 1 and Type 2) for tracking changes and deduplication. (Keywords: AUTO CDC, Slow Changing Dimension, SCD, SCD Type 1, SCD Type 2, change data capture, deduplication) +Use streaming tables for windowed aggregations to enable incremental processing. Use materialized views for simple aggregations that recompute fully on each refresh. --- -## Workflow - -1. Determine the task type: +## Task-Based Routing + +After choosing your workflow (see [Choose Your Workflow](#choose-your-workflow)), determine the specific task: + +**Choose documentation by language:** + +### SQL Documentation +| Task | Guide | +|------|-------| +| **SQL syntax basics** | [sql/1-syntax-basics.md](references/sql/1-syntax-basics.md) | +| **Data ingestion (Auto Loader, Kafka)** | [sql/2-ingestion.md](references/sql/2-ingestion.md) | +| **Streaming patterns (deduplication, windows)** | [sql/3-streaming-patterns.md](references/sql/3-streaming-patterns.md) | +| **CDC patterns (AUTO CDC, SCD, queries)** | [sql/4-cdc-patterns.md](references/sql/4-cdc-patterns.md) | +| **Performance tuning** | [sql/5-performance.md](references/sql/5-performance.md) | + +### Python Documentation +| Task | Guide | +|------|-------| +| **Python syntax basics** | [python/1-syntax-basics.md](references/python/1-syntax-basics.md) | +| **Data ingestion (Auto Loader, Kafka)** | [python/2-ingestion.md](references/python/2-ingestion.md) | +| **Streaming patterns (deduplication, windows)** | [python/3-streaming-patterns.md](references/python/3-streaming-patterns.md) | +| **CDC patterns (AUTO CDC, SCD, queries)** | [python/4-cdc-patterns.md](references/python/4-cdc-patterns.md) | +| **Performance tuning** | [python/5-performance.md](references/python/5-performance.md) | + +### General Documentation +| Task | Guide | +|------|-------| +| **Setting up standalone pipeline project** | [1-project-initialization.md](references/1-project-initialization.md) | +| **Rapid iteration with MCP tools** | [2-mcp-approach.md](references/2-mcp-approach.md) | +| **Advanced configuration** | [3-advanced-configuration.md](references/3-advanced-configuration.md) | +| **Migrating from DLT** | [4-dlt-migration.md](references/4-dlt-migration.md) | - **Setting up new project?** → Read [8-project-initialization.md](8-project-initialization.md) first - **Creating new pipeline?** → Read [1-ingestion-patterns.md](1-ingestion-patterns.md) - **Creating stream table?** → Read [2-streaming-patterns.md](2-streaming-patterns.md) - **Querying SCD history tables?** → Read [3-scd-query-patterns.md](3-scd-query-patterns.md) - **Implementing AUTO CDC or SCD?** → Read [9-auto_cdc.md](9-auto_cdc.md) - **Performance issues?** → Read [4-performance-tuning.md](4-performance-tuning.md) - **Using Python API?** → Read [5-python-api.md](5-python-api.md) - **Migrating from DLT?** → Read [6-dlt-migration.md](6-dlt-migration.md) - **Advanced configuration?** → Read [7-advanced-configuration.md](7-advanced-configuration.md) - **Validating?** → Read [validation-checklist.md](validation-checklist.md) - -2. Follow the instructions in the relevant guide - -3. Repeat for next task type --- ## Official Documentation @@ -154,365 +194,137 @@ databricks bundle deploy --target prod - **[Change Data Capture (CDC)](https://docs.databricks.com/aws/en/ldp/cdc)** - AUTO CDC, SCD Type 1/2 -### Medallion Architecture Pattern - **Bronze Layer (Raw)** - - Raw data ingested from sources in original format - - Minimal transformations (append-only, add metadata like `_ingested_at`, `_source_file`) - - Single source of truth preserving data lineage - - **Silver Layer (Validated)** - - Cleaned and validated data. - - Might deduplicate here with auto_cdc, but often wait until the final step for auto_cdc if possible. - - Business logic applied (type casting, quality checks, filtering invalid records) - - Enterprise view of key business entities - - Enables self-service analytics and ML - - **Gold Layer (Business-Ready)** - - Aggregated, denormalized, project-specific tables - - Optimized for consumption (reporting, dashboards, BI tools) - - Fewer joins, read-optimized data models - - Kimball star schema tables - dim_, fact_ - - Deduplication often happens here via Slow Changing Dimensions (SCD), using auto_cdc. Sometimes that will happen upstream in silver instead, such as when joining multiple tables or business users plan to query the table from silver. - - **Typical Flow (Can vary)** - Bronze: read_files() or spark.readStream.format("cloudFiles") → streaming table - Silver: read bronze → filter/clean/validate → streaming table - Gold: read silver → aggregate/denormalize → auto_cdc or materialized view - - Sources: - - https://www.databricks.com/glossary/medallion-architecture - - https://docs.databricks.com/aws/en/lakehouse/medallion - - https://www.databricks.com/blog/2022/06/24/data-warehousing-modeling-techniques-and-their-implementation-on-the-databricks-lakehouse-platform.html - +### Medallion Architecture + +| Layer | SDP Pattern | Common Practices | +|-------|-------------|------------------| +| **Bronze** | `STREAM read_files()` → streaming table | Often adds `_metadata.file_path`, `_ingested_at`. Minimal transforms, append-only. | +| **Silver** | `stream(bronze)` → streaming table | Clean/validate, type casting, quality filters. Prefer `DECIMAL(p,s)` for money. Dedup can happen here or gold. | +| **Gold** | `AUTO CDC INTO` or materialized view | Aggregated, denormalized. SCD/dedup often via `AUTO CDC`. Star schema typically uses `dim_*`/`fact_*`. | + +#### Gold Layer: Preserve Key Dimensions + +When aggregating data in gold tables, **keep the main business dimensions** to enable flexible analysis. Over-aggregating loses information that analysts may need later. + +**Guidance based on context:** +- **If a dashboard is mentioned**: Include all dimensions that appear as filters. Dashboard filters only work if the underlying data has those columns. +- **If analysis by dimension is mentioned** (e.g., "analyze by store", "breakdown by department"): Include those dimensions in the aggregation. +- **If no specific instructions**: Default to keeping key business dimensions (location, department, product line, customer segment, time period) rather than aggregating them away. This preserves flexibility for future analysis. + +**Rule of thumb**: If users might want to slice the data by a dimension, include it in the gold table. It's easier to aggregate further in queries than to recover lost dimensions. + **For medallion architecture** (bronze/silver/gold), two approaches work: - **Flat with naming** (template default): `bronze_*.sql`, `silver_*.sql`, `gold_*.sql` - **Subdirectories**: `bronze/orders.sql`, `silver/cleaned.sql`, `gold/summary.sql` -Both work with the `transformations/**` glob pattern. Choose based on preference. +Both work with the `transformations/**` glob pattern. Choose based on preference/existing. -See **[8-project-initialization.md](8-project-initialization.md)** for complete details on bundle initialization, migration, and troubleshooting. +See **[1-project-initialization.md](references/1-project-initialization.md)** for complete details on bundle initialization, migration, and troubleshooting. --- ## General SDP development guidance -### Step 1: Write Pipeline Files Locally -Create `.sql` or `.py` files in a local folder: - -``` -my_pipeline/ -├── bronze/ -│ ├── ingest_orders.sql # SQL (default for most cases) -│ └── ingest_events.py # Python (for complex logic) -├── silver/ -│ └── clean_orders.sql -└── gold/ - └── daily_summary.sql -``` - -**SQL Example** (`bronze/ingest_orders.sql`): +**SQL Example:** ```sql CREATE OR REFRESH STREAMING TABLE bronze_orders CLUSTER BY (order_date) -AS -SELECT - *, - current_timestamp() AS _ingested_at, - _metadata.file_path AS _source_file -FROM read_files( - '/Volumes/catalog/schema/raw/orders/', - format => 'json', - schemaHints => 'order_id STRING, customer_id STRING, amount DECIMAL(10,2), order_date DATE' -); +AS SELECT *, current_timestamp() AS _ingested_at +FROM STREAM read_files('/Volumes/catalog/schema/raw/orders/', format => 'json'); ``` -**Python Example** (`bronze/ingest_events.py`): +**Python Example:** ```python from pyspark import pipelines as dp -from pyspark.sql.functions import col, current_timestamp - -# Get schema location from pipeline configuration -schema_location_base = spark.conf.get("schema_location_base") @dp.table(name="bronze_events", cluster_by=["event_date"]) def bronze_events(): - return ( - spark.readStream.format("cloudFiles") - .option("cloudFiles.format", "json") - .option("cloudFiles.schemaLocation", f"{schema_location_base}/bronze_events") - .load("/Volumes/catalog/schema/raw/events/") - .withColumn("_ingested_at", current_timestamp()) - .withColumn("_source_file", col("_metadata.file_path")) - ) + return spark.readStream.format("cloudFiles").option("cloudFiles.format", "json").load("/Volumes/...") ``` -**IMPORTANT for Python Pipelines**: When using `spark.readStream.format("cloudFiles")` for cloud storage ingestion, with schema inference (no schema specified), you **must specify a schema location**. - -**Always ask the user** where to store Auto Loader schema metadata. Recommend: -``` -/Volumes/{catalog}/{schema}/{pipeline_name}_metadata/schemas -``` - -Example: `/Volumes/my_catalog/pipeline_metadata/orders_pipeline_metadata/schemas` - -**Never use the source data volume** - this causes permission conflicts. The schema location should be configured in the pipeline settings and accessed via `spark.conf.get("schema_location_base")`. - -**Language Selection:** - -**CRITICAL RULE**: If the user explicitly mentions "Python" in their request (e.g., "Python Spark Declarative Pipeline", "Python SDP", "use Python"), **ALWAYS use Python without asking**. The same applies to SQL - if they say "SQL pipeline", use SQL. - -- **Explicit language request**: User says "Python" → Use Python. User says "SQL" → Use SQL. **Do not ask for clarification.** -- **Auto-detection** (only when no explicit language mentioned): - - **SQL indicators**: "sql files", "simple transformations", "aggregations", "materialized view", "CREATE OR REFRESH" - - **Python indicators**: ".py files", "UDF", "complex logic", "ML inference", "external API", "@dp.table", "pandas", "decorator" -- **Prompt for clarification** only when language intent is truly ambiguous (no explicit mention, mixed signals) -- **Default to SQL** only when ambiguous AND no Python indicators present - -See **[8-project-initialization.md](8-project-initialization.md)** for detailed language detection logic. - - -## Option 1: Pipelines with DABs: -Use asset bundles and pipeline CLI. -See [Quick Start](#quick-start) and **[8-project-initialization.md](8-project-initialization.md)** for complete details. - -## Option 2: Manual Workflow (Advanced) - -For rapid prototyping, experimentation, or when you prefer direct control without Asset Bundles, use the manual workflow with MCP tools. - -Use MCP tools to create, run, and iterate on **serverless SDP pipelines**. The **primary tool is `create_or_update_pipeline`** which handles the entire lifecycle. - -**IMPORTANT: Always create serverless pipelines (default).** Only use classic clusters if user explicitly ask for classic, pro, advances compute or requires R language, Spark RDD APIs, or JAR libraries. - -See **[10-mcp-approach.md](10-mcp-approach.md)** for detailed guide. - +For detailed syntax, see [sql/1-syntax-basics.md](references/sql/1-syntax-basics.md) or [python/1-syntax-basics.md](references/python/1-syntax-basics.md). ## Best Practices (2026) ### Project Structure -- **Default to `databricks pipelines init`** for new projects (creates Asset Bundle) -- **Use Asset Bundles** for multi-environment deployments (dev/staging/prod) -- **Manual structure only** for quick prototypes or legacy migration -- **Medallion architecture**: Two approaches work with Asset Bundles: - - **Flat structure** (template default): `bronze_*.sql`, `silver_*.sql`, `gold_*.sql` in `transformations/` - - **Subdirectories**: `transformations/bronze/`, `transformations/silver/`, `transformations/gold/` - - Both work with the `transformations/**` glob pattern - choose based on team preference -- See **[8-project-initialization.md](8-project-initialization.md)** for project setup details +- **Standalone pipeline projects**: Use `databricks pipelines init` for Asset Bundle with multi-environment support +- **Pipeline in existing bundle**: Add to `resources/*.pipeline.yml` +- **Rapid iteration/prototyping**: Use MCP tools, formalize in bundle later +- See **[1-project-initialization.md](references/1-project-initialization.md)** for project setup details ### Minimal pipeline config pointers - Define parameters in your pipeline’s configuration and access them in code with spark.conf.get("key"). - In Databricks Asset Bundles, set these under resources.pipelines..configuration; validate with databricks bundle validate. ### Modern Defaults -- **CLUSTER BY** (Liquid Clustering), not PARTITION BY - see [4-performance-tuning.md](4-performance-tuning.md) -- **Raw `.sql`/`.py` files**, not notebooks -- **Serverless compute ONLY** - Do not use classic clusters unless explicitly required +- **Always use raw `.sql`/`.py` files for the transformations files** - NO notebooks in your pipeline. Pipeline code must be plain files. +- **Databricks notebook source for explorations** - Use `# Databricks notebook source` format with `# COMMAND ----------` separators for ad-hoc queries. See [examples/exploration_notebook.py](scripts/exploration_notebook.py). +- **Serverless compute** - Do not use classic clusters unless explicitly required (R, RDD APIs, JAR libraries) - **Unity Catalog** (required for serverless) -- **read_files()** when using SQL for cloud storage ingestion - see [1-ingestion-patterns.md](1-ingestion-patterns.md) +- **CLUSTER BY** (Liquid Clustering), not PARTITION BY with ZORDER - see [sql/5-performance.md](references/sql/5-performance.md) or [python/5-performance.md](references/python/5-performance.md) +- **read_files()** for SQL cloud storage ingestion - always consume a folder, not a single file - see [sql/2-ingestion.md](references/sql/2-ingestion.md) ### Multi-Schema Patterns -**Default: Single target schema per pipeline.** Each pipeline has one target `catalog` and `schema` where all tables are written. - - -#### Option 1: Single Pipeline, Single Schema with Prefixes (Recommended) - -Use one schema with table name prefixes to distinguish layers: - -```python -# All tables write to: catalog.schema.bronze_*, silver_*, gold_* -@dp.table(name="bronze_orders") # → catalog.schema.bronze_orders -@dp.table(name="silver_orders") # → catalog.schema.silver_orders -@dp.table(name="gold_summary") # → catalog.schema.gold_summary -``` - -**Advantages:** -- Simpler configuration (one pipeline) -- All tables in one schema for easy discovery - -#### Option 2: -Use varaiables to specific separate catalog and/or schema for different steps. - -Below are Python SDP examples that source variables from pipeline configs via spark.conf.get, and use the default catalog/schema for bronze. - -##### Same catalog, separate schemas; bronze uses pipeline defaults -- Set your pipeline’s default catalog and default schema to the bronze layer (for example, catalog=my_catalog, schema=bronze). When you omit catalog/schema in code, reads/writes go to these defaults. -- Use pipeline parameters for the other schemas and any source schema/path, retrieved in code with spark.conf.get(...). - -```python -from pyspark import pipelines as dp -from pyspark.sql.functions import col - -# Pull variables from pipeline configuration parameters -silver_schema = spark.conf.get("silver_schema") # e.g., "silver" -gold_schema = spark.conf.get("gold_schema") # e.g., "gold" -landing_schema = spark.conf.get("landing_schema") # e.g., "landing" - -# Bronze → uses default catalog/schema (set to bronze in pipeline settings) -@dp.table(name="orders_bronze") -def orders_bronze(): - # Read from another schema in the same default catalog - return spark.readStream.table(f"{landing_schema}.orders_raw") - -# Silver → same catalog, schema from parameter -@dp.table(name=f"{silver_schema}.orders_clean") -def orders_clean(): - return (spark.read.table("orders_bronze") # unqualified = default catalog/schema - .filter(col("order_id").isNotNull())) - -# Gold → same catalog, schema from parameter -@dp.materialized_view(name=f"{gold_schema}.orders_by_date") -def orders_by_date(): - return (spark.read.table(f"{silver_schema}.orders_clean") - .groupBy("order_date") - .count().withColumnRenamed("count", "order_count")) -``` -- Using unqualified names for bronze ensures it lands in the pipeline’s default catalog/schema; silver/gold are explicitly schema-qualified within the same catalog. +**Preferred: One pipeline writing to multiple schemas** using fully qualified table names (`catalog.schema.table`). This keeps dependencies clear and is simpler to manage than multiple pipelines. ---- +- **Python**: `@dp.table(name="catalog.bronze_schema.orders")` +- **SQL**: `CREATE OR REFRESH STREAMING TABLE catalog.silver_schema.orders_clean AS ...` -##### Custom catalog/schema per layer; bronze still uses pipeline defaults -- Keep bronze in the pipeline defaults (default catalog/schema set to your bronze layer). For silver/gold, use fully-qualified names with catalog and schema variables from pipeline configuration. +For detailed examples, see **[3-advanced-configuration.md](references/3-advanced-configuration.md#multi-schema-patterns)**. -```python -from pyspark import pipelines as dp -from pyspark.sql.functions import col - -# Pull variables from pipeline configuration parameters -silver_catalog = spark.conf.get("silver_catalog") # e.g., "my_catalog" -silver_schema = spark.conf.get("silver_schema") # e.g., "silver" -gold_catalog = spark.conf.get("gold_catalog") # e.g., "my_catalog" -gold_schema = spark.conf.get("gold_schema") # e.g., "gold" -landing_catalog = spark.conf.get("landing_catalog") # optional, if source is in another catalog -landing_schema = spark.conf.get("landing_schema") - -# Bronze → uses default catalog/schema (set to bronze) -@dp.table(name="orders_bronze") -def orders_bronze(): - # If source is in a specified catalog/schema: - return spark.readStream.table(f"{landing_catalog}.{landing_schema}.orders_raw") - -# Silver → custom catalog + schema via parameters -@dp.table(name=f"{silver_catalog}.{silver_schema}.orders_clean") -def orders_clean(): - # Read bronze by its unqualified name (defaults), or fully qualify if preferred - return (spark.read.table("orders_bronze") - .filter(col("order_id").isNotNull())) - -# Gold → custom catalog + schema via parameters -@dp.materialized_view(name=f"{gold_catalog}.{gold_schema}.orders_by_date}") -def orders_by_date(): - return (spark.read.table(f"{silver_catalog}.{silver_schema}.orders_clean") - .groupBy("order_date") - .count().withColumnRenamed("count", "order_count")) -``` -- Multipart names in the decorator’s name argument let you publish to explicit catalog.schema targets within one pipeline. -- Unqualified reads/writes use the pipeline defaults; use fully-qualified names when crossing catalogs or when you need explicit namespace control. +**Fallback**: If all tables must be in the same schema, use name prefixes (`bronze_*`, `silver_*`, `gold_*`). --- +## Post-Run Validation (Required) -**Note:** The `@dp.table()` decorator does not currently support separate for `schema=` or `catalog=` parameters. The table parameter is a string that contains the catalog.schema.table_name, or it can leave off catalog and or schema to use the pipeilnes configured default target schema. - -### Reading Tables in Python - -**Modern SDP Best Practice:** -- Use `spark.read.table()` for batch reads -- Use `spark.readStream.table()` for streaming reads -- Don't use `dp.read()` or `dp.read_stream()` (old syntax, no longer documented) -- Don't use `dlt.read()` or `dlt.read_stream()` (legacy DLT API) - -**Key Point:** SDP automatically tracks table dependencies from standard Spark DataFrame operations. No special read APIs are needed. - -#### Three-Tier Identifier Resolution +After running a pipeline (via DAB or MCP), you **MUST** validate both the execution status AND the actual data. -SDP supports three levels of table name qualification: +### Step 1: Check Pipeline Execution Status -| Level | Syntax | When to Use | -|-------|--------|-------------| -| **Unqualified** | `spark.read.table("my_table")` | Reading tables within the same pipeline's target catalog/schema (recommended) | -| **Partially-qualified** | `spark.read.table("other_schema.my_table")` | Reading from different schema in same catalog | -| **Fully-qualified** | `spark.read.table("other_catalog.other_schema.my_table")` | Reading from external catalogs/schemas | +**From MCP (`manage_pipeline(action="run")` or `manage_pipeline(action="create_or_update")`):** +- Check `result["success"]` and `result["state"]` +- If failed, check `result["message"]` and `result["errors"]` for details -#### Option 1: Unqualified Names (Recommended for Pipeline Tables) +**From DAB (`databricks bundle run`):** +- Check the command output for success/failure +- Use `manage_pipeline(action="get", pipeline_id=...)` to get detailed status and recent events -**Best practice for tables within the same pipeline.** SDP resolves unqualified names to the pipeline's configured target catalog and schema. This makes code portable across environments (dev/prod). +### Step 2: Validate Output Data -```python -@dp.table(name="silver_clean") -def silver_clean(): - # Reads from pipeline's target catalog/schema (e.g., dev_catalog.dev_schema.bronze_raw) - return ( - spark.read.table("bronze_raw") - .filter(F.col("valid") == True) - ) - -@dp.table(name="silver_events") -def silver_events(): - # Streaming read from same pipeline's bronze_events table - return ( - spark.readStream.table("bronze_events") - .withColumn("processed_at", F.current_timestamp()) - ) -``` - -#### Option 2: Pipeline Parameters (For External Sources) +Even if the pipeline reports SUCCESS, you **MUST** verify the data is correct: -**Use `spark.conf.get()` to parameterize external catalog/schema references.** Define parameters in pipeline configuration, then reference them at the module level. - -```python -from pyspark import pipelines as dp -from pyspark.sql import functions as F - -# Get parameterized values at module level (evaluated once at pipeline start) -source_catalog = spark.conf.get("source_catalog") -source_schema = spark.conf.get("source_schema", "sales") # with default - -@dp.table(name="transaction_summary") -def transaction_summary(): - return ( - spark.read.table(f"{source_catalog}.{source_schema}.transactions") - .groupBy("account_id") - .agg( - F.count("txn_id").alias("txn_count"), - F.sum("txn_amount").alias("account_revenue") - ) - ) ``` - -**Configure parameters in pipeline settings:** -- **Asset Bundles**: Add to `pipeline.yml` under `configuration:` -- **Manual/MCP**: Pass via `extra_settings.configuration` dict - -```yaml -# In resources/my_pipeline.pipeline.yml -configuration: - source_catalog: "shared_catalog" - source_schema: "sales" +# MCP Tool: get_table_stats_and_schema - validates schema, row counts, and stats +get_table_stats_and_schema( + catalog="my_catalog", + schema="my_schema", + table_names=["bronze_*", "silver_*", "gold_*"] # Use glob patterns +) ``` -#### Option 3: Fully-Qualified Names (For Fixed External References) +**Check for:** +- Empty tables (row_count = 0) - indicates ingestion or filtering issues +- Unexpected row counts - joins may have exploded or filtered too much +- Missing columns - schema mismatch or transformation errors +- NULL values in key columns - data quality issues -Use when referencing specific external tables that don't change across environments: +### Step 3: Debug Data Issues -```python -@dp.table(name="enriched_orders") -def enriched_orders(): - # Pipeline-internal table (unqualified) - orders = spark.read.table("bronze_orders") +If validation reveals problems, trace upstream to find the root cause: - # External reference table (fully-qualified) - products = spark.read.table("shared_catalog.reference.products") - - return orders.join(products, "product_id") -``` +1. **Start from the problematic table** - identify what's wrong (empty, wrong counts, bad data) +2. **Check its source table** - use `get_table_stats_and_schema` on the upstream table +3. **Trace back to bronze** - continue until you find where the issue originates +4. **Common causes:** + - Bronze empty → source files missing or path incorrect + - Silver empty → filter too aggressive or join condition wrong + - Gold wrong counts → aggregation logic error or duplicate keys + - Data mismatch → type casting issues or NULL handling -#### Choosing the Right Approach +5. **Fix the SQL/Python code**, re-upload, and re-run the pipeline -| Scenario | Recommended Approach | -|----------|---------------------| -| Reading tables created in same pipeline | **Unqualified names** - portable, uses target catalog/schema | -| Reading from external source that varies by environment | **Pipeline parameters** - configurable per deployment | -| Reading from shared/reference tables with fixed location | **Fully-qualified names** - explicit and clear | -| Mixed pipeline (some internal, some external) | **Combine approaches** - unqualified for internal, parameters for external | +**Do NOT use `execute_sql` with COUNT queries for validation** - `get_table_stats_and_schema` is faster and returns more information in a single call. --- @@ -520,23 +332,23 @@ def enriched_orders(): | Issue | Solution | |-------|----------| -| **Empty output tables** | Use `get_table_details` to verify, check upstream sources | +| **Empty output tables** | Use `get_table_stats_and_schema` to check upstream sources. Verify source files exist and paths are correct. | | **Pipeline stuck INITIALIZING** | Normal for serverless, wait a few minutes | | **"Column not found"** | Check `schemaHints` match actual data | | **Streaming reads fail** | For file ingestion in a streaming table, you must use the `STREAM` keyword with `read_files`: `FROM STREAM read_files(...)`. For table streams use `FROM stream(table)`. See [read_files — Usage in streaming tables](https://docs.databricks.com/aws/en/sql/language-manual/functions/read_files#usage-in-streaming-tables). | -| **Timeout during run** | Increase `timeout`, or use `wait_for_completion=False` and poll with `get_update` | +| **Timeout during run** | Increase `timeout`, or use `wait_for_completion=False` and check status with `manage_pipeline(action="get")` | | **MV doesn't refresh** | Enable row tracking on source tables | -| **SCD2: query column not found** | Lakeflow uses `__START_AT` and `__END_AT` (double underscore), not `START_AT`/`END_AT`. Use `WHERE __END_AT IS NULL` for current rows. See [3-scd-patterns.md](3-scd-patterns.md). | -| **AUTO CDC parse error at APPLY/SEQUENCE** | Put `APPLY AS DELETE WHEN` **before** `SEQUENCE BY`. Only list columns in `COLUMNS * EXCEPT (...)` that exist in the source (omit `_rescued_data` unless bronze uses rescue data). Omit `TRACK HISTORY ON *` if it causes "end of input" errors; default is equivalent. See [2-streaming-patterns.md](2-streaming-patterns.md). | -| **"Cannot create streaming table from batch query"** | In a streaming table query, use `FROM STREAM read_files(...)` so `read_files` leverages Auto Loader; `FROM read_files(...)` alone is batch. See [1-ingestion-patterns.md](1-ingestion-patterns.md) and [read_files — Usage in streaming tables](https://docs.databricks.com/aws/en/sql/language-manual/functions/read_files#usage-in-streaming-tables). | +| **SCD2: query column not found** | Lakeflow uses `__START_AT` and `__END_AT` (double underscore), not `START_AT`/`END_AT`. Use `WHERE __END_AT IS NULL` for current rows. See [sql/4-cdc-patterns.md](references/sql/4-cdc-patterns.md). | +| **AUTO CDC parse error at APPLY/SEQUENCE** | Put `APPLY AS DELETE WHEN` **before** `SEQUENCE BY`. Only list columns in `COLUMNS * EXCEPT (...)` that exist in the source (omit `_rescued_data` unless bronze uses rescue data). Omit `TRACK HISTORY ON *` if it causes "end of input" errors; default is equivalent. See [sql/4-cdc-patterns.md](references/sql/4-cdc-patterns.md). | +| **"Cannot create streaming table from batch query"** | In a streaming table query, use `FROM STREAM read_files(...)` so `read_files` leverages Auto Loader; `FROM read_files(...)` alone is batch. See [sql/2-ingestion.md](references/sql/2-ingestion.md) and [read_files — Usage in streaming tables](https://docs.databricks.com/aws/en/sql/language-manual/functions/read_files#usage-in-streaming-tables). | -**For detailed errors**, the `result["message"]` from `create_or_update_pipeline` includes suggested next steps. Use `get_pipeline_events(pipeline_id=...)` for full stack traces. +**For detailed errors**, the `result["message"]` from `manage_pipeline(action="create_or_update")` includes suggested next steps. Use `manage_pipeline(action="get", pipeline_id=...)` which includes recent events and error details. --- ## Advanced Pipeline Configuration -For advanced configuration options (development mode, continuous pipelines, custom clusters, notifications, Python dependencies, etc.), see **[7-advanced-configuration.md](7-advanced-configuration.md)**. +For advanced configuration options (development mode, continuous pipelines, custom clusters, notifications, Python dependencies, etc.), see **[3-advanced-configuration.md](references/3-advanced-configuration.md)**. --- @@ -572,6 +384,6 @@ For advanced configuration options (development mode, continuous pipelines, cust ## Related Skills - **[databricks-jobs](../databricks-jobs/SKILL.md)** - for orchestrating and scheduling pipeline runs -- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - for multi-environment deployment of pipeline projects -- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - for generating test data to feed into pipelines +- **[databricks-bundles](../databricks-bundles/SKILL.md)** - for multi-environment deployment of pipeline projects +- **[databricks-synthetic-data-gen](../databricks-synthetic-data-gen/SKILL.md)** - for generating test data to feed into pipelines - **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - for catalog/schema/volume management and governance diff --git a/.claude/skills/databricks-spark-declarative-pipelines/8-project-initialization.md b/.claude/skills/databricks-spark-declarative-pipelines/references/1-project-initialization.md similarity index 59% rename from .claude/skills/databricks-spark-declarative-pipelines/8-project-initialization.md rename to .claude/skills/databricks-spark-declarative-pipelines/references/1-project-initialization.md index 0f272db..fbab69b 100644 --- a/.claude/skills/databricks-spark-declarative-pipelines/8-project-initialization.md +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/1-project-initialization.md @@ -1,19 +1,16 @@ -# Project Initialization with databricks pipelines init +# Project Initialization -## Overview +Two approaches for creating SDP pipelines with Declarative Automation Bundles (DABs): +- **Option A**: Standalone new project using `databricks pipelines init` +- **Option B**: Adding a pipeline to an existing bundle -The `databricks pipelines init` command scaffolds a complete Databricks Asset Bundle project for Lakeflow Spark Declarative Pipelines, providing a production-ready structure with multi-environment support, pipeline configuration, and sample transformation files. +--- -**Benefits of Asset Bundles:** -- Multi-environment deployments (dev/staging/prod) -- Infrastructure as code with `databricks.yml` -- Built-in CI/CD integration -- Version control for pipeline configuration -- Automated deployment workflows +## Option A: Standalone New Pipeline Project ---- +Use `databricks pipelines init` to scaffold a complete DAB project with multi-environment support, pipeline configuration, and sample transformation files. -## Command Reference +### Command Reference ### Interactive Mode @@ -241,185 +238,84 @@ databricks pipelines start-update --pipeline-id --- -## Language Detection (for Claude) - -When a user requests a new Lakeflow pipeline, Claude should detect the appropriate language from keywords in the prompt. - -### CRITICAL: Explicit Language Requests - -**If the user explicitly mentions a language, use it without asking:** - -| User Says | Action | -|-----------|--------| -| "Python pipeline", "Python SDP", "use Python" | **Use Python immediately** | -| "SQL pipeline", "SQL files", "use SQL" | **Use SQL immediately** | -| "Python Spark Declarative Pipeline" | **Use Python immediately** | +## Medallion Architecture -**DO NOT ask for clarification when the user explicitly states a language.** This is the most common mistake - ignoring an explicit language request. +For bronze/silver/gold organization, two file structure approaches work with Declarative Automation Bundles (DABs): -### SQL Indicators (Default Choice When Ambiguous) +### Option 1: Flat Structure with Prefixes (Recommended) -**Keywords:** -- "sql files", ".sql" -- "simple", "basic", "straightforward" -- "aggregations", "joins", "transformations" -- "materialized view", "CREATE OR REFRESH" -- "SELECT", "GROUP BY", "WHERE" +``` +transformations/ +├── bronze_orders.sql +├── bronze_events.sql +├── silver_orders.sql +├── silver_events.sql +├── gold_daily_metrics.sql +└── gold_summary.sql +``` -**Context:** -- User mentions only data transformations without complex logic -- Request focuses on filtering, joining, aggregating data -- No mention of custom functions or external integrations -- **No explicit mention of "Python"** +### Option 2: Subdirectories by Layer -**Default Behavior**: Prefer SQL only when ambiguous AND no Python indicators present +``` +transformations/ +├── bronze/ +│ └── orders.sql +├── silver/ +│ └── orders.sql +└── gold/ + └── daily_metrics.sql +``` -### Python Indicators +Both work with `transformations/**` glob pattern. Choose based on team preference. -**Keywords:** -- "Python", "python files", ".py", "@dp.table" -- "UDF", "user-defined function", "custom function" -- "complex logic", "complex transformations" -- "ML", "machine learning", "inference", "model" -- "API", "external API", "REST", "HTTP" -- "pandas", "numpy", "pyspark" -- "decorator", "pyspark.pipelines" +For syntax examples, see: +- **[sql/1-syntax-basics.md](sql/1-syntax-basics.md)** - SQL table definitions +- **[python/1-syntax-basics.md](python/1-syntax-basics.md)** - Python decorators +- **[sql/2-ingestion.md](sql/2-ingestion.md)** - Bronze layer ingestion patterns -**Context:** -- User needs custom data processing beyond SQL capabilities -- Request mentions integrating with external services -- Task requires ML model inference or scoring -- Dynamic schema or path generation needed +--- -### Ambiguous Cases (Ask User) +## Option B: Adding a Pipeline to an Existing Bundle -**Only ask when ALL conditions are met:** -- User did NOT explicitly mention "Python" or "SQL" -- Mixed signals present (some SQL keywords, some Python keywords) -- OR no clear indicators either way +If you already have a `databricks.yml` for a larger project (e.g., an app with jobs, dashboards, etc.) and want to add a pipeline: -**Response:** -``` -I can create this pipeline using either SQL or Python: +### Step 1: Create Pipeline Resource File -- **SQL**: Best for transformations, aggregations, joins (simpler, faster to develop) -- **Python**: Best for custom logic, UDFs, ML inference, external APIs +Create `resources/my_pipeline.pipeline.yml`: -Which would you prefer? +```yaml +resources: + pipelines: + my_pipeline: + name: my_pipeline + catalog: ${var.catalog} + schema: ${var.schema} + serverless: true + libraries: + - file: + path: ../src/pipelines/my_pipeline/ ``` ---- - -## Medallion Architecture - -For bronze/silver/gold organization, Asset Bundles support two approaches. Both work with the `transformations/**` glob pattern in pipeline configuration. +### Step 2: Add Pipeline Source Files -### Option 1: Flat Structure with Naming (Template Default, SQL Example) +Create your pipeline transformation files: ``` -transformations/ -├── bronze_raw_orders.sql # Raw data ingestion -├── bronze_raw_events.sql -├── bronze_raw_customers.sql -├── silver_cleaned_orders.sql # Cleaned and validated -├── silver_joined_data.sql -├── silver_customer_profiles.sql -├── gold_daily_metrics.sql # Business aggregations -├── gold_customer_summary.sql -└── gold_revenue_analysis.sql +src/pipelines/my_pipeline/ +├── bronze_ingest.sql +├── silver_clean.sql +└── gold_summary.sql ``` -**Advantages:** -- Matches the official `databricks pipelines init` template structure -- All files visible at one level -- Simple file listing and discovery -- Clear naming provides logical organization - -### Option 2: Subdirectories by Layer, SQL Example +### Step 3: Deploy -``` -transformations/ -├── bronze/ -│ ├── raw_orders.sql -│ ├── raw_events.sql -│ └── raw_customers.sql -├── silver/ -│ ├── cleaned_orders.sql -│ ├── joined_data.sql -│ └── customer_profiles.sql -└── gold/ - ├── daily_metrics.sql - ├── customer_summary.sql - └── revenue_analysis.sql -``` - -**Advantages:** -- Physical separation of layers -- Familiar structure for teams using manual workflow -- Easier to navigate large projects with many files -- Works with `transformations/**` glob pattern - -**Both approaches are technically valid** - the `**` in the glob pattern matches files recursively. Choose based on team preference and project size. - -### Example Bronze Layer (SQL) - -```sql --- File: bronze_raw_orders.sql -CREATE OR REFRESH STREAMING TABLE bronze_raw_orders -CLUSTER BY (order_date) -COMMENT "Raw order data ingested from cloud storage" -AS -SELECT - *, - current_timestamp() AS _ingested_at, - _metadata.file_path AS _source_file -FROM read_files( - '/Volumes/main/raw_data/orders/', - format => 'json', - schemaHints => 'order_id STRING, customer_id STRING, amount DECIMAL(10,2), order_date DATE' -); -``` - -### Example Silver Layer (SQL) - -```sql --- File: silver_cleaned_orders.sql -CREATE OR REFRESH MATERIALIZED VIEW silver_cleaned_orders -CLUSTER BY (order_date) -COMMENT "Cleaned and validated orders with customer enrichment" -AS -SELECT - o.order_id, - o.customer_id, - o.amount, - o.order_date, - c.customer_name, - c.customer_segment -FROM LIVE.bronze_raw_orders o -INNER JOIN LIVE.bronze_raw_customers c - ON o.customer_id = c.customer_id -WHERE o.amount > 0 -- Remove invalid orders - AND o.order_date >= '2020-01-01'; -``` - -### Example Gold Layer (SQL) - -```sql --- File: gold_daily_metrics.sql -CREATE OR REFRESH MATERIALIZED VIEW gold_daily_metrics -CLUSTER BY (metric_date) -COMMENT "Daily business metrics for reporting" -AS -SELECT - order_date AS metric_date, - COUNT(DISTINCT customer_id) AS unique_customers, - COUNT(*) AS total_orders, - SUM(amount) AS total_revenue, - AVG(amount) AS avg_order_value -FROM LIVE.silver_cleaned_orders -GROUP BY order_date; +```bash +databricks bundle deploy +databricks bundle run my_pipeline ``` +That's it - the pipeline is now part of your existing bundle and shares the same targets/variables. + --- ## Migration from Manual Structure @@ -609,7 +505,7 @@ For advanced pipeline configuration options beyond the bundle initialization: - **Custom notifications**: Email or webhook alerts - **Non-serverless clusters**: When serverless limitations apply -See [7-advanced-configuration.md](7-advanced-configuration.md) for detailed examples. +See [3-advanced-configuration.md](3-advanced-configuration.md) for detailed examples. --- @@ -668,45 +564,22 @@ resources: ## Best Practices -### Project Organization - -1. **Use descriptive file names**: `bronze_orders_raw.sql` not just `orders.sql` -2. **Choose structure approach**: - - **Flat with prefixes**: `bronze_*`, `silver_*`, `gold_*` (template default) - - **Subdirectories**: `bronze/`, `silver/`, `gold/` folders (also valid) - - Both work with `transformations/**` glob pattern -3. **One table per file**: Each file defines a single table or view -4. **Be consistent**: Pick one approach and use it throughout the project - -### Configuration Management - -1. **Use variables**: Parameterize catalog and schema names -2. **Separate environments**: Define dev/staging/prod targets -3. **Version control**: Track `databricks.yml` and pipeline configs in git -4. **Sensitive data**: Use secrets, not hardcoded values - -### Development Workflow - -1. **Start with dev**: Always test in development environment first -2. **Validate locally**: Run `databricks bundle validate` before deploy -3. **Incremental changes**: Deploy and test small changes frequently -4. **Use explorations**: Ad-hoc notebooks for data exploration - -### Deployment Strategy +1. **One table per file** - Each `.sql` or `.py` file defines a single table/view +2. **Use variables** - Parameterize catalog and schema names for environment portability +3. **Sensitive data** - Use secrets (`{{secrets/scope/key}}`), not hardcoded values +4. **Test in dev first** - Run `databricks bundle validate` before deploy +5. **Version control** - Track `databricks.yml` and pipeline configs in git -1. **CI/CD integration**: Automate deployments with GitHub Actions, GitLab CI -2. **Approval gates**: Require approval for production deployments -3. **Rollback plan**: Keep previous bundle versions for quick rollback -4. **Monitor pipelines**: Set up notifications for failures +For technical best practices (Liquid Clustering, serverless, etc.), see **[SKILL.md](SKILL.md#best-practices-2026)**. --- ## References -- **[SKILL.md](SKILL.md)** - Main development workflow and MCP tools -- **[Databricks Asset Bundles Documentation](https://docs.databricks.com/dev-tools/bundles/)** - Official bundle reference +- **[SKILL.md](../SKILL.md)** - Main development workflow and MCP tools +- **[Declarative Automation Bundles (DABs) Documentation](https://docs.databricks.com/dev-tools/bundles/)** - Official bundle reference - **[Pipeline Configuration Reference](https://docs.databricks.com/aws/en/ldp/configure-pipeline)** - Pipeline settings - **[Databricks CLI Reference](https://docs.databricks.com/dev-tools/cli/)** - CLI commands and options -- **[1-ingestion-patterns.md](1-ingestion-patterns.md)** - Data ingestion patterns -- **[2-streaming-patterns.md](2-streaming-patterns.md)** - Streaming transformations -- **[7-advanced-configuration.md](7-advanced-configuration.md)** - Advanced pipeline settings +- **[sql/2-ingestion.md](sql/2-ingestion.md)** or **[python/2-ingestion.md](python/2-ingestion.md)** - Data ingestion patterns +- **[sql/3-streaming-patterns.md](sql/3-streaming-patterns.md)** or **[python/3-streaming-patterns.md](python/3-streaming-patterns.md)** - Streaming transformations +- **[3-advanced-configuration.md](3-advanced-configuration.md)** - Advanced pipeline settings diff --git a/.claude/skills/databricks-spark-declarative-pipelines/references/2-mcp-approach.md b/.claude/skills/databricks-spark-declarative-pipelines/references/2-mcp-approach.md new file mode 100644 index 0000000..87e0ed7 --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/2-mcp-approach.md @@ -0,0 +1,163 @@ +Use MCP tools to create, run, and iterate on **SDP pipelines**. The **primary tool is `manage_pipeline`** which handles the entire lifecycle. + +**IMPORTANT: Default to serverless pipelines.** Only use classic clusters if user explicitly requires R language, Spark RDD APIs, or JAR libraries. + +### Step 1: Write Pipeline Files Locally + +Create `.sql` or `.py` files in a local folder. For syntax examples, see: +- [sql/1-syntax-basics.md](sql/1-syntax-basics.md) for SQL syntax +- [python/1-syntax-basics.md](python/1-syntax-basics.md) for Python syntax + +### Step 2: Upload to Databricks Workspace + +``` +# MCP Tool: manage_workspace_files +manage_workspace_files( + action="upload", + local_path="/path/to/my_pipeline", + workspace_path="/Workspace/Users/user@example.com/my_pipeline" +) +``` + +### Step 3: Create/Update and Run Pipeline + +Use **`manage_pipeline`** with `action="create_or_update"` to manage the resource: + +``` +# MCP Tool: manage_pipeline +manage_pipeline( + action="create_or_update", + name="my_orders_pipeline", + root_path="/Workspace/Users/user@example.com/my_pipeline", + catalog="my_catalog", + schema="my_schema", + workspace_file_paths=[ + "/Workspace/Users/user@example.com/my_pipeline/bronze/ingest_orders.sql", + "/Workspace/Users/user@example.com/my_pipeline/silver/clean_orders.sql", + "/Workspace/Users/user@example.com/my_pipeline/gold/daily_summary.sql" + ], + start_run=True, # Automatically run after create/update + wait_for_completion=True, # Wait for run to finish + full_refresh=True # Reprocess all data +) +``` + +**Result contains actionable information:** +```json +{ + "success": true, + "pipeline_id": "abc-123", + "pipeline_name": "my_orders_pipeline", + "created": true, + "state": "COMPLETED", + "catalog": "my_catalog", + "schema": "my_schema", + "duration_seconds": 45.2, + "message": "Pipeline created and completed successfully in 45.2s. Tables written to my_catalog.my_schema", + "error_message": null, + "errors": [] +} +``` + +### Alternative: Run Pipeline Separately + +If you want to run an existing pipeline or control the run separately: + +``` +# MCP Tool: manage_pipeline_run +manage_pipeline_run( + action="start", + pipeline_id="", + full_refresh=True, + wait=True, # Wait for completion + timeout=1800 # 30 minute timeout +) +``` + +### Step 4: Validate Results + +**On Success** - Use `get_table_stats_and_schema` to verify tables (NOT manual SQL COUNT queries): +``` +# MCP Tool: get_table_stats_and_schema +get_table_stats_and_schema( + catalog="my_catalog", + schema="my_schema", + table_names=["bronze_orders", "silver_orders", "gold_daily_summary"] +) +# Returns schema, row counts, and column stats for all tables in one call +``` + +**On Failure** - Check `run_result["message"]` for suggested next steps, then get detailed errors: +``` +# MCP Tool: manage_pipeline +manage_pipeline(action="get", pipeline_id="") +# Returns pipeline details enriched with recent events and error messages + +# Or get events/logs directly: +# MCP Tool: manage_pipeline_run +manage_pipeline_run( + action="get_events", + pipeline_id="", + event_log_level="ERROR", # ERROR, WARN, or INFO + max_results=10 +) +``` + +### Step 5: Iterate Until Working + +1. Review errors from run result or `manage_pipeline(action="get")` +2. Fix issues in local files +3. Re-upload with `manage_workspace_files(action="upload")` +4. Run `manage_pipeline(action="create_or_update", start_run=True)` again (it will update, not recreate) +5. Repeat until `result["success"] == True` + +--- + +## Quick Reference: MCP Tools + +### manage_pipeline - Pipeline Lifecycle + +| Action | Description | Required Params | +|--------|-------------|-----------------| +| `create` | Create new pipeline | name, root_path, catalog, schema, workspace_file_paths | +| `create_or_update` | **Main entry point.** Idempotent create/update, optionally run | name, root_path, catalog, schema, workspace_file_paths | +| `get` | Get pipeline details by ID | pipeline_id | +| `update` | Update pipeline config | pipeline_id + fields to change | +| `delete` | Delete a pipeline | pipeline_id | +| `find_by_name` | Find pipeline by name | name | + +**create_or_update options:** +- `start_run=True`: Automatically run after create/update +- `wait_for_completion=True`: Block until run finishes +- `full_refresh=True`: Reprocess all data (default) +- `timeout=1800`: Max wait time in seconds + +### manage_pipeline_run - Run Management + +| Action | Description | Required Params | +|--------|-------------|-----------------| +| `start` | Start pipeline update | pipeline_id | +| `get` | Get run status | pipeline_id, update_id | +| `stop` | Stop running pipeline | pipeline_id | +| `get_events` | Get events/logs for debugging | pipeline_id | + +**start options:** +- `wait=True`: Block until complete (default) +- `full_refresh=True`: Reprocess all data +- `validate_only=True`: Dry run without writing data +- `refresh_selection=["table1", "table2"]`: Refresh specific tables only + +**get_events options:** +- `event_log_level`: "ERROR", "WARN" (default), "INFO" +- `max_results`: Number of events (default 5) +- `update_id`: Filter to specific run + +### Supporting Tools + +| Tool | Description | +|------|-------------| +| `manage_workspace_files(action="upload")` | Upload files/folders to workspace | +| `get_table_stats_and_schema` | **Use this to validate tables** - returns schema, row counts, and stats in one call | +| `execute_sql` | Run ad-hoc SQL to inspect actual data content (not for row counts) | + +--- diff --git a/.claude/skills/databricks-spark-declarative-pipelines/7-advanced-configuration.md b/.claude/skills/databricks-spark-declarative-pipelines/references/3-advanced-configuration.md similarity index 76% rename from .claude/skills/databricks-spark-declarative-pipelines/7-advanced-configuration.md rename to .claude/skills/databricks-spark-declarative-pipelines/references/3-advanced-configuration.md index a6c8ecf..b637f46 100644 --- a/.claude/skills/databricks-spark-declarative-pipelines/7-advanced-configuration.md +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/3-advanced-configuration.md @@ -142,7 +142,7 @@ Install pip dependencies for serverless pipelines: | Field | Type | Description | |-------|------|-------------| -| `kind` | str | `"BUNDLE"` (Databricks Asset Bundles) or `"DEFAULT"` | +| `kind` | str | `"BUNDLE"` (DABs) or `"DEFAULT"` | | `metadata_file_path` | str | Path to deployment metadata file | ### Edition Comparison @@ -159,7 +159,7 @@ Install pip dependencies for serverless pipelines: ### Development Mode Pipeline -Use `create_or_update_pipeline` tool with: +Use `manage_pipeline(action="create_or_update")` tool with: - `name`: "my_dev_pipeline" - `root_path`: "/Workspace/Users/user@example.com/my_pipeline" - `catalog`: "dev_catalog" @@ -176,7 +176,7 @@ Use `create_or_update_pipeline` tool with: ### Non-Serverless with Dedicated Cluster -Use `create_or_update_pipeline` tool with `extra_settings`: +Use `manage_pipeline(action="create_or_update")` tool with `extra_settings`: ```json { "serverless": false, @@ -193,7 +193,7 @@ Use `create_or_update_pipeline` tool with `extra_settings`: ### Continuous Streaming Pipeline -Use `create_or_update_pipeline` tool with `extra_settings`: +Use `manage_pipeline(action="create_or_update")` tool with `extra_settings`: ```json { "continuous": true, @@ -205,7 +205,7 @@ Use `create_or_update_pipeline` tool with `extra_settings`: ### Using Instance Pool -Use `create_or_update_pipeline` tool with `extra_settings`: +Use `manage_pipeline(action="create_or_update")` tool with `extra_settings`: ```json { "serverless": false, @@ -220,7 +220,7 @@ Use `create_or_update_pipeline` tool with `extra_settings`: ### Custom Event Log Location -Use `create_or_update_pipeline` tool with `extra_settings`: +Use `manage_pipeline(action="create_or_update")` tool with `extra_settings`: ```json { "event_log": { @@ -233,7 +233,7 @@ Use `create_or_update_pipeline` tool with `extra_settings`: ### Pipeline with Email Notifications -Use `create_or_update_pipeline` tool with `extra_settings`: +Use `manage_pipeline(action="create_or_update")` tool with `extra_settings`: ```json { "notifications": [{ @@ -245,7 +245,7 @@ Use `create_or_update_pipeline` tool with `extra_settings`: ### Production Pipeline with Autoscaling -Use `create_or_update_pipeline` tool with `extra_settings`: +Use `manage_pipeline(action="create_or_update")` tool with `extra_settings`: ```json { "serverless": false, @@ -274,7 +274,7 @@ Use `create_or_update_pipeline` tool with `extra_settings`: ### Run as Service Principal -Use `create_or_update_pipeline` tool with `extra_settings`: +Use `manage_pipeline(action="create_or_update")` tool with `extra_settings`: ```json { "run_as": { @@ -285,7 +285,7 @@ Use `create_or_update_pipeline` tool with `extra_settings`: ### Continuous Pipeline with Restart Window -Use `create_or_update_pipeline` tool with `extra_settings`: +Use `manage_pipeline(action="create_or_update")` tool with `extra_settings`: ```json { "continuous": true, @@ -299,7 +299,7 @@ Use `create_or_update_pipeline` tool with `extra_settings`: ### Serverless with Python Dependencies -Use `create_or_update_pipeline` tool with `extra_settings`: +Use `manage_pipeline(action="create_or_update")` tool with `extra_settings`: ```json { "serverless": true, @@ -348,3 +348,77 @@ You can copy pipeline settings from the Databricks UI (Pipeline Settings > JSON) ``` **Note**: Explicit tool parameters (`name`, `root_path`, `catalog`, `schema`, `workspace_file_paths`) always take precedence over values in `extra_settings`. + +--- + +## Multi-Schema Patterns + +**Recommended: One pipeline writing to multiple schemas** using fully qualified table names. This is simpler than creating multiple pipelines and keeps all dependencies in one place. + +For simple cases where all tables go to the same schema, use name prefixes (`bronze_*`, `silver_*`, `gold_*`). + +### Option 1: Same Catalog, Separate Schemas + +Set pipeline defaults to bronze, use parameters for silver/gold: + +```python +from pyspark import pipelines as dp +from pyspark.sql.functions import col + +# Pull variables from pipeline configuration +silver_schema = spark.conf.get("silver_schema") # e.g., "silver" +gold_schema = spark.conf.get("gold_schema") # e.g., "gold" +landing_schema = spark.conf.get("landing_schema") # e.g., "landing" + +# Bronze → uses default catalog/schema (set to bronze in pipeline settings) +@dp.table(name="orders_bronze") +def orders_bronze(): + return spark.readStream.table(f"{landing_schema}.orders_raw") + +# Silver → same catalog, schema from parameter +@dp.table(name=f"{silver_schema}.orders_clean") +def orders_clean(): + return spark.read.table("orders_bronze").filter(col("order_id").isNotNull()) + +# Gold → same catalog, schema from parameter +@dp.materialized_view(name=f"{gold_schema}.orders_by_date") +def orders_by_date(): + return (spark.read.table(f"{silver_schema}.orders_clean") + .groupBy("order_date").count()) +``` + +### Option 2: Custom Catalog/Schema Per Layer + +For cross-catalog scenarios: + +```python +from pyspark import pipelines as dp +from pyspark.sql.functions import col + +# Pull variables from pipeline configuration +silver_catalog = spark.conf.get("silver_catalog") +silver_schema = spark.conf.get("silver_schema") +gold_catalog = spark.conf.get("gold_catalog") +gold_schema = spark.conf.get("gold_schema") + +# Bronze → uses pipeline defaults +@dp.table(name="orders_bronze") +def orders_bronze(): + return spark.readStream.format("cloudFiles").load("/Volumes/...") + +# Silver → custom catalog + schema +@dp.table(name=f"{silver_catalog}.{silver_schema}.orders_clean") +def orders_clean(): + return spark.read.table("orders_bronze").filter(col("order_id").isNotNull()) + +# Gold → custom catalog + schema +@dp.materialized_view(name=f"{gold_catalog}.{gold_schema}.orders_by_date") +def orders_by_date(): + return (spark.read.table(f"{silver_catalog}.{silver_schema}.orders_clean") + .groupBy("order_date").count()) +``` + +**Key points:** +- Multipart names in `@dp.table(name=...)` let you publish to explicit catalog.schema targets +- Unqualified names use pipeline defaults +- Use fully-qualified names when crossing catalogs diff --git a/.claude/skills/databricks-spark-declarative-pipelines/references/4-dlt-migration.md b/.claude/skills/databricks-spark-declarative-pipelines/references/4-dlt-migration.md new file mode 100644 index 0000000..dbde0d9 --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/4-dlt-migration.md @@ -0,0 +1,447 @@ +# Migration Guide: DLT to SDP + +Guide for migrating from Delta Live Tables (DLT) to Spark Declarative Pipelines (SDP). + +**Two migration paths:** +1. **DLT Python → SDP Python** (dlt → dp): Same language, new API +2. **DLT Python → SDP SQL**: Change language for simpler pipelines + +--- + +## Migration Path 1: DLT Python → SDP Python (dlt → dp) + +Use this when staying with Python but moving to the modern `pyspark.pipelines` API. + +### Quick Reference + +| Aspect | Legacy (`dlt`) | Modern (`dp`) | +|--------|---------------|----------------| +| **Import** | `import dlt` | `from pyspark import pipelines as dp` | +| **Table decorator** | `@dlt.table()` | `@dp.table()` | +| **Read table** | `dlt.read("table")` | `spark.read.table("table")` | +| **Read stream** | `dlt.read_stream("table")` | `spark.readStream.table("table")` | +| **CDC/SCD** | `dlt.apply_changes()` | `dp.create_auto_cdc_flow()` | +| **Clustering** | `partition_cols=["date"]` | `cluster_by=["date", "col2"]` | + +### Step-by-Step Migration + +#### Step 1: Update Imports + +```python +# Before +import dlt + +# After +from pyspark import pipelines as dp +``` + +#### Step 2: Update Decorators + +```python +# Before +@dlt.table(name="my_table") + +# After +@dp.table(name="my_table") +``` + +#### Step 3: Update Table Reads + +```python +# Before +@dlt.table(name="silver_events") +def silver_events(): + return dlt.read("bronze_events").filter(...) + +# After +@dp.table(name="silver_events") +def silver_events(): + return spark.read.table("bronze_events").filter(...) +``` + +```python +# Before (streaming) +@dlt.table(name="silver_events") +def silver_events(): + return dlt.read_stream("bronze_events").filter(...) + +# After (streaming) +@dp.table(name="silver_events") +def silver_events(): + return spark.readStream.table("bronze_events").filter(...) +``` + +#### Step 4: Update Expectations + +```python +# Before +@dlt.table(name="silver") +@dlt.expect_or_drop("valid_id", "id IS NOT NULL") + +# After (identical syntax, just change dlt → dp) +@dp.table(name="silver") +@dp.expect_or_drop("valid_id", "id IS NOT NULL") +``` + +#### Step 5: Update CDC/SCD Operations + +```python +# Before +dlt.create_streaming_table("customers_history") +dlt.apply_changes( + target="customers_history", + source="customers_cdc", + keys=["customer_id"], + sequence_by="event_timestamp", + stored_as_scd_type="2" +) + +# After +from pyspark.sql.functions import col + +dp.create_streaming_table("customers_history") +dp.create_auto_cdc_flow( + target="customers_history", + source="customers_cdc", + keys=["customer_id"], + sequence_by=col("event_timestamp"), # Note: use col() + stored_as_scd_type=2 # Note: integer, not string +) +``` + +**Key differences:** +- `apply_changes()` → `create_auto_cdc_flow()` +- `sequence_by` takes a Column object (`col("...")`) not a string +- `stored_as_scd_type` is integer `2` for Type 2, string `"1"` for Type 1 + +#### Step 6: Update Clustering (Partitioning → Liquid Clustering) + +```python +# Before (legacy partitioning) +@dlt.table( + name="bronze_events", + partition_cols=["event_date"], + table_properties={"pipelines.autoOptimize.zOrderCols": "event_type"} +) + +# After (Liquid Clustering) +@dp.table( + name="bronze_events", + cluster_by=["event_date", "event_type"] +) +``` + +### Complete Before/After Example + +**Before (DLT):** +```python +import dlt +from pyspark.sql import functions as F + +@dlt.table(name="bronze_orders", partition_cols=["order_date"]) +def bronze_orders(): + return spark.readStream.format("cloudFiles").load("/data/orders") + +@dlt.table(name="silver_orders") +@dlt.expect_or_drop("valid_amount", "amount > 0") +def silver_orders(): + return dlt.read_stream("bronze_orders").filter(F.col("status") == "completed") + +dlt.create_streaming_table("dim_customers") +dlt.apply_changes( + target="dim_customers", + source="customers_cdc", + keys=["customer_id"], + sequence_by="updated_at", + stored_as_scd_type="2" +) +``` + +**After (SDP):** +```python +from pyspark import pipelines as dp +from pyspark.sql import functions as F + +@dp.table(name="bronze_orders", cluster_by=["order_date"]) +def bronze_orders(): + return spark.readStream.format("cloudFiles").load("/data/orders") + +@dp.table(name="silver_orders") +@dp.expect_or_drop("valid_amount", "amount > 0") +def silver_orders(): + return spark.readStream.table("bronze_orders").filter(F.col("status") == "completed") + +dp.create_streaming_table("dim_customers") +dp.create_auto_cdc_flow( + target="dim_customers", + source="customers_cdc", + keys=["customer_id"], + sequence_by=F.col("updated_at"), + stored_as_scd_type=2 +) +``` + +--- + +## Migration Path 2: DLT Python → SDP SQL + +Use this when simplifying pipelines by converting to SQL. + +### Decision Matrix + +| Feature/Pattern | DLT Python | SDP SQL | Recommendation | +|-----------------|------------|---------|----------------| +| Simple transformations | ✓ | ✓ | **Migrate to SQL** | +| Aggregations | ✓ | ✓ | **Migrate to SQL** | +| Filtering, WHERE clauses | ✓ | ✓ | **Migrate to SQL** | +| CASE expressions | ✓ | ✓ | **Migrate to SQL** | +| SCD Type 1/2 | ✓ | ✓ | **Migrate to SQL** (AUTO CDC) | +| Simple joins | ✓ | ✓ | **Migrate to SQL** | +| Auto Loader | ✓ | ✓ | **Migrate to SQL** (read_files) | +| Streaming sources (Kafka) | ✓ | ✓ | **Migrate to SQL** (read_kafka) | +| Complex Python UDFs | ✓ | ❌ | **Stay in Python** | +| External API calls | ✓ | ❌ | **Stay in Python** | +| Custom libraries | ✓ | ❌ | **Stay in Python** | +| ML model inference | ✓ | ❌ | **Stay in Python** | + +**Rule**: If 80%+ is SQL-expressible, migrate to SDP SQL. If heavy Python logic, stay with Python (use modern `dp` API). + +### Side-by-Side Conversions + +#### Basic Streaming Table + +**DLT Python:** +```python +@dlt.table(name="bronze_sales", comment="Raw sales") +def bronze_sales(): + return ( + spark.readStream.format("cloudFiles") + .option("cloudFiles.format", "json") + .load("/Volumes/my_catalog/my_schema/raw/sales") + .withColumn("_ingested_at", F.current_timestamp()) + ) +``` + +**SDP SQL:** +```sql +CREATE OR REFRESH STREAMING TABLE bronze_sales +COMMENT 'Raw sales' +AS +SELECT *, current_timestamp() AS _ingested_at +FROM STREAM read_files('/Volumes/my_catalog/my_schema/raw/sales', format => 'json'); +``` + +#### Filtering and Transformations + +**DLT Python:** +```python +@dlt.table(name="silver_sales") +@dlt.expect_or_drop("valid_amount", "amount > 0") +@dlt.expect_or_drop("valid_sale_id", "sale_id IS NOT NULL") +def silver_sales(): + return ( + dlt.read_stream("bronze_sales") + .withColumn("sale_date", F.to_date("sale_date")) + .withColumn("amount", F.col("amount").cast("decimal(10,2)")) + .select("sale_id", "customer_id", "amount", "sale_date") + ) +``` + +**SDP SQL:** +```sql +CREATE OR REFRESH STREAMING TABLE silver_sales AS +SELECT + sale_id, customer_id, + CAST(amount AS DECIMAL(10,2)) AS amount, + CAST(sale_date AS DATE) AS sale_date +FROM STREAM bronze_sales +WHERE amount > 0 AND sale_id IS NOT NULL; +``` + +#### SCD Type 2 + +**DLT Python:** +```python +dlt.create_streaming_table("customers_history") + +dlt.apply_changes( + target="customers_history", + source="customers_cdc_clean", + keys=["customer_id"], + sequence_by="event_timestamp", + stored_as_scd_type="2", + track_history_column_list=["*"] +) +``` + +**SDP SQL:** +```sql +CREATE OR REFRESH STREAMING TABLE customers_history; + +CREATE FLOW customers_scd2_flow AS +AUTO CDC INTO customers_history +FROM stream(customers_cdc_clean) +KEYS (customer_id) +APPLY AS DELETE WHEN operation = "DELETE" +SEQUENCE BY event_timestamp +COLUMNS * EXCEPT (operation, _ingested_at, _source_file) +STORED AS SCD TYPE 2; +``` + +**Note:** In SQL, put `APPLY AS DELETE WHEN` before `SEQUENCE BY`. Only list columns in `COLUMNS * EXCEPT (...)` that exist in the source. + +#### Joins + +**DLT Python:** +```python +@dlt.table(name="silver_sales_enriched") +def silver_sales_enriched(): + sales = dlt.read_stream("silver_sales") + products = dlt.read("dim_products") + return sales.join(products, "product_id", "left") +``` + +**SDP SQL:** +```sql +CREATE OR REFRESH STREAMING TABLE silver_sales_enriched AS +SELECT s.*, p.product_name, p.category +FROM STREAM silver_sales s +LEFT JOIN dim_products p ON s.product_id = p.product_id; +``` + +### Handling Expectations + +**DLT Python:** +```python +@dlt.expect_or_drop("valid_amount", "amount > 0") +@dlt.expect_or_fail("critical_id", "id IS NOT NULL") +``` + +**SDP SQL - Basic** (equivalent to expect_or_drop): +```sql +WHERE amount > 0 AND id IS NOT NULL +``` + +**SDP SQL - Quarantine Pattern** (for auditing dropped records): +```sql +-- Flag invalid records +CREATE OR REFRESH STREAMING TABLE bronze_data_flagged AS +SELECT *, + CASE WHEN amount <= 0 OR id IS NULL THEN TRUE ELSE FALSE END AS is_invalid +FROM STREAM bronze_data; + +-- Clean for downstream +CREATE OR REFRESH STREAMING TABLE silver_data_clean AS +SELECT * FROM STREAM bronze_data_flagged WHERE NOT is_invalid; + +-- Quarantine for investigation +CREATE OR REFRESH STREAMING TABLE silver_data_quarantine AS +SELECT * FROM STREAM bronze_data_flagged WHERE is_invalid; +``` + +### Handling UDFs + +#### Simple UDFs → SQL CASE + +**DLT Python:** +```python +@F.udf(returnType=StringType()) +def categorize_amount(amount): + if amount > 1000: return "High" + elif amount > 100: return "Medium" + else: return "Low" + +@dlt.table(name="sales_categorized") +def sales_categorized(): + return dlt.read("sales").withColumn("category", categorize_amount(F.col("amount"))) +``` + +**SDP SQL:** +```sql +CREATE OR REFRESH MATERIALIZED VIEW sales_categorized AS +SELECT *, + CASE + WHEN amount > 1000 THEN 'High' + WHEN amount > 100 THEN 'Medium' + ELSE 'Low' + END AS category +FROM sales; +``` + +#### Complex UDFs → Stay in Python + +Keep in Python if: +- Complex conditional logic +- External API calls +- Custom algorithms +- ML inference + +Use modern `dp` API instead of `dlt`. + +--- + +## Migration Process + +### Step 1: Inventory + +Document: +- Number of tables/views +- Python UDFs (simple vs complex) +- External dependencies +- Expectations and quality rules + +### Step 2: Choose Path + +- **80%+ SQL-expressible** → Migrate to SDP SQL +- **Heavy Python logic** → Migrate to SDP Python (`dp` API) +- **Mixed** → Hybrid (SQL for most, Python for complex) + +### Step 3: Migrate by Layer + +1. **Bronze** (ingestion): `cloudFiles` → `read_files()` or keep `cloudFiles` with `dp` +2. **Silver** (cleansing): `dlt.expect*` → WHERE clause or `dp.expect*` +3. **Gold** (aggregations): Usually straightforward +4. **SCD/CDC**: `apply_changes` → AUTO CDC or `create_auto_cdc_flow` + +### Step 4: Test + +- Run both pipelines in parallel +- Compare outputs for correctness +- Validate performance +- Check quality metrics + +--- + +## When NOT to Migrate + +**Stay with current approach if:** +1. Pipeline works well and team is comfortable +2. Heavy Python UDF usage (>30% of logic) +3. External API calls required +4. Custom ML model inference +5. Complex stateful operations not expressible in SQL +6. Limited time/resources for migration + +**Key**: DLT and SDP are both fully supported. Migrate for simplicity or new features, not necessity. + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| `sequence_by` type error | Use `col("column")` not string in `dp.create_auto_cdc_flow()` | +| UDF doesn't translate | Keep in Python or refactor with SQL built-ins | +| Expectations differ | Use quarantine pattern to audit dropped records | +| Performance degradation | Use `CLUSTER BY` for Liquid Clustering | +| Schema evolution different | Use `mode => 'PERMISSIVE'` in `read_files()` | +| AUTO CDC parse error | Put `APPLY AS DELETE WHEN` before `SEQUENCE BY` | + +--- + +## Related Documentation + +- **[python/1-syntax-basics.md](python/1-syntax-basics.md)** - Modern `dp` API reference +- **[python/4-cdc-patterns.md](python/4-cdc-patterns.md)** - Python CDC patterns +- **[sql/4-cdc-patterns.md](sql/4-cdc-patterns.md)** - SQL CDC patterns +- **[SKILL.md](../SKILL.md)** - Main skill entry point diff --git a/.claude/skills/databricks-spark-declarative-pipelines/references/python/1-syntax-basics.md b/.claude/skills/databricks-spark-declarative-pipelines/references/python/1-syntax-basics.md new file mode 100644 index 0000000..9d00cde --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/python/1-syntax-basics.md @@ -0,0 +1,321 @@ +# Python Syntax Basics + +Core Python syntax for Spark Declarative Pipelines (SDP) using the modern `pyspark.pipelines` API. + +**Import**: `from pyspark import pipelines as dp` + +--- + +## Decorators + +### `@dp.table()` + +Creates a streaming table or batch table. + +```python +from pyspark import pipelines as dp +from pyspark.sql import functions as F + +@dp.table( + name="bronze_events", # Table name (can be fully qualified: catalog.schema.table) + comment="Raw event data", # Optional description + cluster_by=["event_type", "date"], # Liquid Clustering columns (recommended) + table_properties={ # Delta table properties + "delta.autoOptimize.optimizeWrite": "true", + "delta.autoOptimize.autoCompact": "true" + }, + schema="col1 STRING, col2 INT", # Optional explicit schema + path="/path/to/external/location" # Optional external location +) +def bronze_events(): + return ( + spark.readStream.format("cloudFiles") + .option("cloudFiles.format", "json") + .load("/Volumes/catalog/schema/raw/events/") + .withColumn("_ingested_at", F.current_timestamp()) + .withColumn("_source_file", F.col("_metadata.file_path")) + ) +``` + +**Parameters:** +| Parameter | Type | Description | +|-----------|------|-------------| +| `name` | str | Table name. Can be unqualified (`my_table`), schema-qualified (`schema.table`), or fully qualified (`catalog.schema.table`). | +| `comment` | str | Table description | +| `cluster_by` | list | Columns for Liquid Clustering. Use `["AUTO"]` for automatic selection. | +| `table_properties` | dict | Delta table properties | +| `schema` | str/StructType | Explicit schema (optional, usually inferred) | +| `path` | str | External storage location (optional) | + +**Streaming vs Batch:** +- Return `spark.readStream...` for streaming table +- Return `spark.read...` for batch table + +### `@dp.materialized_view()` + +Creates a materialized view (batch, incrementally refreshed). + +```python +@dp.materialized_view( + name="gold_daily_summary", + comment="Daily aggregated metrics", + cluster_by=["report_date"] +) +def gold_daily_summary(): + return ( + spark.read.table("silver_orders") + .groupBy("report_date") + .agg(F.sum("amount").alias("total_amount")) + ) +``` + +**Parameters:** Same as `@dp.table()`. + +### `@dp.temporary_view()` + +Creates a pipeline-scoped temporary view (not persisted, exists only during pipeline execution). + +```python +@dp.temporary_view() +def orders_with_calculations(): + """Intermediate view for complex logic before AUTO CDC.""" + return ( + spark.readStream.table("bronze_orders") + .withColumn("total", F.col("quantity") * F.col("price")) + .filter(F.col("total") > 0) + ) +``` + +**Constraints:** +- Cannot specify `catalog` or `schema` (pipeline-scoped only) +- Cannot use `cluster_by` (not persisted) +- Useful for intermediate transformations before AUTO CDC + +--- + +## Expectation Decorators (Data Quality) + +```python +@dp.table(name="silver_validated") +@dp.expect("valid_id", "id IS NOT NULL") # Warn only, keep all rows +@dp.expect_or_drop("valid_amount", "amount > 0") # Drop invalid rows +@dp.expect_or_fail("critical_field", "timestamp IS NOT NULL") # Fail pipeline if violated +def silver_validated(): + return spark.read.table("bronze_events") +``` + +| Decorator | Behavior | +|-----------|----------| +| `@dp.expect(name, condition)` | Log warning, keep all rows | +| `@dp.expect_or_drop(name, condition)` | Drop rows that violate | +| `@dp.expect_or_fail(name, condition)` | Fail pipeline if any row violates | + +--- + +## Functions + +### `dp.create_streaming_table()` + +Creates an empty streaming table (typically used before `create_auto_cdc_flow`). + +```python +dp.create_streaming_table( + name="customers_history", + comment="SCD Type 2 customer dimension" +) +``` + +### `dp.create_auto_cdc_flow()` + +Creates a Change Data Capture flow for SCD Type 1 or Type 2. + +```python +from pyspark.sql.functions import col + +dp.create_streaming_table("dim_customers") + +dp.create_auto_cdc_flow( + target="dim_customers", + source="customers_cdc_clean", + keys=["customer_id"], + sequence_by=col("event_timestamp"), # Note: use col(), not string + stored_as_scd_type=2, # Integer for Type 2 + apply_as_deletes=col("operation") == "DELETE", # Optional + except_column_list=["operation", "_ingested_at"], # Columns to exclude + track_history_column_list=["price", "status"] # Type 2: only track these +) +``` + +**Parameters:** +| Parameter | Type | Description | +|-----------|------|-------------| +| `target` | str | Target table name | +| `source` | str | Source table/view name | +| `keys` | list | Primary key columns | +| `sequence_by` | Column | Column for ordering changes (**use `col()`**) | +| `stored_as_scd_type` | int/str | `2` for Type 2 (history), `"1"` for Type 1 (overwrite) | +| `apply_as_deletes` | Column | Condition identifying delete operations | +| `apply_as_truncates` | Column | Condition identifying truncate operations | +| `except_column_list` | list | Columns to exclude from target | +| `track_history_column_list` | list | Type 2 only: columns that trigger new versions | + +**Important:** `stored_as_scd_type` is integer `2` for Type 2, string `"1"` for Type 1. + +### `dp.create_auto_cdc_from_snapshot_flow()` + +Creates CDC from periodic snapshots (compares consecutive snapshots to detect changes). + +```python +dp.create_streaming_table("dim_products") + +dp.create_auto_cdc_from_snapshot_flow( + target="dim_products", + source="products_snapshot", + keys=["product_id"], + stored_as_scd_type=2 +) +``` + +### `dp.append_flow()` + +Appends data from a source to a target table. + +```python +dp.create_streaming_table("events_archive") + +dp.append_flow( + target="events_archive", + source="old_events_source" +) +``` + +### `dp.create_sink()` + +Creates a custom sink for streaming data. + +```python +def write_to_kafka(batch_df, batch_id): + batch_df.write.format("kafka").option("topic", "output").save() + +dp.create_sink( + name="kafka_sink", + sink_fn=write_to_kafka +) +``` + +--- + +## Reading Data + +**Use standard Spark APIs** - SDP automatically tracks dependencies: + +```python +# Batch read (for materialized views or batch tables) +df = spark.read.table("catalog.schema.source_table") + +# Streaming read (for streaming tables) +df = spark.readStream.table("catalog.schema.source_table") + +# Unqualified name (uses pipeline's default catalog/schema) +df = spark.read.table("source_table") + +# Read from file with Auto Loader (schema location managed automatically in SDP) +df = spark.readStream.format("cloudFiles") \ + .option("cloudFiles.format", "json") \ + .load("/Volumes/catalog/schema/raw/data/") +``` + +**Do NOT use:** +- `dp.read()` or `dp.read_stream()` - not part of modern API +- `dlt.read()` or `dlt.read_stream()` - legacy API +- `dlt.apply_changes()` - legacy API; use `dp.create_auto_cdc_flow()` instead +- `import dlt` - legacy module; use `from pyspark import pipelines as dp` + +--- + +## Table Name Resolution + +| Level | Example | When to Use | +|-------|---------|-------------| +| Unqualified | `spark.read.table("my_table")` | Tables in same pipeline (recommended) | +| Schema-qualified | `spark.read.table("other_schema.my_table")` | Different schema, same catalog | +| Fully-qualified | `spark.read.table("other_catalog.schema.table")` | External catalogs | + +**Best practice:** Use unqualified names for pipeline-internal tables. + +### Multi-Schema Pattern (One Pipeline) + +Write to multiple schemas from a single pipeline using fully qualified names: + +```python +from pyspark import pipelines as dp + +# Bronze → writes to bronze schema +@dp.table(name="my_catalog.bronze.raw_orders") +def bronze_orders(): + return spark.readStream.format("cloudFiles") \ + .option("cloudFiles.format", "json") \ + .load("/Volumes/my_catalog/raw/orders/") + +# Silver → writes to silver schema, reads from bronze +@dp.table(name="my_catalog.silver.clean_orders") +def silver_orders(): + return spark.readStream.table("my_catalog.bronze.raw_orders") \ + .filter("order_id IS NOT NULL") +``` + +--- + +## Pipeline Parameters + +Access configuration values set in pipeline settings: + +```python +# Get parameter value +catalog = spark.conf.get("target_catalog") +schema = spark.conf.get("target_schema") + +# With default +env = spark.conf.get("environment", "dev") + +@dp.table(name=f"{catalog}.{schema}.my_table") +def my_table(): + return spark.readStream.format("cloudFiles") \ + .option("cloudFiles.format", "json") \ + .load("/Volumes/...") +``` + +--- + +## Prohibited Operations + +**Do NOT include these in dataset definitions:** + +```python +# These cause unexpected behavior +@dp.table(name="bad_example") +def bad_example(): + df = spark.read.table("source") + df.collect() # No collect() + df.count() # No count() + df.toPandas() # No toPandas() + df.save(...) # No save() + df.saveAsTable(...) # No saveAsTable() + return df +``` + +Dataset functions should only contain code to define the transformation, not execute actions. + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| `sequence_by` type error | Use `col("column")` not string in `create_auto_cdc_flow()` | +| SCD type syntax error | Type 2 uses integer `2`, Type 1 uses string `"1"` | +| Table not found | Check catalog/schema qualification or pipeline default settings | +| Parameter not resolved | Use `spark.conf.get("param_name")` | +| Actions in definition | Remove `collect()`, `count()`, `save()` from table functions | +| Using legacy `dlt` API | Replace `import dlt` with `from pyspark import pipelines as dp` | +| Using `input_file_name()` | Use `F.col("_metadata.file_path")` | diff --git a/.claude/skills/databricks-spark-declarative-pipelines/references/python/2-ingestion.md b/.claude/skills/databricks-spark-declarative-pipelines/references/python/2-ingestion.md new file mode 100644 index 0000000..06ddad2 --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/python/2-ingestion.md @@ -0,0 +1,150 @@ +# Python Data Ingestion + +Data ingestion patterns using the modern `pyspark.pipelines` API. + +**Official Documentation:** +- [Auto Loader options](https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/options) +- [Structured Streaming + Kafka](https://docs.databricks.com/aws/en/structured-streaming/kafka) + +--- + +## Auto Loader (Cloud Files) + +Auto Loader incrementally processes new files. In SDP pipelines, schema location and checkpoints are managed automatically. + +### Basic Pattern + +```python +from pyspark import pipelines as dp +from pyspark.sql import functions as F + +@dp.table(name="bronze_orders", cluster_by=["order_date"]) +def bronze_orders(): + return ( + spark.readStream + .format("cloudFiles") + .option("cloudFiles.format", "json") + .option("cloudFiles.inferColumnTypes", "true") + .load("/Volumes/my_catalog/my_schema/raw/orders/") + .withColumn("_ingested_at", F.current_timestamp()) + .withColumn("_source_file", F.col("_metadata.file_path")) + ) +``` + +**Key options:** +- `cloudFiles.format`: `json`, `csv`, `parquet`, `avro`, `text`, `binaryFile` +- `cloudFiles.inferColumnTypes`: Infer types (default strings) +- `cloudFiles.schemaHints`: Hint specific column types + +### Rescue Data (Quarantine Pattern) + +```python +@dp.table(name="bronze_events", cluster_by=["ingestion_date"]) +def bronze_events(): + return ( + spark.readStream + .format("cloudFiles") + .option("cloudFiles.format", "json") + .option("rescuedDataColumn", "_rescued_data") + .load("/Volumes/catalog/schema/raw/events/") + .withColumn("_ingested_at", F.current_timestamp()) + .withColumn("_has_errors", F.col("_rescued_data").isNotNull()) + ) + +@dp.table(name="bronze_quarantine") +def bronze_quarantine(): + return spark.readStream.table("bronze_events").filter("_has_errors = true") + +@dp.table(name="silver_clean") +def silver_clean(): + return spark.readStream.table("bronze_events").filter("_has_errors = false") +``` + +--- + +## Streaming Sources + +### Kafka + +```python +@dp.table(name="bronze_kafka_events") +def bronze_kafka_events(): + kafka_brokers = spark.conf.get("kafka_brokers") + return ( + spark.readStream + .format("kafka") + .option("kafka.bootstrap.servers", kafka_brokers) + .option("subscribe", "events-topic") + .option("startingOffsets", "latest") + .load() + .selectExpr( + "CAST(key AS STRING) AS event_key", + "CAST(value AS STRING) AS event_value", + "topic", "partition", "offset", + "timestamp AS kafka_timestamp" + ) + .withColumn("_ingested_at", F.current_timestamp()) + ) +``` + +### Parse JSON from Kafka + +```python +from pyspark.sql.types import StructType, StructField, StringType, TimestampType + +event_schema = StructType([ + StructField("event_id", StringType()), + StructField("event_type", StringType()), + StructField("timestamp", TimestampType()) +]) + +@dp.table(name="silver_events") +def silver_events(): + return ( + spark.readStream.table("bronze_kafka_events") + .withColumn("data", F.from_json("event_value", event_schema)) + .select("data.*", "kafka_timestamp", "_ingested_at") + ) +``` + +--- + +## Authentication + +### Databricks Secrets + +```python +username = dbutils.secrets.get(scope="kafka", key="username") +password = dbutils.secrets.get(scope="kafka", key="password") +``` + +### Pipeline Parameters + +```python +kafka_brokers = spark.conf.get("kafka_brokers") +input_path = spark.conf.get("input_path") +``` + +--- + +## Best Practices + +1. **Add ingestion metadata:** +```python +.withColumn("_ingested_at", F.current_timestamp()) +.withColumn("_source_file", F.col("_metadata.file_path")) +``` + +2. **Handle rescue data** - route malformed records to quarantine + +3. **Use pipeline parameters** for paths and connection strings + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| Files not picked up | Verify path and format match actual files | +| Schema evolution breaking | Use `rescuedDataColumn` and monitor `_rescued_data` | +| Kafka lag increasing | Check downstream bottlenecks | diff --git a/.claude/skills/databricks-spark-declarative-pipelines/references/python/3-streaming-patterns.md b/.claude/skills/databricks-spark-declarative-pipelines/references/python/3-streaming-patterns.md new file mode 100644 index 0000000..44fd619 --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/python/3-streaming-patterns.md @@ -0,0 +1,382 @@ +# Python Streaming Patterns + +Streaming-specific patterns including deduplication, windowed aggregations, late-arriving data handling, and stateful operations. + +**Import**: `from pyspark import pipelines as dp` + +--- + +## Deduplication Patterns + +### By Key + +```python +from pyspark import pipelines as dp +from pyspark.sql import functions as F +from pyspark.sql.window import Window + +@dp.table(name="silver_events_dedup", cluster_by=["event_date"]) +def silver_events_dedup(): + """Deduplicate by event_id, keeping first occurrence.""" + window_spec = Window.partitionBy("event_id").orderBy("event_timestamp") + return ( + spark.readStream.table("bronze_events") + .withColumn("rn", F.row_number().over(window_spec)) + .filter(F.col("rn") == 1) + .drop("rn") + ) +``` + +### With Time Window + +Deduplicate within time window to handle late arrivals: + +```python +@dp.table(name="silver_events_dedup") +def silver_events_dedup(): + return ( + spark.readStream.table("bronze_events") + .groupBy( + "event_id", "user_id", "event_type", "event_timestamp", + F.window("event_timestamp", "1 hour") + ) + .agg(F.min("_ingested_at").alias("first_seen_at")) + ) +``` + +### Composite Key + +```python +@dp.table(name="silver_transactions_dedup") +def silver_transactions_dedup(): + return ( + spark.readStream.table("bronze_transactions") + .groupBy("transaction_id", "customer_id", "amount", "transaction_timestamp") + .agg(F.min("_ingested_at").alias("_ingested_at")) + ) +``` + +--- + +## Windowed Aggregations + +### Tumbling Windows + +Non-overlapping fixed-size windows: + +```python +@dp.table(name="silver_sensor_5min", cluster_by=["sensor_id"]) +def silver_sensor_5min(): + """5-minute tumbling window aggregations.""" + return ( + spark.readStream.table("bronze_sensor_events") + .groupBy( + F.col("sensor_id"), + F.window("event_timestamp", "5 minutes") + ) + .agg( + F.avg("temperature").alias("avg_temperature"), + F.min("temperature").alias("min_temperature"), + F.max("temperature").alias("max_temperature"), + F.count("*").alias("event_count") + ) + ) +``` + +### Multiple Window Sizes + +```python +# 1-minute for real-time monitoring +@dp.table(name="gold_sensor_1min") +def gold_sensor_1min(): + return ( + spark.readStream.table("silver_sensor_data") + .groupBy( + "sensor_id", + F.window("event_timestamp", "1 minute") + ) + .agg( + F.avg("value").alias("avg_value"), + F.count("*").alias("event_count") + ) + .select( + "sensor_id", + F.col("window.start").alias("window_start"), + F.col("window.end").alias("window_end"), + "avg_value", + "event_count" + ) + ) + +# 1-hour for trend analysis +@dp.table(name="gold_sensor_1hour") +def gold_sensor_1hour(): + return ( + spark.readStream.table("silver_sensor_data") + .groupBy( + "sensor_id", + F.window("event_timestamp", "1 hour") + ) + .agg( + F.avg("value").alias("avg_value"), + F.stddev("value").alias("stddev_value") + ) + ) +``` + +### Session Windows + +Group events into sessions based on inactivity gaps: + +```python +@dp.table(name="silver_user_sessions") +def silver_user_sessions(): + """Group user events into sessions with 30-minute inactivity timeout.""" + return ( + spark.readStream.table("bronze_user_events") + .groupBy( + F.col("user_id"), + F.session_window("event_timestamp", "30 minutes") + ) + .agg( + F.min("event_timestamp").alias("session_start"), + F.max("event_timestamp").alias("session_end"), + F.count("*").alias("event_count"), + F.collect_list("event_type").alias("event_sequence") + ) + ) +``` + +--- + +## Late-Arriving Data + +### Event-Time vs Processing-Time + +Always use event timestamp for business logic: + +```python +@dp.table(name="gold_daily_orders") +def gold_daily_orders(): + return ( + spark.readStream.table("silver_orders") + .groupBy(F.to_date("order_timestamp").alias("order_date")) # Event time + .agg( + F.count("*").alias("order_count"), + F.sum("amount").alias("total_amount") + ) + ) +``` + +**Keep processing time for debugging:** +```python +.select( + "order_id", "order_timestamp", # Event time (business logic) + "customer_id", "amount", + "_ingested_at" # Processing time (debugging only) +) +``` + +--- + +## Joins + +### Stream-to-Static Joins + +Enrich streaming data with dimension tables: + +```python +@dp.table(name="silver_sales_enriched", cluster_by=["product_id"]) +def silver_sales_enriched(): + """Enrich streaming sales with static product dimension.""" + sales = spark.readStream.table("bronze_sales") + products = spark.read.table("dim_products") + return ( + sales.join(products, "product_id", "left") + .select( + "sale_id", "product_id", "quantity", "sale_timestamp", + "product_name", "category", "price" + ) + .withColumn("total_amount", F.col("quantity") * F.col("price")) + ) +``` + +### Stream-to-Stream Joins + +```python +@dp.table(name="silver_orders_with_payments") +def silver_orders_with_payments(): + """Join orders with payments within 1-hour window.""" + orders = spark.readStream.table("bronze_orders") + payments = spark.readStream.table("bronze_payments") + + return ( + orders.join( + payments, + (orders.order_id == payments.order_id) & + (payments.payment_timestamp >= orders.order_timestamp) & + (payments.payment_timestamp <= orders.order_timestamp + F.expr("INTERVAL 1 HOUR")), + "inner" + ) + .select( + orders.order_id, + orders.customer_id, + orders.order_timestamp, + orders.amount.alias("order_amount"), + payments.payment_id, + payments.payment_timestamp, + payments.amount.alias("payment_amount") + ) + ) +``` + +**Important:** Use time bounds in join condition to limit state retention. + +--- + +## Incremental Aggregations + +### Running Totals + +```python +@dp.table(name="silver_customer_running_totals") +def silver_customer_running_totals(): + return ( + spark.readStream.table("bronze_transactions") + .groupBy("customer_id") + .agg( + F.sum("amount").alias("total_spent"), + F.count("*").alias("transaction_count"), + F.max("transaction_timestamp").alias("last_transaction_at") + ) + ) +``` + +--- + +## Anomaly Detection + +### Real-Time Outlier Detection + +```python +@dp.table(name="silver_sensor_with_anomalies") +def silver_sensor_with_anomalies(): + window_spec = Window.partitionBy("sensor_id").orderBy("event_timestamp").rowsBetween(-100, 0) + + return ( + spark.readStream.table("bronze_sensor_events") + .withColumn("rolling_avg", F.avg("temperature").over(window_spec)) + .withColumn("rolling_stddev", F.stddev("temperature").over(window_spec)) + .withColumn("anomaly_flag", + F.when(F.col("temperature") > F.col("rolling_avg") + (3 * F.col("rolling_stddev")), "HIGH_OUTLIER") + .when(F.col("temperature") < F.col("rolling_avg") - (3 * F.col("rolling_stddev")), "LOW_OUTLIER") + .otherwise("NORMAL") + ) + ) + +@dp.table(name="silver_sensor_anomalies") +def silver_sensor_anomalies(): + return ( + spark.readStream.table("silver_sensor_with_anomalies") + .filter(F.col("anomaly_flag").isin("HIGH_OUTLIER", "LOW_OUTLIER")) + ) +``` + +### Threshold-Based Filtering + +```python +@dp.table(name="silver_high_value_transactions") +def silver_high_value_transactions(): + return ( + spark.readStream.table("bronze_transactions") + .filter(F.col("amount") > 10000) + ) +``` + +--- + +## Monitoring Lag + +```python +@dp.table(name="monitoring_lag") +def monitoring_lag(): + return ( + spark.readStream.table("bronze_kafka_events") + .groupBy(F.window("kafka_timestamp", "1 minute")) + .agg( + F.lit("kafka_events").alias("source"), + F.max("kafka_timestamp").alias("max_event_timestamp"), + F.current_timestamp().alias("processing_timestamp") + ) + .withColumn("lag_seconds", + F.unix_timestamp("processing_timestamp") - F.unix_timestamp("max_event_timestamp") + ) + ) +``` + +--- + +## Best Practices + +### 1. Use Event Timestamps + +```python +# Correct: Event timestamp for logic +.groupBy(F.date_trunc("hour", "event_timestamp")) + +# Avoid: Processing timestamp +# .groupBy(F.date_trunc("hour", "_ingested_at")) +``` + +### 2. Window Size Selection + +- **1-5 minutes**: Real-time monitoring +- **15-60 minutes**: Operational dashboards +- **1-24 hours**: Analytical reports + +### 3. State Management + +Higher cardinality = more state: + +```python +# High state: 1M users x 10K products x 100M sessions +.groupBy("user_id", "product_id", "session_id") + +# Lower state: 1M users x 100 categories x days +.groupBy("user_id", "product_category", F.to_date("event_time")) +``` + +Use time windows to bound state retention. + +### 4. Deduplicate Early + +Apply at bronze → silver transition: + +```python +# Bronze: Accept duplicates +@dp.table(name="bronze_events") +def bronze_events(): + return spark.readStream.format("cloudFiles")... + +# Silver: Deduplicate immediately +@dp.table(name="silver_events") +def silver_events(): + return spark.readStream.table("bronze_events").dropDuplicates(["event_id"]) + +# Gold: Work with clean data +@dp.table(name="gold_metrics") +def gold_metrics(): + return spark.readStream.table("silver_events")... +``` + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| High memory with windows | Use larger windows, reduce group-by cardinality | +| Duplicate events in output | Add explicit deduplication by unique key | +| Missing late-arriving events | Increase window size or use longer retention | +| Stream-to-stream join empty | Verify join conditions and time bounds | +| State growth over time | Add time windows, reduce cardinality, materialize intermediates | diff --git a/.claude/skills/databricks-spark-declarative-pipelines/references/python/4-cdc-patterns.md b/.claude/skills/databricks-spark-declarative-pipelines/references/python/4-cdc-patterns.md new file mode 100644 index 0000000..9e05370 --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/python/4-cdc-patterns.md @@ -0,0 +1,449 @@ +# Python CDC Patterns (AUTO CDC & SCD) + +Change Data Capture patterns using AUTO CDC for SCD Type 1 and Type 2, plus querying SCD history tables. + +**Import**: `from pyspark import pipelines as dp` + +--- + +## Overview + +AUTO CDC automatically handles Change Data Capture to track changes using Slow Changing Dimensions (SCD). It provides automatic deduplication, change tracking, and handles late-arriving data correctly. + +**Where to apply AUTO CDC:** +- **Silver layer**: When business users need deduplicated or historical data +- **Gold layer**: When implementing dimensional modeling (star schema) + +--- + +## SCD Type 1 vs Type 2 + +### SCD Type 1 (In-place updates) +- **Overwrites** old values with new values +- **No history preserved** - only current state +- **Use for**: Error corrections, attributes where history doesn't matter +- **Syntax**: `stored_as_scd_type="1"` (string) + +### SCD Type 2 (History tracking) +- **Creates new row** for each change +- **Preserves full history** with `__START_AT` and `__END_AT` timestamps +- **Use for**: Tracking changes over time (addresses, prices, roles) +- **Syntax**: `stored_as_scd_type=2` (integer) + +**Important:** Type 2 uses integer `2`, Type 1 uses string `"1"`. + +--- + +## Creating AUTO CDC Flows + +### SCD Type 2 + +```python +from pyspark import pipelines as dp +from pyspark.sql.functions import col + +target_schema = spark.conf.get("target_schema") +source_schema = spark.conf.get("source_schema") + +# Step 1: Create target table +dp.create_streaming_table(f"{target_schema}.dim_customers") + +# Step 2: Create AUTO CDC flow +dp.create_auto_cdc_flow( + target=f"{target_schema}.dim_customers", + source=f"{source_schema}.customers_cdc_clean", + keys=["customer_id"], + sequence_by=col("event_timestamp"), # Note: use col(), not string + stored_as_scd_type=2, # Integer for Type 2 + apply_as_deletes=col("operation") == "DELETE", + except_column_list=["operation", "_ingested_at", "_source_file"] +) +``` + +### SCD Type 1 + +```python +dp.create_streaming_table(f"{target_schema}.orders_current") + +dp.create_auto_cdc_flow( + target=f"{target_schema}.orders_current", + source=f"{source_schema}.orders_clean", + keys=["order_id"], + sequence_by=col("updated_timestamp"), + stored_as_scd_type="1" # String for Type 1 +) +``` + +### Selective History Tracking + +Track history only when specific columns change: + +```python +dp.create_auto_cdc_flow( + target="gold.dim_products", + source="silver.products_clean", + keys=["product_id"], + sequence_by=col("modified_at"), + stored_as_scd_type=2, + track_history_column_list=["price", "cost"] # Only track these columns +) +``` + +When `price` or `cost` changes, a new version is created. Other column changes update the current record without new versions. + +--- + +## Complete Pattern: Clean + AUTO CDC + +### Step 1: Clean and Validate Source Data + +```python +from pyspark import pipelines as dp +from pyspark.sql import functions as F + +schema = spark.conf.get("schema") + +@dp.table( + name=f"{schema}.users_clean", + comment="Cleaned and validated user data", + cluster_by=["user_id"] +) +def users_clean(): + """ + Clean data with proper typing and quality checks. + """ + return ( + spark.readStream.table("bronze_users") + .filter(F.col("user_id").isNotNull()) + .filter(F.col("email").isNotNull()) + .filter(F.col("email").rlike(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")) + .withColumn("created_timestamp", F.to_timestamp("created_timestamp")) + .withColumn("updated_timestamp", F.to_timestamp("updated_timestamp")) + .drop("_rescued_data") + .select( + "user_id", "email", "name", "subscription_tier", "country", + "created_timestamp", "updated_timestamp", + "_ingested_at", "_source_file" + ) + ) +``` + +### Step 2: Apply AUTO CDC + +```python +from pyspark.sql.functions import col + +target_schema = spark.conf.get("target_schema") +source_schema = spark.conf.get("source_schema") + +dp.create_streaming_table(f"{target_schema}.dim_users") + +dp.create_auto_cdc_flow( + target=f"{target_schema}.dim_users", + source=f"{source_schema}.users_clean", + keys=["user_id"], + sequence_by=col("updated_timestamp"), + stored_as_scd_type=2, + except_column_list=["_ingested_at", "_source_file"] +) +``` + +--- + +## Using Temporary Views with AUTO CDC + +`@dp.temporary_view()` creates in-pipeline temporary views useful for intermediate transformations before AUTO CDC. + +**Key Constraints:** +- Cannot specify `catalog` or `schema` (pipeline-scoped only) +- Cannot use `cluster_by` (not persisted) +- Only exists during pipeline execution + +```python +from pyspark import pipelines as dp +from pyspark.sql import functions as F + +# Step 1: Temporary view for complex business logic +@dp.temporary_view() +def orders_with_calculated_fields(): + """ + Temporary view for complex calculations. + No catalog/schema needed - exists only in pipeline. + """ + return ( + spark.readStream.table("bronze.orders") + .withColumn("order_total", F.col("quantity") * F.col("unit_price")) + .withColumn("discount_amount", F.col("order_total") * F.col("discount_rate")) + .withColumn("final_amount", F.col("order_total") - F.col("discount_amount")) + .withColumn("order_category", + F.when(F.col("final_amount") > 1000, "large") + .when(F.col("final_amount") > 100, "medium") + .otherwise("small") + ) + .filter(F.col("order_id").isNotNull()) + .filter(F.col("final_amount") > 0) + ) + +# Step 2: Apply AUTO CDC using the temporary view as source +target_schema = spark.conf.get("target_schema") + +dp.create_streaming_table(f"{target_schema}.orders_current") +dp.create_auto_cdc_flow( + target=f"{target_schema}.orders_current", + source="orders_with_calculated_fields", # Reference temporary view by name + keys=["order_id"], + sequence_by=col("order_date"), + stored_as_scd_type="1" +) +``` + +--- + +## Querying SCD Type 2 Tables + +SCD Type 2 tables include temporal columns: +- `__START_AT` - When this version became effective +- `__END_AT` - When this version expired (NULL for current) + +### Current State + +```python +@dp.materialized_view(name="dim_customers_current") +def dim_customers_current(): + """All current records.""" + return ( + spark.read.table("dim_customers") + .filter(F.col("__END_AT").isNull()) + .select( + "customer_id", "customer_name", "email", "phone", "address", + F.col("__START_AT").alias("valid_from") + ) + ) +``` + +### Point-in-Time Queries + +Get state as of a specific date: + +```python +@dp.materialized_view(name="products_as_of_date") +def products_as_of_date(): + """Products as of January 1, 2024.""" + as_of_date = "2024-01-01" + return ( + spark.read.table("products_history") + .filter(F.col("__START_AT") <= as_of_date) + .filter( + (F.col("__END_AT") > as_of_date) | + F.col("__END_AT").isNull() + ) + ) +``` + +### Change Analysis + +Track all changes for an entity: + +```python +def get_customer_history(customer_id: str): + """Get complete history for a customer.""" + return ( + spark.read.table("dim_customers") + .filter(F.col("customer_id") == customer_id) + .withColumn("days_active", + F.coalesce( + F.datediff("__END_AT", "__START_AT"), + F.datediff(F.current_timestamp(), "__START_AT") + ) + ) + .orderBy(F.col("__START_AT").desc()) + ) +``` + +--- + +## Joining Facts with Historical Dimensions + +### At Transaction Time + +```python +@dp.materialized_view(name="sales_with_historical_prices") +def sales_with_historical_prices(): + """Join sales with product prices at time of sale.""" + sales = spark.read.table("sales_fact") + products = spark.read.table("products_history") + + return ( + sales.join( + products, + (sales.product_id == products.product_id) & + (sales.sale_date >= products.__START_AT) & + ((sales.sale_date < products.__END_AT) | products.__END_AT.isNull()), + "inner" + ) + .select( + sales.sale_id, + sales.product_id, + sales.sale_date, + sales.quantity, + products.product_name, + products.price.alias("unit_price_at_sale_time"), + (sales.quantity * products.price).alias("calculated_amount"), + products.category + ) + ) +``` + +### With Current Dimension + +```python +@dp.materialized_view(name="sales_with_current_prices") +def sales_with_current_prices(): + """Join sales with current product information.""" + sales = spark.read.table("sales_fact") + products_current = spark.read.table("products_history").filter(F.col("__END_AT").isNull()) + + return ( + sales.join(products_current, "product_id", "inner") + .select( + "sale_id", "product_id", "sale_date", "quantity", + sales.amount.alias("amount_at_sale"), + products_current.product_name.alias("current_product_name"), + products_current.price.alias("current_price") + ) + ) +``` + +--- + +## Common Patterns + +### Pattern 1: Gold Dimensional Model + +```python +# Silver: Cleaned streaming tables +@dp.table(name="silver.customers_clean") +def customers_clean(): + return spark.readStream.table("bronze.customers").filter(...) + +# Gold: SCD Type 2 dimension +dp.create_streaming_table("gold.dim_customers") +dp.create_auto_cdc_flow( + target="gold.dim_customers", + source="silver.customers_clean", + keys=["customer_id"], + sequence_by=col("updated_at"), + stored_as_scd_type=2 +) + +# Gold: Fact table (no AUTO CDC) +@dp.table(name="gold.fact_orders") +def fact_orders(): + return spark.read.table("silver.orders_clean") +``` + +### Pattern 2: Silver Deduplication for Joins + +```python +# Silver: AUTO CDC for deduplication +dp.create_streaming_table("silver.products_dedupe") +dp.create_auto_cdc_flow( + target="silver.products_dedupe", + source="bronze.products", + keys=["product_id"], + sequence_by=col("modified_at"), + stored_as_scd_type="1" # Type 1: just dedupe, no history +) + +# Silver: Join with deduplicated data +@dp.table(name="silver.orders_enriched") +def orders_enriched(): + orders = spark.readStream.table("bronze.orders") + products = spark.read.table("silver.products_dedupe") + return orders.join(products, "product_id") +``` + +### Pattern 3: Mixed SCD Types + +```python +# SCD Type 2: Need history +dp.create_auto_cdc_flow( + target="gold.dim_customers", + source="silver.customers", + keys=["customer_id"], + sequence_by=col("updated_at"), + stored_as_scd_type=2 # Track address changes over time +) + +# SCD Type 1: Corrections only +dp.create_auto_cdc_flow( + target="gold.dim_products", + source="silver.products", + keys=["product_id"], + sequence_by=col("modified_at"), + stored_as_scd_type="1" # Current product info only +) +``` + +--- + +## Best Practices + +### 1. Clean Data Before AUTO CDC + +Apply type casting, validation, and filtering first: + +```python +@dp.table(name="users_clean") +def users_clean(): + return ( + spark.readStream.table("bronze_users") + .filter(F.col("user_id").isNotNull()) + .filter(F.col("email").isNotNull()) + .withColumn("updated_at", F.to_timestamp("updated_at")) + ) + +# Then apply AUTO CDC +dp.create_auto_cdc_flow( + target="dim_users", + source="users_clean", + keys=["user_id"], + sequence_by=col("updated_at"), + stored_as_scd_type=2 +) +``` + +### 2. Use col() for sequence_by + +```python +# Correct +sequence_by=col("event_timestamp") + +# Wrong - causes error +# sequence_by="event_timestamp" +``` + +### 3. Choose the Right SCD Type + +- **Type 2** (`stored_as_scd_type=2`): Need to query historical states +- **Type 1** (`stored_as_scd_type="1"`): Only need current state or deduplication + +### 4. Use meaningful sequence_by column + +Should reflect true chronological order of changes: +- `updated_timestamp` +- `modified_at` +- `event_timestamp` + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| `sequence_by` type error | Use `col("column")` not string | +| SCD type syntax error | Type 2 uses integer `2`, Type 1 uses string `"1"` | +| Duplicates still appearing | Check `keys` include all business key columns | +| Missing `__START_AT`/`__END_AT` | These only appear in SCD Type 2, not Type 1 | +| Late data not handled | Ensure `sequence_by` reflects true event time | +| Performance issues | Use `track_history_column_list` to limit version triggers | diff --git a/.claude/skills/databricks-spark-declarative-pipelines/references/python/5-performance.md b/.claude/skills/databricks-spark-declarative-pipelines/references/python/5-performance.md new file mode 100644 index 0000000..0cdcc94 --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/python/5-performance.md @@ -0,0 +1,423 @@ +# Python Performance Tuning + +Performance optimization strategies including Liquid Clustering, materialized view refresh, state management, and compute configuration. + +**Import**: `from pyspark import pipelines as dp` + +--- + +## Liquid Clustering (Recommended) + +Liquid Clustering is the recommended approach for data layout optimization. It replaces manual partitioning and Z-ORDER. + +### Benefits + +- **Adaptive**: Adjusts to data distribution changes +- **Multi-dimensional**: Clusters on multiple columns simultaneously +- **Automatic file sizing**: Maintains optimal file sizes +- **Self-optimizing**: Reduces manual OPTIMIZE commands + +### Basic Syntax + +```python +from pyspark import pipelines as dp + +@dp.table(cluster_by=["event_type", "event_date"]) +def bronze_events(): + return spark.readStream.format("cloudFiles").load("/data") +``` + +### Automatic Key Selection + +```python +@dp.table(cluster_by=["AUTO"]) +def bronze_events(): + return spark.readStream.format("cloudFiles").load("/data") +``` + +**When to use AUTO**: Learning phase, unknown access patterns, prototyping +**When to define manually**: Well-known query patterns, production workloads + +--- + +## Cluster Key Selection by Layer + +### Bronze Layer + +Cluster by event type + date: + +```python +@dp.table( + name="bronze_events", + cluster_by=["event_type", "ingestion_date"], + table_properties={"delta.autoOptimize.optimizeWrite": "true"} +) +def bronze_events(): + return ( + spark.readStream.format("cloudFiles") + .option("cloudFiles.format", "json") + .load("/Volumes/my_catalog/my_schema/raw/events/") + .withColumn("_ingested_at", F.current_timestamp()) + .withColumn("ingestion_date", F.current_date()) + ) +``` + +**Why**: Bronze filtered by event type for processing and by date for incremental loads. + +### Silver Layer + +Cluster by primary key + business dimension: + +```python +@dp.table( + name="silver_orders", + cluster_by=["customer_id", "order_date"] +) +def silver_orders(): + return ( + spark.readStream.table("bronze_orders") + .withColumn("order_date", F.to_date("order_timestamp")) + .select("order_id", "customer_id", "product_id", "amount", "order_date") + ) +``` + +**Why**: Entity lookups (by ID) and time-range queries (by date). + +### Gold Layer + +Cluster by aggregation dimensions: + +```python +@dp.materialized_view( + name="gold_sales_summary", + cluster_by=["product_category", "year_month"] +) +def gold_sales_summary(): + return ( + spark.read.table("silver_orders") + .withColumn("year_month", F.date_format("order_date", "yyyy-MM")) + .groupBy("product_category", "year_month") + .agg( + F.sum("amount").alias("total_sales"), + F.count("*").alias("transaction_count"), + F.avg("amount").alias("avg_order_value") + ) + ) +``` + +**Why**: Dashboard filters (category, region, time period). + +### Selection Guidelines + +| Layer | Good Keys | Rationale | +|-------|-----------|-----------| +| **Bronze** | event_type, ingestion_date | Filter by type; date for incremental | +| **Silver** | primary_key, business_date | Entity lookups + time ranges | +| **Gold** | aggregation_dimensions | Dashboard filters | + +**Best practices:** +- First key: Most selective filter (e.g., customer_id) +- Second key: Next common filter (e.g., date) +- Order matters: Most selective first +- Limit to 4 keys: Diminishing returns beyond 4 +- **Use `["AUTO"]` if unsure** + +--- + +## Table Properties + +### Auto-Optimize + +```python +@dp.table( + name="bronze_events", + table_properties={ + "delta.autoOptimize.optimizeWrite": "true", + "delta.autoOptimize.autoCompact": "true" + } +) +def bronze_events(): + return spark.readStream.format("cloudFiles").load(...) +``` + +### Change Data Feed + +```python +@dp.table( + name="silver_customers", + table_properties={"delta.enableChangeDataFeed": "true"} +) +def silver_customers(): + return spark.readStream.table("bronze_customers") +``` + +**Use when**: Downstream systems need efficient change tracking. + +### Retention Periods + +```python +@dp.table( + name="bronze_high_volume", + table_properties={ + "delta.logRetentionDuration": "7 days", + "delta.deletedFileRetentionDuration": "7 days" + } +) +def bronze_high_volume(): + return spark.readStream.format("cloudFiles").load(...) +``` + +**Use for**: High-volume tables to reduce storage costs. + +--- + +## State Management for Streaming + +### Understand State Growth + +Higher cardinality = more state: + +```python +# High state: 1M users x 10K products x 100M sessions - Massive state! +.groupBy("user_id", "product_id", "session_id") +``` + +### Reduce State Size + +**Strategy 1: Reduce cardinality** + +```python +@dp.table(name="user_category_stats") +def user_category_stats(): + return ( + spark.readStream.table("bronze_events") + .groupBy( + "user_id", + "product_category", # 100 categories (not 10K products) + F.to_date("event_time").alias("event_date") + ) + .agg(F.count("*").alias("events")) + ) +``` + +**Strategy 2: Use time windows** + +```python +@dp.table(name="user_hourly_stats") +def user_hourly_stats(): + return ( + spark.readStream.table("bronze_events") + .groupBy( + "user_id", + F.window("event_time", "1 hour") + ) + .agg(F.count("*").alias("events")) + ) +``` + +**Strategy 3: Materialize intermediates** + +```python +# Streaming aggregation (maintains state) +@dp.table(name="user_daily_stats") +def user_daily_stats(): + return ( + spark.readStream.table("bronze_events") + .groupBy("user_id", F.to_date("event_time").alias("event_date")) + .agg(F.count("*").alias("event_count")) + ) + +# Batch aggregation (no streaming state) +@dp.materialized_view(name="user_monthly_stats") +def user_monthly_stats(): + return ( + spark.read.table("user_daily_stats") + .groupBy("user_id", F.date_trunc("month", "event_date").alias("month")) + .agg(F.sum("event_count").alias("total_events")) + ) +``` + +--- + +## Join Optimization + +### Stream-to-Static (Efficient) + +```python +@dp.table(name="sales_enriched") +def sales_enriched(): + """Small static dimension, large streaming fact.""" + sales = spark.readStream.table("bronze_sales") + products = spark.read.table("dim_products") # Small, broadcast + + return ( + sales.join(products, "product_id", "left") + .select("sale_id", "product_id", "amount", "product_name", "category") + ) +``` + +**Best practice**: Keep static dimensions small (<10K rows) for broadcast. + +### Stream-to-Stream (Stateful) + +```python +@dp.table(name="orders_with_payments") +def orders_with_payments(): + """Time bounds limit state retention.""" + orders = spark.readStream.table("bronze_orders") + payments = spark.readStream.table("bronze_payments") + + return orders.join( + payments, + (orders.order_id == payments.order_id) & + (payments.payment_time >= orders.order_time) & + (payments.payment_time <= orders.order_time + F.expr("INTERVAL 1 HOUR")), + "inner" + ) +``` + +--- + +## Query Optimization + +### Filter Early + +```python +# Filter at source +@dp.table(name="silver_recent") +def silver_recent(): + return ( + spark.readStream.table("bronze_events") + .filter(F.col("event_date") >= F.current_date() - 7) + ) + +# Avoid filtering late in separate table +# @dp.table(name="silver_all") +# def silver_all(): return spark.readStream.table("bronze_events") +# @dp.materialized_view(name="gold_recent") +# def gold_recent(): return spark.read.table("silver_all").filter(...) +``` + +### Select Specific Columns + +```python +# Only needed columns +.select("customer_id", "order_date", "amount") + +# Avoid SELECT * +# .select("*") +``` + +--- + +## Pre-Aggregation + +```python +@dp.materialized_view(name="orders_monthly") +def orders_monthly(): + """Pre-aggregate for fast queries.""" + return ( + spark.read.table("large_orders_table") + .groupBy( + "customer_id", + F.year("order_date").alias("year"), + F.month("order_date").alias("month") + ) + .agg(F.sum("amount").alias("total")) + ) + +# Query the MV directly - much faster than querying large_orders_table +``` + +--- + +## Compute Configuration + +### Serverless vs Classic + +| Aspect | Serverless | Classic | +|--------|-----------|---------| +| Startup | Fast (seconds) | Slower (minutes) | +| Scaling | Automatic, instant | Manual/autoscaling | +| Cost | Pay-per-use | Pay for cluster time | +| Best for | Variable workloads, dev/test | Steady workloads | + +### Serverless (Recommended) + +Enable at pipeline level: + +```yaml +execution_mode: continuous # or triggered +serverless: true +``` + +**Advantages**: No cluster management, instant scaling, lower cost for bursty workloads. + +--- + +## Complete Example + +```python +from pyspark import pipelines as dp +from pyspark.sql import functions as F + +# Bronze: Optimized ingestion +@dp.table( + name="bronze_orders", + cluster_by=["order_date"], + table_properties={ + "delta.autoOptimize.optimizeWrite": "true", + "delta.autoOptimize.autoCompact": "true" + } +) +def bronze_orders(): + return ( + spark.readStream.format("cloudFiles") + .option("cloudFiles.format", "json") + .load("/Volumes/my_catalog/my_schema/raw/orders/") + .withColumn("_ingested_at", F.current_timestamp()) + .withColumn("order_date", F.to_date("order_timestamp")) + ) + +# Silver: Efficient clustering for joins +@dp.table( + name="silver_orders", + cluster_by=["customer_id", "order_date"] +) +@dp.expect_or_drop("valid_amount", "amount > 0") +def silver_orders(): + return ( + spark.readStream.table("bronze_orders") + .filter(F.col("order_date") >= F.current_date() - 90) # Filter early + .withColumn("amount", F.col("amount").cast("decimal(10,2)")) # DECIMAL for monetary + .select("order_id", "customer_id", "amount", "order_date") # Select specific + ) + +# Gold: Pre-aggregated for dashboards +@dp.materialized_view( + name="gold_daily_revenue", + cluster_by=["order_date"] +) +def gold_daily_revenue(): + return ( + spark.read.table("silver_orders") + .groupBy("order_date") + .agg( + F.sum("amount").alias("total_revenue"), + F.count("order_id").alias("order_count"), + F.countDistinct("customer_id").alias("unique_customers") + ) + ) +``` + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| Pipeline running slowly | Check clustering, state size, join patterns | +| High memory usage | Unbounded state - add time windows, reduce cardinality | +| Many small files | Enable auto-optimize table properties | +| Expensive queries on large tables | Add clustering, create filtered MVs | +| MV refresh slow | Enable row tracking on source | diff --git a/.claude/skills/databricks-spark-declarative-pipelines/references/sql/1-syntax-basics.md b/.claude/skills/databricks-spark-declarative-pipelines/references/sql/1-syntax-basics.md new file mode 100644 index 0000000..54e45df --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/sql/1-syntax-basics.md @@ -0,0 +1,243 @@ +# SQL Syntax Basics + +Core SQL syntax for Spark Declarative Pipelines (SDP). + +--- + +## Table Types + +### Streaming Table + +Processes data incrementally. Use for continuous ingestion and transformations. + +```sql +CREATE OR REFRESH STREAMING TABLE bronze_events +COMMENT 'Raw event data' +CLUSTER BY (event_type, event_date) +TBLPROPERTIES ( + 'delta.autoOptimize.optimizeWrite' = 'true', + 'delta.autoOptimize.autoCompact' = 'true' +) +AS +SELECT + *, + current_timestamp() AS _ingested_at, + _metadata.file_path AS _source_file +FROM STREAM read_files('/Volumes/my_catalog/my_schema/raw/events/', format => 'json'); +``` + +**Key points:** +- Use `STREAM` keyword with source for incremental processing +- `CLUSTER BY` enables Liquid Clustering (recommended over PARTITION BY) +- Returns streaming DataFrame + +### Materialized View + +Batch table with automatic incremental refresh. + +```sql +CREATE OR REFRESH MATERIALIZED VIEW gold_daily_summary +COMMENT 'Daily aggregated metrics' +CLUSTER BY (report_date) +AS +SELECT + report_date, + SUM(amount) AS total_amount, + COUNT(*) AS transaction_count +FROM silver_orders +GROUP BY report_date; +``` + +**Key points:** +- No `STREAM` keyword - reads batch +- Automatically refreshes incrementally when source changes +- Use for aggregations and reporting tables + +### View (Persisted) + +A regular view published to Unity Catalog. Unlike materialized views, it doesn't store data - the query runs each time the view is accessed. + +```sql +CREATE VIEW taxi_raw AS +SELECT * FROM read_files("/Volumes/catalog/schema/raw/taxi/"); + +CREATE VIEW active_customers AS +SELECT customer_id, name, email +FROM dim_customers +WHERE status = 'active'; +``` + +**Key points:** +- Persisted in Unity Catalog (visible outside pipeline) +- No data storage - query executes on access +- Cannot use streaming queries or constraints +- Requires Unity Catalog pipeline with default publishing mode + +**Documentation:** [CREATE VIEW reference](https://docs.databricks.com/aws/en/ldp/developer/ldp-sql-ref-create-view) + +### Temporary View + +Pipeline-scoped view, not persisted. Useful for intermediate transformations. + +```sql +CREATE TEMPORARY VIEW orders_with_calculations AS +SELECT + *, + quantity * price AS total, + quantity * price * discount_rate AS discount_amount +FROM STREAM bronze_orders +WHERE quantity > 0; +``` + +**Key points:** +- Exists only during pipeline execution +- No storage cost +- Not visible outside pipeline +- Useful before AUTO CDC flows + +### Choosing Between View Types + +| Type | Persisted | Stores Data | Streaming | Use Case | +|------|-----------|-------------|-----------|----------| +| **Materialized View** | Yes | Yes | No | Aggregations, reporting tables | +| **View** | Yes | No | No | Simple transformations, external access | +| **Temporary View** | No | No | Yes | Intermediate steps, before AUTO CDC | + +--- + +## Data Quality (Expectations) +**Documentation:** [Expectations]https://docs.databricks.com/aws/en/ldp/expectations) + +### Constraint Syntax + +```sql +CREATE OR REFRESH STREAMING TABLE silver_orders ( + CONSTRAINT valid_amount EXPECT (amount > 0) ON VIOLATION DROP ROW, + CONSTRAINT valid_customer EXPECT (customer_id IS NOT NULL) ON VIOLATION DROP ROW, + CONSTRAINT critical_field EXPECT (order_id IS NOT NULL) ON VIOLATION FAIL UPDATE +) +AS +SELECT * FROM STREAM bronze_orders; +``` + +| Violation Action | Behavior | +|-----------------|----------| +| `ON VIOLATION DROP ROW` | Drop rows that violate | +| `ON VIOLATION FAIL UPDATE` | Fail pipeline if any row violates | +| (no action) | Log warning, keep all rows | + +### WHERE Clause Alternative + +For simple filtering without tracking: + +```sql +CREATE OR REFRESH STREAMING TABLE silver_orders AS +SELECT * FROM STREAM bronze_orders +WHERE amount > 0 AND customer_id IS NOT NULL; +``` + +--- + +## Liquid Clustering + +Use `CLUSTER BY` instead of legacy `PARTITION BY`. See **[5-performance.md](5-performance.md#liquid-clustering-recommended)** for detailed guidance on key selection by layer. + +```sql +CREATE OR REFRESH STREAMING TABLE bronze_events +CLUSTER BY (event_type, event_date) +AS SELECT ...; +``` + +--- + +## Table Properties + +```sql +CREATE OR REFRESH STREAMING TABLE bronze_events +TBLPROPERTIES ( + 'delta.autoOptimize.optimizeWrite' = 'true', -- Optimize file sizes on write + 'delta.autoOptimize.autoCompact' = 'true', -- Automatic compaction + 'delta.enableChangeDataFeed' = 'true', -- Enable CDF for downstream + 'delta.logRetentionDuration' = '7 days', -- Log retention + 'delta.deletedFileRetentionDuration' = '7 days' -- Deleted file retention +) +AS SELECT ...; +``` + +--- + +## Refresh Scheduling (Materialized Views) + +```sql +-- Near-real-time +CREATE OR REFRESH MATERIALIZED VIEW gold_live_metrics +REFRESH EVERY 5 MINUTES +AS SELECT ...; + +-- Daily +CREATE OR REFRESH MATERIALIZED VIEW gold_daily_summary +REFRESH EVERY 1 DAY +AS SELECT ...; +``` + +--- + +## Table Name Resolution + +| Level | Example | When to Use | +|-------|---------|-------------| +| Unqualified | `FROM bronze_orders` | Tables in same pipeline (recommended) | +| Schema-qualified | `FROM other_schema.orders` | Different schema, same catalog | +| Fully-qualified | `FROM other_catalog.schema.orders` | External catalogs | + +**Best practice:** Use unqualified names for pipeline-internal tables. + +### Multi-Schema Pattern (One Pipeline) + +Write to multiple schemas from a single pipeline using fully qualified names: + +```sql +-- bronze_orders.sql → writes to bronze schema +CREATE OR REFRESH STREAMING TABLE my_catalog.bronze.raw_orders +AS SELECT *, current_timestamp() AS _ingested_at +FROM STREAM read_files('/Volumes/my_catalog/raw/orders/', format => 'json'); + +-- silver_orders.sql → writes to silver schema, reads from bronze +CREATE OR REFRESH STREAMING TABLE my_catalog.silver.clean_orders +AS SELECT * FROM STREAM my_catalog.bronze.raw_orders +WHERE order_id IS NOT NULL; +``` + +--- + +## Pipeline Parameters + +Reference configuration values in SQL: + +```sql +-- In SQL, use ${variable_name} syntax +CREATE OR REFRESH STREAMING TABLE bronze_orders AS +SELECT * FROM STREAM read_files( + '${input_path}/orders/', + format => 'json' +); +``` + +Define in pipeline configuration (YAML): +```yaml +configuration: + input_path: /Volumes/my_catalog/my_schema/raw +``` + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| Missing `STREAM` keyword | Use `FROM STREAM table_name` for streaming tables | +| Constraint syntax error | Use `CONSTRAINT name EXPECT (condition)` | +| Cluster key not working | Verify column exists, limit to 4 keys | +| Parameter not resolved | Check `${var}` syntax and pipeline configuration | +| Using legacy `LIVE` keyword | Use `CREATE OR REFRESH STREAMING TABLE` \| `MATERIALIZED VIEW`, not `CREATE LIVE TABLE` \| `STREAMING LIVE TABLE` | +| Using `input_file_name()` | Use `_metadata.file_path` | diff --git a/.claude/skills/databricks-spark-declarative-pipelines/references/sql/2-ingestion.md b/.claude/skills/databricks-spark-declarative-pipelines/references/sql/2-ingestion.md new file mode 100644 index 0000000..61f98f6 --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/sql/2-ingestion.md @@ -0,0 +1,161 @@ +# SQL Data Ingestion + +Data ingestion patterns for cloud storage and streaming sources. + +**Official Documentation:** +- [read_files function reference](https://docs.databricks.com/aws/en/sql/language-manual/functions/read_files) +- [Auto Loader options](https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/options) + +--- + +## Auto Loader (Cloud Files) + +Auto Loader incrementally processes new files. Use `STREAM read_files()` in streaming table queries. + +### Basic Pattern + +```sql +CREATE OR REFRESH STREAMING TABLE bronze_orders AS +SELECT + *, + current_timestamp() AS _ingested_at, + _metadata.file_path AS _source_file +FROM STREAM read_files( + '/Volumes/my_catalog/my_schema/raw/orders/', + format => 'json', + schemaHints => 'order_id STRING, amount DECIMAL(10,2)' +); +``` + +**Key points:** +- Use `FROM STREAM read_files(...)` for streaming tables (not `FROM read_files(...)` which is batch) +- `format` supports: `json`, `csv`, `parquet`, `avro`, `text`, `binaryFile` +- `schemaHints` recommended for production to prevent schema drift +- `_metadata` provides file path, modification time, size + +### Schema Handling + +```sql +-- Explicit hints (recommended for production) +FROM STREAM read_files( + '/Volumes/catalog/schema/raw/', + format => 'json', + schemaHints => 'id STRING, amount DECIMAL(10,2), date DATE' +) + +-- Schema evolution with rescue data +FROM STREAM read_files( + '/Volumes/catalog/schema/raw/', + format => 'json', + schemaHints => 'id STRING', + mode => 'PERMISSIVE' +) +``` + +### Rescue Data (Quarantine Pattern) + +Handle malformed records: + +```sql +-- Flag records with parsing errors +CREATE OR REFRESH STREAMING TABLE bronze_events AS +SELECT + *, + current_timestamp() AS _ingested_at, + CASE WHEN _rescued_data IS NOT NULL THEN TRUE ELSE FALSE END AS _has_errors +FROM STREAM read_files('/Volumes/catalog/schema/raw/events/', format => 'json'); + +-- Quarantine bad records +CREATE OR REFRESH STREAMING TABLE bronze_quarantine AS +SELECT * FROM STREAM bronze_events WHERE _rescued_data IS NOT NULL; + +-- Clean records for downstream +CREATE OR REFRESH STREAMING TABLE silver_clean AS +SELECT * FROM STREAM bronze_events WHERE _rescued_data IS NULL; +``` + +--- + +## Streaming Sources + +### Kafka + +```sql +CREATE OR REFRESH STREAMING TABLE bronze_kafka_events AS +SELECT + CAST(key AS STRING) AS event_key, + CAST(value AS STRING) AS event_value, + topic, partition, offset, + timestamp AS kafka_timestamp, + current_timestamp() AS _ingested_at +FROM read_kafka( + bootstrapServers => '${kafka_brokers}', + subscribe => 'events-topic', + startingOffsets => 'latest' +); +``` + +**Documentation:** [read_kafka function](https://docs.databricks.com/aws/en/sql/language-manual/functions/read_kafka) + +### Parse JSON from Kafka + +```sql +CREATE OR REFRESH STREAMING TABLE silver_events AS +SELECT + from_json(event_value, 'event_id STRING, event_type STRING, timestamp TIMESTAMP') AS data, + kafka_timestamp, _ingested_at +FROM STREAM bronze_kafka_events; +``` + +--- + +## Authentication + +### Databricks Secrets + +```sql +-- Kafka +`kafka.sasl.jaas.config` => '...username="{{secrets/kafka/username}}" password="{{secrets/kafka/password}}";' + +-- Event Hub +`eventhubs.connectionString` => '{{secrets/eventhub/connection-string}}' +``` + +### Pipeline Variables + +```sql +-- Reference in SQL +FROM STREAM read_files('${input_path}/orders/', format => 'json') +``` + +Define in pipeline configuration: +```yaml +configuration: + input_path: /Volumes/my_catalog/my_schema/raw +``` + +--- + +## Best Practices + +1. **Always add ingestion metadata:** +```sql +SELECT *, current_timestamp() AS _ingested_at, _metadata.file_path AS _source_file +``` + +2. **Use schemaHints for production** - prevents unexpected schema changes + +3. **Handle rescue data** - route malformed records to quarantine table + +4. **Use STREAM keyword** - `FROM STREAM read_files(...)` for streaming tables + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| Files not picked up | Verify path and format match actual files | +| "Cannot create streaming table from batch query" | Use `FROM STREAM read_files(...)` not `FROM read_files(...)` | +| Schema evolution breaking | Use `mode => 'PERMISSIVE'` and monitor `_rescued_data` | +| Kafka lag increasing | Check downstream bottlenecks | diff --git a/.claude/skills/databricks-spark-declarative-pipelines/2-streaming-patterns.md b/.claude/skills/databricks-spark-declarative-pipelines/references/sql/3-streaming-patterns.md similarity index 70% rename from .claude/skills/databricks-spark-declarative-pipelines/2-streaming-patterns.md rename to .claude/skills/databricks-spark-declarative-pipelines/references/sql/3-streaming-patterns.md index c1ec63b..fc42702 100644 --- a/.claude/skills/databricks-spark-declarative-pipelines/2-streaming-patterns.md +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/sql/3-streaming-patterns.md @@ -1,4 +1,4 @@ -# Streaming Patterns for SDP +# SQL Streaming Patterns Streaming-specific patterns including deduplication, windowed aggregations, late-arriving data handling, and stateful operations. @@ -10,12 +10,12 @@ Streaming-specific patterns including deduplication, windowed aggregations, late ```sql -- Bronze: Ingest all (may contain duplicates) -CREATE OR REPLACE STREAMING TABLE bronze_events AS +CREATE OR REFRESH STREAMING TABLE bronze_events AS SELECT *, current_timestamp() AS _ingested_at -FROM read_stream(...); +FROM STREAM read_files(...); -- Silver: Deduplicate by event_id -CREATE OR REPLACE STREAMING TABLE silver_events_dedup AS +CREATE OR REFRESH STREAMING TABLE silver_events_dedup AS SELECT event_id, user_id, event_type, event_timestamp, _ingested_at FROM ( @@ -32,21 +32,21 @@ WHERE rn = 1; Deduplicate within time window to handle late arrivals: ```sql -CREATE OR REPLACE STREAMING TABLE silver_events_dedup AS +CREATE OR REFRESH STREAMING TABLE silver_events_dedup AS SELECT event_id, user_id, event_type, event_timestamp, MIN(_ingested_at) AS first_seen_at FROM STREAM bronze_events GROUP BY event_id, user_id, event_type, event_timestamp, - window(event_timestamp, '1 hour') -- Deduplicate within 1-hour windows + window(event_timestamp, '1 hour') HAVING COUNT(*) >= 1; ``` ### Composite Key ```sql -CREATE OR REPLACE STREAMING TABLE silver_transactions_dedup AS +CREATE OR REFRESH STREAMING TABLE silver_transactions_dedup AS SELECT transaction_id, customer_id, amount, transaction_timestamp, MIN(_ingested_at) AS _ingested_at @@ -60,9 +60,11 @@ GROUP BY transaction_id, customer_id, amount, transaction_timestamp; ### Tumbling Windows +Non-overlapping fixed-size windows: + ```sql --- 5-minute non-overlapping windows -CREATE OR REPLACE STREAMING TABLE silver_sensor_5min AS +-- 5-minute windows +CREATE OR REFRESH STREAMING TABLE silver_sensor_5min AS SELECT sensor_id, window(event_timestamp, '5 minutes') AS time_window, @@ -78,7 +80,7 @@ GROUP BY sensor_id, window(event_timestamp, '5 minutes'); ```sql -- 1-minute for real-time monitoring -CREATE OR REPLACE STREAMING TABLE gold_sensor_1min AS +CREATE OR REFRESH STREAMING TABLE gold_sensor_1min AS SELECT sensor_id, window(event_timestamp, '1 minute').start AS window_start, @@ -89,7 +91,7 @@ FROM STREAM silver_sensor_data GROUP BY sensor_id, window(event_timestamp, '1 minute'); -- 1-hour for trend analysis -CREATE OR REPLACE STREAMING TABLE gold_sensor_1hour AS +CREATE OR REFRESH STREAMING TABLE gold_sensor_1hour AS SELECT sensor_id, window(event_timestamp, '1 hour').start AS window_start, @@ -99,25 +101,35 @@ FROM STREAM silver_sensor_data GROUP BY sensor_id, window(event_timestamp, '1 hour'); ``` +### Session Windows + +Group events into sessions based on inactivity gaps: + +```sql +-- 30-minute inactivity timeout +CREATE OR REFRESH STREAMING TABLE silver_user_sessions AS +SELECT + user_id, + session_window(event_timestamp, '30 minutes') AS session, + MIN(event_timestamp) AS session_start, + MAX(event_timestamp) AS session_end, + COUNT(*) AS event_count, + COLLECT_LIST(event_type) AS event_sequence +FROM STREAM bronze_user_events +GROUP BY user_id, session_window(event_timestamp, '30 minutes'); +``` + --- ## Late-Arriving Data ### Event-Time vs Processing-Time -Always use event timestamp for business logic, not ingestion timestamp: +Always use event timestamp for business logic: ```sql --- ✅ Use event timestamp -CREATE OR REPLACE STREAMING TABLE silver_orders AS -SELECT - order_id, order_timestamp, -- Event time from source - customer_id, amount, - _ingested_at -- Processing time (debugging only) -FROM STREAM bronze_orders; - --- Group by event time -CREATE OR REPLACE STREAMING TABLE gold_daily_orders AS +-- Use event timestamp for aggregations +CREATE OR REFRESH STREAMING TABLE gold_daily_orders AS SELECT CAST(order_timestamp AS DATE) AS order_date, -- Event time COUNT(*) AS order_count, @@ -126,32 +138,23 @@ FROM STREAM silver_orders GROUP BY CAST(order_timestamp AS DATE); ``` -### Handling Out-of-Order with SCD2 - -Use SEQUENCE BY with event timestamp. **Clause order matters**: put `APPLY AS DELETE WHEN` before `SEQUENCE BY`. Only list columns in `COLUMNS * EXCEPT (...)` that actually exist in the source (omit `_rescued_data` unless the bronze table uses rescue data). Omit `TRACK HISTORY ON *` if it causes parse errors; the default is equivalent. - +**Keep processing time for debugging:** ```sql -CREATE OR REFRESH STREAMING TABLE silver_customers_history; - -CREATE FLOW customers_scd2_flow AS -AUTO CDC INTO silver_customers_history -FROM stream(bronze_customer_cdc) -KEYS (customer_id) -APPLY AS DELETE WHEN operation = "DELETE" -SEQUENCE BY event_timestamp -- Handles out-of-order -COLUMNS * EXCEPT (operation, _ingested_at, _source_file) -STORED AS SCD TYPE 2; +SELECT + order_id, order_timestamp, -- Event time (business logic) + customer_id, amount, + _ingested_at -- Processing time (debugging only) +FROM STREAM bronze_orders; ``` --- -## Stateful Operations +## Joins ### Stream-to-Stream Joins ```sql --- Join two streaming sources -CREATE OR REPLACE STREAMING TABLE silver_orders_with_payments AS +CREATE OR REFRESH STREAMING TABLE silver_orders_with_payments AS SELECT o.order_id, o.customer_id, o.order_timestamp, o.amount AS order_amount, p.payment_id, p.payment_timestamp, p.payment_method, p.amount AS payment_amount @@ -161,17 +164,19 @@ INNER JOIN STREAM bronze_payments p AND p.payment_timestamp BETWEEN o.order_timestamp AND o.order_timestamp + INTERVAL 1 HOUR; ``` +**Important:** Use time bounds in join condition to limit state retention. + ### Stream-to-Static Joins Enrich streaming data with dimension tables: ```sql --- Static dimension (changes infrequently) +-- Static dimension CREATE OR REPLACE TABLE dim_products AS SELECT * FROM catalog.schema.products; -- Stream-to-static join -CREATE OR REPLACE STREAMING TABLE silver_sales_enriched AS +CREATE OR REFRESH STREAMING TABLE silver_sales_enriched AS SELECT s.sale_id, s.product_id, s.quantity, s.sale_timestamp, p.product_name, p.category, p.price, @@ -180,11 +185,14 @@ FROM STREAM bronze_sales s LEFT JOIN dim_products p ON s.product_id = p.product_id; ``` -### Incremental Aggregations +--- + +## Incremental Aggregations + +### Running Totals ```sql --- Running totals by customer (stateful) -CREATE OR REPLACE STREAMING TABLE silver_customer_running_totals AS +CREATE OR REFRESH STREAMING TABLE silver_customer_running_totals AS SELECT customer_id, SUM(amount) AS total_spent, @@ -196,32 +204,12 @@ GROUP BY customer_id; --- -## Session Windows - -Group events into sessions based on inactivity gaps: - -```sql --- 30-minute inactivity timeout -CREATE OR REPLACE STREAMING TABLE silver_user_sessions AS -SELECT - user_id, - session_window(event_timestamp, '30 minutes') AS session, - MIN(event_timestamp) AS session_start, - MAX(event_timestamp) AS session_end, - COUNT(*) AS event_count, - COLLECT_LIST(event_type) AS event_sequence -FROM STREAM bronze_user_events -GROUP BY user_id, session_window(event_timestamp, '30 minutes'); -``` - ---- - ## Anomaly Detection ### Real-Time Outlier Detection ```sql -CREATE OR REPLACE STREAMING TABLE silver_sensor_with_anomalies AS +CREATE OR REFRESH STREAMING TABLE silver_sensor_with_anomalies AS SELECT sensor_id, event_timestamp, temperature, AVG(temperature) OVER ( @@ -240,7 +228,7 @@ SELECT FROM STREAM bronze_sensor_events; -- Route anomalies for alerting -CREATE OR REPLACE STREAMING TABLE silver_sensor_anomalies AS +CREATE OR REFRESH STREAMING TABLE silver_sensor_anomalies AS SELECT * FROM STREAM silver_sensor_with_anomalies WHERE anomaly_flag IN ('HIGH_OUTLIER', 'LOW_OUTLIER'); @@ -249,7 +237,7 @@ WHERE anomaly_flag IN ('HIGH_OUTLIER', 'LOW_OUTLIER'); ### Threshold-Based Filtering ```sql -CREATE OR REPLACE STREAMING TABLE silver_high_value_transactions AS +CREATE OR REFRESH STREAMING TABLE silver_high_value_transactions AS SELECT transaction_id, customer_id, amount, transaction_timestamp FROM STREAM bronze_transactions WHERE amount > 10000; @@ -257,38 +245,51 @@ WHERE amount > 10000; --- +## Monitoring Lag + +```sql +CREATE OR REFRESH STREAMING TABLE monitoring_lag AS +SELECT + 'kafka_events' AS source, + MAX(kafka_timestamp) AS max_event_timestamp, + current_timestamp() AS processing_timestamp, + (unix_timestamp(current_timestamp()) - unix_timestamp(MAX(kafka_timestamp))) AS lag_seconds +FROM STREAM bronze_kafka_events +GROUP BY window(kafka_timestamp, '1 minute'); +``` + +--- + ## Execution Modes Configure at pipeline level (not in SQL): -**Continuous** (real-time, sub-second latency): ```yaml +# Continuous (real-time, sub-second latency) execution_mode: continuous serverless: true -``` -**Triggered** (scheduled, cost-optimized): -```yaml +# Triggered (scheduled, cost-optimized) execution_mode: triggered schedule: "0 * * * *" # Hourly ``` -**When to use**: +**When to use:** - **Continuous**: Real-time dashboards, alerting, sub-minute SLAs - **Triggered**: Daily/hourly reports, batch processing --- -## Key Patterns +## Best Practices ### 1. Use Event Timestamps ```sql --- ✅ Event timestamp for logic +-- Correct: Event timestamp for logic GROUP BY date_trunc('hour', event_timestamp) --- ❌ Processing timestamp -GROUP BY date_trunc('hour', _ingested_at) +-- Avoid: Processing timestamp +-- GROUP BY date_trunc('hour', _ingested_at) ``` ### 2. Window Size Selection @@ -302,10 +303,10 @@ GROUP BY date_trunc('hour', _ingested_at) Higher cardinality = more state: ```sql --- High state: 1M users × 10K products × 100M sessions +-- High state: 1M users x 10K products x 100M sessions GROUP BY user_id, product_id, session_id --- Lower state: 1M users × 100 categories × days +-- Lower state: 1M users x 100 categories x days GROUP BY user_id, product_category, DATE(event_time) ``` @@ -317,32 +318,19 @@ Apply at bronze → silver transition: ```sql -- Bronze: Accept duplicates -CREATE OR REPLACE STREAMING TABLE bronze_events AS -SELECT * FROM read_stream(...); +CREATE OR REFRESH STREAMING TABLE bronze_events AS +SELECT * FROM STREAM read_files(...); -- Silver: Deduplicate immediately -CREATE OR REPLACE STREAMING TABLE silver_events AS +CREATE OR REFRESH STREAMING TABLE silver_events AS SELECT DISTINCT event_id, event_type, event_timestamp, user_id FROM STREAM bronze_events; -- Gold: Work with clean data -CREATE OR REPLACE STREAMING TABLE gold_metrics AS +CREATE OR REFRESH STREAMING TABLE gold_metrics AS SELECT ... FROM STREAM silver_events; ``` -### 5. Monitor Lag - -```sql -CREATE OR REPLACE STREAMING TABLE monitoring_lag AS -SELECT - 'kafka_events' AS source, - MAX(kafka_timestamp) AS max_event_timestamp, - current_timestamp() AS processing_timestamp, - (unix_timestamp(current_timestamp()) - unix_timestamp(MAX(kafka_timestamp))) AS lag_seconds -FROM STREAM bronze_kafka_events -GROUP BY window(kafka_timestamp, '1 minute'); -``` - --- ## Common Issues diff --git a/.claude/skills/databricks-spark-declarative-pipelines/references/sql/4-cdc-patterns.md b/.claude/skills/databricks-spark-declarative-pipelines/references/sql/4-cdc-patterns.md new file mode 100644 index 0000000..d9977c2 --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/sql/4-cdc-patterns.md @@ -0,0 +1,323 @@ +# SQL CDC Patterns (AUTO CDC & SCD) + +Change Data Capture patterns using AUTO CDC for SCD Type 1 and Type 2, plus querying SCD history tables. + +--- + +## Overview + +AUTO CDC automatically handles Change Data Capture to track changes using Slow Changing Dimensions (SCD). It provides automatic deduplication, change tracking, and handles late-arriving data correctly. + +**Where to apply AUTO CDC:** +- **Silver layer**: When business users need deduplicated or historical data +- **Gold layer**: When implementing dimensional modeling (star schema) + +--- + +## SCD Type 1 vs Type 2 + +### SCD Type 1 (In-place updates) +- **Overwrites** old values with new values +- **No history preserved** - only current state +- **Use for**: Error corrections, attributes where history doesn't matter +- **Syntax**: `STORED AS SCD TYPE 1` + +### SCD Type 2 (History tracking) +- **Creates new row** for each change +- **Preserves full history** with `__START_AT` and `__END_AT` timestamps +- **Use for**: Tracking changes over time (addresses, prices, roles) +- **Syntax**: `STORED AS SCD TYPE 2` + +--- + +## Creating AUTO CDC Flows + +### SCD Type 2 + +```sql +-- Step 1: Create target table +CREATE OR REFRESH STREAMING TABLE dim_customers; + +-- Step 2: Create AUTO CDC flow +CREATE FLOW customers_scd2_flow AS +AUTO CDC INTO dim_customers +FROM stream(customers_cdc_clean) +KEYS (customer_id) +APPLY AS DELETE WHEN operation = "DELETE" +SEQUENCE BY event_timestamp +COLUMNS * EXCEPT (operation, _ingested_at, _source_file) +STORED AS SCD TYPE 2; +``` + +**Important:** Put `APPLY AS DELETE WHEN` before `SEQUENCE BY`. Only list columns in `COLUMNS * EXCEPT (...)` that exist in the source. + +### SCD Type 1 + +```sql +-- Step 1: Create target table +CREATE OR REFRESH STREAMING TABLE orders_current; + +-- Step 2: Create AUTO CDC flow +CREATE FLOW orders_scd1_flow AS +AUTO CDC INTO orders_current +FROM stream(orders_clean) +KEYS (order_id) +SEQUENCE BY updated_timestamp +COLUMNS * EXCEPT (_ingested_at) +STORED AS SCD TYPE 1; +``` + +### Selective History Tracking + +Track history only when specific columns change: + +```sql +CREATE FLOW products_scd2_flow AS +AUTO CDC INTO products_history +FROM stream(products_clean) +KEYS (product_id) +SEQUENCE BY modified_at +COLUMNS * EXCEPT (operation) +STORED AS SCD TYPE 2 +TRACK HISTORY ON price, cost; +``` + +When `price` or `cost` changes, a new version is created. Other column changes update the current record without new versions. + +--- + +## Complete Pattern: Clean + AUTO CDC + +### Step 1: Clean and Validate Source Data + +```sql +CREATE OR REFRESH STREAMING TABLE customers_cdc_clean AS +SELECT + customer_id, + customer_name, + email, + phone, + address, + CAST(updated_at AS TIMESTAMP) AS event_timestamp, + operation +FROM STREAM bronze_customers_cdc +WHERE customer_id IS NOT NULL + AND email IS NOT NULL; +``` + +### Step 2: Apply AUTO CDC + +```sql +CREATE OR REFRESH STREAMING TABLE dim_customers; + +CREATE FLOW customers_scd2_flow AS +AUTO CDC INTO dim_customers +FROM stream(customers_cdc_clean) +KEYS (customer_id) +APPLY AS DELETE WHEN operation = "DELETE" +SEQUENCE BY event_timestamp +COLUMNS * EXCEPT (operation) +STORED AS SCD TYPE 2; +``` + +--- + +## Querying SCD Type 2 Tables + +SCD Type 2 tables include temporal columns: +- `__START_AT` - When this version became effective +- `__END_AT` - When this version expired (NULL for current) + +### Current State + +```sql +-- All current records +CREATE OR REFRESH MATERIALIZED VIEW dim_customers_current AS +SELECT + customer_id, customer_name, email, phone, address, + __START_AT AS valid_from +FROM dim_customers +WHERE __END_AT IS NULL; + +-- Specific customer +SELECT * +FROM dim_customers +WHERE customer_id = '12345' + AND __END_AT IS NULL; +``` + +### Point-in-Time Queries + +Get state as of a specific date: + +```sql +-- Products as of January 1, 2024 +CREATE OR REFRESH MATERIALIZED VIEW products_as_of_2024_01_01 AS +SELECT + product_id, product_name, price, category, + __START_AT, __END_AT +FROM products_history +WHERE __START_AT <= '2024-01-01' + AND (__END_AT > '2024-01-01' OR __END_AT IS NULL); +``` + +### Change Analysis + +Track all changes for an entity: + +```sql +SELECT + customer_id, customer_name, email, phone, + __START_AT, __END_AT, + COALESCE( + DATEDIFF(DAY, __START_AT, __END_AT), + DATEDIFF(DAY, __START_AT, CURRENT_TIMESTAMP()) + ) AS days_active +FROM dim_customers +WHERE customer_id = '12345' +ORDER BY __START_AT DESC; +``` + +Changes within a time period: + +```sql +-- Customers who changed during Q1 2024 +SELECT + customer_id, customer_name, + __START_AT AS change_timestamp, + 'UPDATE' AS change_type +FROM dim_customers +WHERE __START_AT BETWEEN '2024-01-01' AND '2024-03-31' + AND __START_AT != ( + SELECT MIN(__START_AT) + FROM dim_customers ch2 + WHERE ch2.customer_id = dim_customers.customer_id + ) +ORDER BY __START_AT; +``` + +--- + +## Joining Facts with Historical Dimensions + +### At Transaction Time + +```sql +-- Join sales with product prices at time of sale +CREATE OR REFRESH MATERIALIZED VIEW sales_with_historical_prices AS +SELECT + s.sale_id, s.product_id, s.sale_date, s.quantity, + p.product_name, p.price AS unit_price_at_sale_time, + s.quantity * p.price AS calculated_amount, + p.category +FROM sales_fact s +INNER JOIN products_history p + ON s.product_id = p.product_id + AND s.sale_date >= p.__START_AT + AND (s.sale_date < p.__END_AT OR p.__END_AT IS NULL); +``` + +### With Current Dimension + +```sql +CREATE OR REFRESH MATERIALIZED VIEW sales_with_current_prices AS +SELECT + s.sale_id, s.product_id, s.sale_date, s.quantity, + s.amount AS amount_at_sale, + p.product_name AS current_product_name, + p.price AS current_price +FROM sales_fact s +INNER JOIN products_history p + ON s.product_id = p.product_id + AND p.__END_AT IS NULL; +``` + +--- + +## Optimization Patterns + +### Pre-Filter Materialized Views + +```sql +-- Current state view (most common pattern) +CREATE OR REFRESH MATERIALIZED VIEW dim_products_current AS +SELECT * FROM products_history WHERE __END_AT IS NULL; + +-- Recent changes only +CREATE OR REFRESH MATERIALIZED VIEW dim_recent_changes AS +SELECT * FROM products_history +WHERE __START_AT >= CURRENT_DATE() - INTERVAL 90 DAYS; + +-- Change frequency stats +CREATE OR REFRESH MATERIALIZED VIEW product_change_stats AS +SELECT + product_id, + COUNT(*) AS version_count, + MIN(__START_AT) AS first_seen, + MAX(__START_AT) AS last_updated +FROM products_history +GROUP BY product_id; +``` + +--- + +## Best Practices + +### 1. Filter by __END_AT for Current + +```sql +-- Efficient +WHERE __END_AT IS NULL + +-- Less efficient +WHERE __START_AT = (SELECT MAX(__START_AT) FROM table WHERE ...) +``` + +### 2. Use Inclusive Lower, Exclusive Upper + +```sql +WHERE __START_AT <= '2024-01-01' + AND (__END_AT > '2024-01-01' OR __END_AT IS NULL) +``` + +### 3. Clean Data Before AUTO CDC + +Apply type casting, validation, and filtering first: + +```sql +-- Clean source +CREATE OR REFRESH STREAMING TABLE users_clean AS +SELECT + user_id, + TRIM(email) AS email, + CAST(updated_at AS TIMESTAMP) AS updated_timestamp +FROM STREAM bronze_users +WHERE user_id IS NOT NULL AND email IS NOT NULL; + +-- Then apply AUTO CDC +CREATE FLOW users_scd2_flow AS +AUTO CDC INTO dim_users +FROM stream(users_clean) +KEYS (user_id) +SEQUENCE BY updated_timestamp +STORED AS SCD TYPE 2; +``` + +### 4. Choose the Right SCD Type + +- **Type 2**: Need to query historical states +- **Type 1**: Only need current state or deduplication + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| Multiple rows for same key | Missing `__END_AT IS NULL` filter for current state | +| Point-in-time no results | Use `__START_AT <= date AND (__END_AT > date OR __END_AT IS NULL)` | +| Slow temporal join | Create materialized view for specific time period | +| Unexpected duplicates | Multiple changes same day - use SEQUENCE BY with high precision | +| Parse error on AUTO CDC | Put `APPLY AS DELETE WHEN` before `SEQUENCE BY` | +| Columns not in target | Only list existing columns in `COLUMNS * EXCEPT (...)` | +| Type syntax error | Use `SCD TYPE 1` or `SCD TYPE 2` (not quoted) | diff --git a/.claude/skills/databricks-spark-declarative-pipelines/4-performance-tuning.md b/.claude/skills/databricks-spark-declarative-pipelines/references/sql/5-performance.md similarity index 72% rename from .claude/skills/databricks-spark-declarative-pipelines/4-performance-tuning.md rename to .claude/skills/databricks-spark-declarative-pipelines/references/sql/5-performance.md index bd1c1dc..aa9ffaf 100644 --- a/.claude/skills/databricks-spark-declarative-pipelines/4-performance-tuning.md +++ b/.claude/skills/databricks-spark-declarative-pipelines/references/sql/5-performance.md @@ -1,14 +1,14 @@ -# Performance Tuning for SDP +# SQL Performance Tuning -Performance optimization strategies including **Liquid Clustering** (modern approach), materialized view refresh, state management, and compute configuration. +Performance optimization strategies including Liquid Clustering, materialized view refresh, state management, and compute configuration. --- ## Liquid Clustering (Recommended) -**Liquid Clustering** is the recommended approach for data layout optimization. It replaces manual `PARTITION BY` and `Z-ORDER`. +Liquid Clustering is the recommended approach for data layout optimization. It replaces manual `PARTITION BY` and `Z-ORDER`. -### What is Liquid Clustering? +### Benefits - **Adaptive**: Adjusts to data distribution changes - **Multi-dimensional**: Clusters on multiple columns simultaneously @@ -17,32 +17,22 @@ Performance optimization strategies including **Liquid Clustering** (modern appr ### Basic Syntax -**SQL**: ```sql -CREATE OR REPLACE STREAMING TABLE bronze_events +CREATE OR REFRESH STREAMING TABLE bronze_events CLUSTER BY (event_type, event_date) AS SELECT *, current_timestamp() AS _ingested_at, CAST(current_date() AS DATE) AS event_date -FROM read_files('/mnt/raw/events/', format => 'json'); +FROM STREAM read_files('/Volumes/my_catalog/my_schema/raw/events/', format => 'json'); ``` -**Python**: -```python -from pyspark import pipelines as dp - -@dp.table(cluster_by=["event_type", "event_date"]) -def bronze_events(): - return spark.readStream.format("cloudFiles").load("/data") -``` - -### Automatic Cluster Key Selection +### Automatic Key Selection ```sql -- Let Databricks choose based on query patterns -CREATE OR REPLACE STREAMING TABLE bronze_events +CREATE OR REFRESH STREAMING TABLE bronze_events CLUSTER BY (AUTO) AS SELECT ...; ``` @@ -59,7 +49,7 @@ AS SELECT ...; Cluster by event type + date: ```sql -CREATE OR REPLACE STREAMING TABLE bronze_events +CREATE OR REFRESH STREAMING TABLE bronze_events CLUSTER BY (event_type, ingestion_date) TBLPROPERTIES ('delta.autoOptimize.optimizeWrite' = 'true') AS @@ -67,7 +57,7 @@ SELECT *, current_timestamp() AS _ingested_at, CAST(current_date() AS DATE) AS ingestion_date -FROM read_files('/mnt/raw/events/', format => 'json'); +FROM STREAM read_files('/Volumes/my_catalog/my_schema/raw/events/', format => 'json'); ``` **Why**: Bronze filtered by event type for processing and by date for incremental loads. @@ -77,11 +67,12 @@ FROM read_files('/mnt/raw/events/', format => 'json'); Cluster by primary key + business dimension: ```sql -CREATE OR REPLACE STREAMING TABLE silver_orders +CREATE OR REFRESH STREAMING TABLE silver_orders CLUSTER BY (customer_id, order_date) AS SELECT - order_id, customer_id, product_id, amount, + order_id, customer_id, product_id, + CAST(amount AS DECIMAL(10,2)) AS amount, -- DECIMAL for monetary values CAST(order_timestamp AS DATE) AS order_date, order_timestamp FROM STREAM bronze_orders; @@ -94,7 +85,7 @@ FROM STREAM bronze_orders; Cluster by aggregation dimensions: ```sql -CREATE OR REPLACE MATERIALIZED VIEW gold_sales_summary +CREATE OR REFRESH MATERIALIZED VIEW gold_sales_summary CLUSTER BY (product_category, year_month) AS SELECT @@ -117,7 +108,7 @@ GROUP BY product_category, DATE_FORMAT(order_date, 'yyyy-MM'); | **Silver** | primary_key, business_date | Entity lookups + time ranges | | **Gold** | aggregation_dimensions | Dashboard filters | -**Best practices**: +**Best practices:** - First key: Most selective filter (e.g., customer_id) - Second key: Next common filter (e.g., date) - Order matters: Most selective first @@ -131,7 +122,7 @@ GROUP BY product_category, DATE_FORMAT(order_date, 'yyyy-MM'); ### Before (Legacy) ```sql -CREATE OR REPLACE STREAMING TABLE events +CREATE OR REFRESH STREAMING TABLE events PARTITIONED BY (date DATE) TBLPROPERTIES ('pipelines.autoOptimize.zOrderCols' = 'user_id,event_type') AS SELECT ...; @@ -139,10 +130,10 @@ AS SELECT ...; **Issues**: Fixed keys, small file problem, skewed distribution, manual OPTIMIZE required. -### After (Modern with Liquid Clustering) +### After (Modern) ```sql -CREATE OR REPLACE STREAMING TABLE events +CREATE OR REFRESH STREAMING TABLE events CLUSTER BY (date, user_id, event_type) AS SELECT ...; ``` @@ -157,8 +148,6 @@ AS SELECT ...; 3. **Compatibility**: Older Delta Lake versions (< DBR 13.3) 4. **Existing large tables**: Migration cost outweighs benefits -**Otherwise, prefer Liquid Clustering.** - --- ## Table Properties @@ -166,20 +155,18 @@ AS SELECT ...; ### Auto-Optimize ```sql -CREATE OR REPLACE STREAMING TABLE bronze_events +CREATE OR REFRESH STREAMING TABLE bronze_events TBLPROPERTIES ( 'delta.autoOptimize.optimizeWrite' = 'true', 'delta.autoOptimize.autoCompact' = 'true' ) -AS SELECT * FROM read_files(...); +AS SELECT * FROM STREAM read_files(...); ``` -**Benefits**: Reduces small files, improves reads, automatic compaction. - ### Change Data Feed ```sql -CREATE OR REPLACE STREAMING TABLE silver_customers +CREATE OR REFRESH STREAMING TABLE silver_customers TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true') AS SELECT * FROM STREAM bronze_customers; ``` @@ -189,12 +176,12 @@ AS SELECT * FROM STREAM bronze_customers; ### Retention Periods ```sql -CREATE OR REPLACE STREAMING TABLE bronze_high_volume +CREATE OR REFRESH STREAMING TABLE bronze_high_volume TBLPROPERTIES ( 'delta.logRetentionDuration' = '7 days', 'delta.deletedFileRetentionDuration' = '7 days' ) -AS SELECT * FROM read_files(...); +AS SELECT * FROM STREAM read_files(...); ``` **Use for**: High-volume tables to reduce storage costs. @@ -206,8 +193,8 @@ AS SELECT * FROM read_files(...); ### Refresh Frequency ```sql --- Near-real-time (frequent) -CREATE OR REPLACE MATERIALIZED VIEW gold_live_metrics +-- Near-real-time +CREATE OR REFRESH MATERIALIZED VIEW gold_live_metrics REFRESH EVERY 5 MINUTES AS SELECT @@ -217,8 +204,8 @@ SELECT FROM silver_metrics GROUP BY metric_name; --- Daily reports (scheduled) -CREATE OR REPLACE MATERIALIZED VIEW gold_daily_summary +-- Daily reports +CREATE OR REFRESH MATERIALIZED VIEW gold_daily_summary REFRESH EVERY 1 DAY AS SELECT report_date, SUM(amount) AS total_amount @@ -226,13 +213,12 @@ FROM silver_sales GROUP BY report_date; ``` -### Incremental Refresh (Automatic) +### Incremental Refresh Materialized views auto-use incremental refresh when possible: ```sql --- Refreshes incrementally if source has row tracking -CREATE OR REPLACE MATERIALIZED VIEW gold_aggregates AS +CREATE OR REFRESH MATERIALIZED VIEW gold_aggregates AS SELECT product_id, SUM(quantity) AS total_quantity, @@ -246,8 +232,8 @@ GROUP BY product_id; ### Pre-Aggregation ```sql --- Instead of querying large table repeatedly -CREATE OR REPLACE MATERIALIZED VIEW orders_monthly AS +-- Create pre-aggregated MV for fast queries +CREATE OR REFRESH MATERIALIZED VIEW orders_monthly AS SELECT customer_id, YEAR(order_date) AS year, @@ -282,7 +268,6 @@ GROUP BY user_id, product_id, session_id; -- Massive state! **Strategy 1: Reduce cardinality** ```sql --- Aggregate at higher level SELECT user_id, product_category, -- 100 categories (not 10K products) @@ -295,7 +280,6 @@ GROUP BY user_id, product_category, DATE(event_time); **Strategy 2: Use time windows** ```sql --- Bounded state with windows SELECT user_id, window(event_time, '1 hour') AS time_window, @@ -308,7 +292,7 @@ GROUP BY user_id, window(event_time, '1 hour'); ```sql -- Streaming aggregation (maintains state) -CREATE OR REPLACE STREAMING TABLE user_daily_stats AS +CREATE OR REFRESH STREAMING TABLE user_daily_stats AS SELECT user_id, DATE(event_time) AS event_date, @@ -317,7 +301,7 @@ FROM STREAM bronze_events GROUP BY user_id, DATE(event_time); -- Batch aggregation (no streaming state) -CREATE OR REPLACE MATERIALIZED VIEW user_monthly_stats AS +CREATE OR REFRESH MATERIALIZED VIEW user_monthly_stats AS SELECT user_id, DATE_TRUNC('month', event_date) AS month, @@ -334,10 +318,10 @@ GROUP BY user_id, DATE_TRUNC('month', event_date); ```sql -- Small static dimension, large streaming fact -CREATE OR REPLACE STREAMING TABLE sales_enriched AS +CREATE OR REFRESH STREAMING TABLE sales_enriched AS SELECT s.sale_id, s.product_id, s.amount, - p.product_name, p.category -- From small static table + p.product_name, p.category FROM STREAM bronze_sales s LEFT JOIN dim_products p ON s.product_id = p.product_id; ``` @@ -348,7 +332,7 @@ LEFT JOIN dim_products p ON s.product_id = p.product_id; ```sql -- Time bounds limit state retention -CREATE OR REPLACE STREAMING TABLE orders_with_payments AS +CREATE OR REFRESH STREAMING TABLE orders_with_payments AS SELECT o.order_id, o.amount AS order_amount, p.payment_id, p.amount AS payment_amount @@ -358,32 +342,6 @@ INNER JOIN STREAM bronze_payments p AND p.payment_time BETWEEN o.order_time AND o.order_time + INTERVAL 1 HOUR; ``` -**Optimization**: Use time bounds in join condition. - ---- - -## Compute Configuration - -### Serverless vs Classic - -| Aspect | Serverless | Classic | -|--------|-----------|---------| -| Startup | Fast (seconds) | Slower (minutes) | -| Scaling | Automatic, instant | Manual/autoscaling | -| Cost | Pay-per-use | Pay for cluster time | -| Best for | Variable workloads, dev/test | Steady workloads | - -### Serverless (Recommended) - -Enable at pipeline level: - -```yaml -execution_mode: continuous # or triggered -serverless: true -``` - -**Advantages**: No cluster management, instant scaling, lower cost for bursty workloads. - --- ## Query Optimization @@ -391,47 +349,53 @@ serverless: true ### Filter Early ```sql --- ✅ Filter at source -CREATE OR REPLACE STREAMING TABLE silver_recent AS +-- Filter at source +CREATE OR REFRESH STREAMING TABLE silver_recent AS SELECT * FROM STREAM bronze_events WHERE event_date >= CURRENT_DATE() - INTERVAL 7 DAYS; --- ❌ Filter late -CREATE OR REPLACE STREAMING TABLE silver_all AS -SELECT * FROM STREAM bronze_events; - -CREATE OR REPLACE MATERIALIZED VIEW gold_recent AS -SELECT * FROM silver_all -WHERE event_date >= CURRENT_DATE() - INTERVAL 7 DAYS; +-- Avoid filtering late +-- CREATE OR REFRESH STREAMING TABLE silver_all AS SELECT * FROM STREAM bronze_events; +-- CREATE OR REFRESH MATERIALIZED VIEW gold_recent AS SELECT * FROM silver_all WHERE ...; ``` ### Select Specific Columns ```sql --- ❌ Reads all columns -SELECT * FROM large_table; - --- ✅ Only needed columns +-- Only needed columns SELECT customer_id, order_date, amount FROM large_table; + +-- Avoid SELECT * +-- SELECT * FROM large_table; ``` -### Use GROUP BY Over DISTINCT +--- -```sql --- ❌ Expensive on high-cardinality -SELECT DISTINCT transaction_id FROM huge_table; +## Compute Configuration + +### Serverless vs Classic + +| Aspect | Serverless | Classic | +|--------|-----------|---------| +| Startup | Fast (seconds) | Slower (minutes) | +| Scaling | Automatic, instant | Manual/autoscaling | +| Cost | Pay-per-use | Pay for cluster time | +| Best for | Variable workloads, dev/test | Steady workloads | + +### Serverless (Recommended) --- ✅ Better -SELECT transaction_id, COUNT(*) FROM huge_table GROUP BY transaction_id; +Enable at pipeline level: + +```yaml +execution_mode: continuous # or triggered +serverless: true ``` --- ## Monitoring -Track key metrics: - ```sql -- Data freshness SELECT @@ -455,7 +419,7 @@ GROUP BY table_name; | Issue | Solution | |-------|----------| -| Pipeline running slowly | Check partitioning, state size, join patterns | +| Pipeline running slowly | Check clustering, state size, join patterns | | High memory usage | Unbounded state - add time windows, reduce cardinality | | Many small files | Enable auto-optimize, run OPTIMIZE command | | Expensive queries on large tables | Add clustering, create filtered MVs | diff --git a/.claude/skills/databricks-spark-declarative-pipelines/scripts/exploration_notebook.py b/.claude/skills/databricks-spark-declarative-pipelines/scripts/exploration_notebook.py new file mode 100644 index 0000000..f3f6785 --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/scripts/exploration_notebook.py @@ -0,0 +1,81 @@ +# Databricks notebook source +# MAGIC %md +# MAGIC # Data Exploration Notebook +# MAGIC +# MAGIC Explore raw data in Volumes before building pipeline transformations. +# MAGIC +# MAGIC **Note:** Pipeline transformations should use raw `.sql` or `.py` files, NOT notebooks. + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 1. Explore Raw Files in Volume +# MAGIC +# MAGIC Query raw parquet/json files directly to understand the data structure. + +# COMMAND ---------- + +# MAGIC %sql +# MAGIC -- Preview raw orders data +# MAGIC SELECT * FROM parquet.`/Volumes/my_catalog/my_schema/raw/orders/` LIMIT 100 + +# COMMAND ---------- + +# MAGIC %sql +# MAGIC -- Check schema and sample values +# MAGIC DESCRIBE SELECT * FROM parquet.`/Volumes/my_catalog/my_schema/raw/orders/` + +# COMMAND ---------- + +# MAGIC %sql +# MAGIC -- Data quality: nulls, distinct values, date range +# MAGIC SELECT +# MAGIC COUNT(*) AS total_rows, +# MAGIC COUNT(order_id) AS non_null_order_id, +# MAGIC COUNT(DISTINCT customer_id) AS unique_customers, +# MAGIC MIN(order_date) AS min_date, +# MAGIC MAX(order_date) AS max_date +# MAGIC FROM parquet.`/Volumes/my_catalog/my_schema/raw/orders/` + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 2. Explore Another Raw Source + +# COMMAND ---------- + +# MAGIC %sql +# MAGIC -- Preview raw customers data +# MAGIC SELECT * FROM parquet.`/Volumes/my_catalog/my_schema/raw/customers/` LIMIT 100 + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 3. Join Raw Data for Exploration +# MAGIC +# MAGIC Test joins before building the pipeline. + +# COMMAND ---------- + +# MAGIC %sql +# MAGIC -- Join orders with customers to validate keys +# MAGIC SELECT +# MAGIC o.order_id, +# MAGIC o.order_date, +# MAGIC o.amount, +# MAGIC c.customer_name, +# MAGIC c.email +# MAGIC FROM parquet.`/Volumes/my_catalog/my_schema/raw/orders/` o +# MAGIC LEFT JOIN parquet.`/Volumes/my_catalog/my_schema/raw/customers/` c +# MAGIC ON o.customer_id = c.customer_id +# MAGIC LIMIT 100 + +# COMMAND ---------- + +# MAGIC %sql +# MAGIC -- Check for orphan orders (no matching customer) +# MAGIC SELECT COUNT(*) AS orphan_orders +# MAGIC FROM parquet.`/Volumes/my_catalog/my_schema/raw/orders/` o +# MAGIC LEFT JOIN parquet.`/Volumes/my_catalog/my_schema/raw/customers/` c +# MAGIC ON o.customer_id = c.customer_id +# MAGIC WHERE c.customer_id IS NULL diff --git a/.claude/skills/databricks-spark-structured-streaming/SKILL.md b/.claude/skills/databricks-spark-structured-streaming/SKILL.md index b1f5930..ddb52a0 100644 --- a/.claude/skills/databricks-spark-structured-streaming/SKILL.md +++ b/.claude/skills/databricks-spark-structured-streaming/SKILL.md @@ -1,6 +1,6 @@ --- name: databricks-spark-structured-streaming -description: Comprehensive guide to Spark Structured Streaming for production workloads. Use when building streaming pipelines, implementing real-time data processing, handling stateful operations, or optimizing streaming performance. +description: "Comprehensive guide to Spark Structured Streaming for production workloads. Use when building streaming pipelines, working with Kafka ingestion, implementing Real-Time Mode (RTM), configuring triggers (processingTime, availableNow), handling stateful operations with watermarks, optimizing checkpoints, performing stream-stream or stream-static joins, writing to multiple sinks, or tuning streaming cost and performance." --- # Spark Structured Streaming diff --git a/.claude/skills/databricks-spark-structured-streaming/kafka-streaming.md b/.claude/skills/databricks-spark-structured-streaming/kafka-streaming.md index 83630e8..9731434 100644 --- a/.claude/skills/databricks-spark-structured-streaming/kafka-streaming.md +++ b/.claude/skills/databricks-spark-structured-streaming/kafka-streaming.md @@ -140,30 +140,30 @@ df_bronze.writeStream \ ### Pattern 3: Real-Time Mode (Sub-Second Latency) -Use RTM for < 800ms latency requirements: +Use RTM for sub-second (as low as 5ms) latency requirements. Requires DBR 16.4 LTS+: ```python -# Real-time trigger (Databricks 13.3+) +# Real-time trigger (DBR 16.4 LTS+) +# Requirements: dedicated cluster, no autoscaling, no Photon, outputMode("update") +# Spark config on cluster: spark.databricks.streaming.realTimeMode.enabled = true query = (enriched_df .select(col("key"), col("value")) .writeStream .format("kafka") .option("kafka.bootstrap.servers", brokers) .option("topic", "output-events") - .trigger(realTime=True) # Enable RTM + .outputMode("update") # RTM only supports update mode + .trigger(realTime="5 minutes") # PySpark requires specifying the checkpoint interval .option("checkpointLocation", checkpoint_path) .start() ) -# RTM Cluster Requirements -spark.conf.set("spark.databricks.photon.enabled", "true") -spark.conf.set("spark.sql.streaming.stateStore.providerClass", - "com.databricks.sql.streaming.state.RocksDBStateProvider") - # When to use RTM: -# - Latency < 800ms required -# - Photon enabled -# - Fixed-size cluster (no autoscaling) +# - Sub-second latency required (achieves as low as 5ms E2E) +# - Photon must be DISABLED (not supported with RTM) +# - Autoscaling must be DISABLED +# - Dedicated (single-user) cluster only +# - forEachBatch is NOT supported in RTM ``` ### Pattern 4: Event Enrichment (Kafka to Kafka with Delta) diff --git a/.claude/skills/databricks-synthetic-data-gen/SKILL.md b/.claude/skills/databricks-synthetic-data-gen/SKILL.md new file mode 100644 index 0000000..c046e48 --- /dev/null +++ b/.claude/skills/databricks-synthetic-data-gen/SKILL.md @@ -0,0 +1,261 @@ +--- +name: databricks-synthetic-data-gen +description: "Generate realistic synthetic data using Spark + Faker (strongly recommended). Supports serverless execution, multiple output formats (Parquet/JSON/CSV/Delta), and scales from thousands to millions of rows. For small datasets (<10K rows), can optionally generate locally and upload to volumes. Use when user mentions 'synthetic data', 'test data', 'generate data', 'demo dataset', 'Faker', or 'sample data'." +--- + +> Catalog and schema are **always user-supplied** — never default to any value. If the user hasn't provided them, ask. For any UC write, **always create the schema if it doesn't exist** before writing data. + +# Databricks Synthetic Data Generation + +Generate realistic, story-driven synthetic data for Databricks using **Spark + Faker + Pandas UDFs** (strongly recommended). + +## Data Must Tell a Business Story + +Synthetic data should demonstrate how Databricks helps solve real business problems. + +**The pattern:** Something goes wrong → business impact ($) → analyze root cause → identify affected customers → fix and prevent. + +**Key principles:** +- **Problem → Impact → Analysis → Solution** — Include an incident, anomaly, or issue that causes measurable business impact. The data lets you find the root cause and act on it. +- **Industry-relevant but simple** — Use domain terms (e.g., "SLA breach", "churn", "stockout") but keep the schema easy to understand. A few tables, clear relationships. +- **Business metrics with $ impact** — Revenue, MRR, cost, conversion rate. Every story needs a dollar sign to show why it matters. +- **Tables explain each other** — Ticket spike? Incident table shows the outage. Revenue drop? Churn table shows who left and why. All data connects. +- **Actionable insights** — Data should answer: What happened? Who's affected? How much did it cost? How do we prevent it? + +**Why no flat distributions:** Uniform data has no story — no spikes, no anomalies, no cohort, no 20/80, no skew, nothing to investigate. It can't show Databricks' value for root cause analysis. + +## References + +| When | Guide | +|------|-------| +| User mentions **ML model training** or complex time patterns | [references/1-data-patterns.md](references/1-data-patterns.md) — ML-ready data, time multipliers, row coherence | +| Errors during generation | [references/2-troubleshooting.md](references/2-troubleshooting.md) — Fixing common issues | + +## Critical Rules + +1. **Data tells a story** — Something goes wrong, impacts $, can be analyzed and fixed. Show Databricks value. +2. **All data serves the story** — Every table and column must be coherent and usable in dashboards or ML models. No orphan data, no random noise — if it doesn't help explain or plot a futur dashboard or predict, don't generate it. +3. **Industry terms, simple schema** — Use domain-specific vocabulary but keep it easy to understand (few tables, clear relationships) +4. **Never uniform distributions** — Skewed categories, log-normal amounts, 80/20 patterns. Flat = no story = useless +5. **Enough data for trends** — ~100K+ rows for main tables so patterns survive aggregation +6. **Ask for catalog/schema** — Never default, always confirm before generating +7. **Present plan for approval** — Show tables, distributions, assumptions before writing code +8. **Master tables first** — Generate parent tables, write to Delta, then create children with valid FKs +9. **Use Spark + Faker + Pandas UDFs** — Scalable, parallel. Polars only if user explicitly wants local + <30K rows +10. **Use Databricks Connect Serverless by default to generate data** — Update databricks-connect on python 3.12 if required (avoid using execute_code unless instructed to not use Databricks Connect) +11. **No `.cache()` or `.persist()`** — Not supported on serverless. Write to Delta, read back for joins +12. **No Python loops or `.collect()`** — Use Spark parallelism. No driver-side iteration, avoid Pandas↔Spark conversions + +## Generation Planning Workflow + +**Before generating any code, you MUST present a plan for user approval.** + +### ⚠️ MUST DO: Confirm Catalog Before Proceeding + +**You MUST explicitly ask the user which catalog to use.** Do not assume or proceed without confirmation. + +Example prompt to user: +> "Which Unity Catalog should I use for this data?" + +When presenting your plan, always show the selected catalog prominently: +``` +📍 Output Location: catalog_name.schema_name + Volume: /Volumes/catalog_name/schema_name/raw_data/ +``` + +This makes it easy for the user to spot and correct if needed. + +### Step 1: Gather Requirements + +Ask the user about: +- **Catalog/Schema** — Which catalog to use? +- **Domain** — E-commerce, support tickets, IoT, financial? (Use industry terms) + +**If user doesn't specify a story:** Propose one. Don't generate bland data — suggest an incident, anomaly, or trend that shows Databricks value (e.g., "I'll include a system outage that causes ticket spike and churn — this lets you demo root cause analysis"). + +### Step 2: Present Plan with Story + +Show a clear specification with **the business story and your assumptions surfaced**: + +``` +📍 Output Location: {user_catalog}.support_demo + Volume: /Volumes/{user_catalog}/support_demo/raw_data/ + +📖 Story: A payment system outage causes support ticket spike. Resolution times + degrade, enterprise customers churn, revenue drops $2.3M. With Databricks we + identify the root cause, affected customers, and prevent future impact. +``` + +| Table | Description | Rows | Key Assumptions | +|-------|-------------|------|-----------------| +| customers | Customer profiles with tier, MRR | 10,000 | Enterprise 10% but 60% of revenue | +| tickets | Support tickets with priority, resolution_time | 80,000 | Spike during outage, SLA breaches | +| incidents | System events (outages, deployments) | 50 | Payment outage mid-month | +| churn_events | Customer cancellations with reason | 500 | Spike after poor support experience | + +**Business metrics:** +- `customers.mrr` — Revenue at risk ($) +- `tickets.resolution_hours` — SLA performance +- `churn_events.lost_mrr` — Churn impact ($) + +**The story this data tells:** +- Incident table shows payment outage on March 15 +- Tickets spike 5x during outage, resolution time degrades from 4h → 18h +- Enterprise customers with SLA breaches churn 3 weeks later +- Total impact: $2.3M lost MRR, traceable to one incident +- **Databricks value:** Root cause analysis, identify at-risk customers, build alerting + +**Ask user**: "Does this story work? Any adjustments?" + +### Step 3: Ask About Data Features + +- [x] Skew (non-uniform distributions) - **Enabled by default** +- [x] Joins (referential integrity) - **Enabled by default** +- [ ] Bad data injection (for data quality testing) +- [ ] Multi-language text +- [ ] Incremental mode (append instead of overwrite) + +### Pre-Generation Checklist + +- [ ] **Catalog confirmed** - User explicitly approved which catalog to use +- [ ] Output location shown prominently in plan (easy to spot/change) +- [ ] Table specification shown and approved +- [ ] Assumptions about distributions confirmed +- [ ] User confirmed compute preference (Databricks Connect on serverless recommended) +- [ ] Data features selected + +**Do NOT proceed to code generation until user approves the plan, including the catalog.** + +### Post-Generation Checklist + +After generating data, use `get_volume_folder_details` to validate the output matches requirements: +- Row counts match the plan +- Schema matches expected columns and types +- Data distributions look reasonable (check column stats) + +## Use Databricks Connect Spark + Faker Pattern + +```python +from databricks.connect import DatabricksSession, DatabricksEnv +from pyspark.sql import functions as F +from pyspark.sql.types import StringType +import pandas as pd + +# Setup serverless with dependencies (MUST list all libs used in UDFs) +env = DatabricksEnv().withDependencies("faker", "holidays") +spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() + +# Pandas UDF pattern - import lib INSIDE the function +@F.pandas_udf(StringType()) +def fake_name(ids: pd.Series) -> pd.Series: + from faker import Faker # Import inside UDF + fake = Faker() + return pd.Series([fake.name() for _ in range(len(ids))]) + +# Generate with spark.range, apply UDFs +customers_df = spark.range(0, 10000, numPartitions=16).select( + F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), + fake_name(F.col("id")).alias("name"), +) + +# Write to Volume as Parquet (default for raw data) +# Path is a folder with table name: /Volumes/catalog/schema/raw_data/customers/ +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") +customers_df.write.mode("overwrite").parquet(f"/Volumes/{CATALOG}/{SCHEMA}/raw_data/customers") +``` + +**Partitions by scale:** `spark.range(N, numPartitions=P)` +- <100K rows: 8 partitions +- 100K-500K: 16 partitions +- 500K-1M: 32 partitions +- 1M+: 64+ partitions + +**Output formats:** +- **Parquet to Volume** (default): `df.write.parquet("/Volumes/.../raw_data/table")` — raw data for pipelines +- **Delta Table**: `df.write.saveAsTable("catalog.schema.table")` — if user wants queryable tables +- **JSON/CSV**: small dimension tables, replicate legacy systems + +## Performance Rules + +Generated scripts must be highly performant. **Never** do these: + +| Anti-Pattern | Why It's Slow | Do This Instead | +|--------------|---------------|-----------------| +| Python loops on driver | Single-threaded, no parallelism | Use `spark.range()` + Spark operations | +| `.collect()` then iterate | Brings all data to driver memory | Keep data in Spark, use DataFrame ops | +| Pandas → Spark → Pandas | Serialization overhead, defeats distribution | Stay in Spark, use `pandas_udf` only for UDFs | +| Read/write temp files | Unnecessary I/O | Chain DataFrame transformations | +| Scalar UDFs | Row-by-row processing | Use `pandas_udf` for batch processing | + +**Good pattern:** `spark.range()` → Spark transforms → `pandas_udf` for Faker → write directly + +## Common Patterns + +### Weighted Categories (never uniform) +```python +F.when(F.rand() < 0.6, "Free").when(F.rand() < 0.9, "Pro").otherwise("Enterprise") +``` + +### Log-Normal Amounts (in a pandas UDF) +Use `np.random.lognormal(mean, sigma)` — always positive, long tail: +- Enterprise: `lognormal(7.5, 0.8)` → ~$1800 median +- Pro: `lognormal(5.5, 0.7)` → ~$245 median +- Free: `lognormal(4.0, 0.6)` → ~$55 median + +### Date Range (Last 6 Months) +```python +END_DATE = datetime.now() +START_DATE = END_DATE - timedelta(days=180) +``` + +### Infrastructure (always create in script) +```python +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") +``` + +### Referential Integrity (FK pattern) +Write master table to Delta first, then read back for FK joins (no `.cache()` on serverless): +```python +# 1. Write master table +customers_df.write.mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.customers") + +# 2. Read back for FK lookup +customer_lookup = spark.table(f"{CATALOG}.{SCHEMA}.customers").select("customer_idx", "customer_id") + +# 3. Generate child table with valid FKs via join +orders_df = spark.range(N_ORDERS).select( + (F.abs(F.hash(F.col("id"))) % N_CUSTOMERS).alias("customer_idx") +) +orders_with_fk = orders_df.join(customer_lookup, on="customer_idx") +``` + +## Setup + +Requires Python 3.12 and databricks-connect>=16.4. Use `uv`: + +```bash +uv pip install "databricks-connect>=16.4,<17.4" faker numpy pandas holidays +``` + +## Related Skills + +- **databricks-unity-catalog** — Managing catalogs, schemas, and volumes +- **databricks-bundles** — DABs for production deployment + +## Common Issues + +| Issue | Solution | +|-------|----------| +| `ImportError: cannot import name 'DatabricksEnv'` | Upgrade: `uv pip install "databricks-connect>=16.4"` | +| Python 3.11 instead of 3.12 | Python 3.12 required. Use `uv` to create env with correct version | +| `ModuleNotFoundError: faker` | Add to `withDependencies()`, import inside UDF | +| Faker UDF is slow | Use `pandas_udf` for batch processing | +| Out of memory | Increase `numPartitions` in `spark.range()` | +| Referential integrity errors | Write master table to Delta first, read back for FK joins | +| `PERSIST TABLE is not supported on serverless` | **NEVER use `.cache()` or `.persist()` with serverless** - write to Delta table first, then read back | +| `F.window` vs `Window` confusion | Use `from pyspark.sql.window import Window` for `row_number()`, `rank()`, etc. `F.window` is for streaming only. | +| Broadcast variables not supported | **NEVER use `spark.sparkContext.broadcast()` with serverless** | + +See [references/2-troubleshooting.md](references/2-troubleshooting.md) for full troubleshooting guide. diff --git a/.claude/skills/databricks-synthetic-data-gen/references/1-data-patterns.md b/.claude/skills/databricks-synthetic-data-gen/references/1-data-patterns.md new file mode 100644 index 0000000..eba6491 --- /dev/null +++ b/.claude/skills/databricks-synthetic-data-gen/references/1-data-patterns.md @@ -0,0 +1,146 @@ +# Data Patterns Guide + +Creating realistic synthetic data that tells a story. + +> **Note:** This guide provides principles and simplified examples. Actual implementations should be more sophisticated — use domain-specific distributions, realistic business rules, and correlations that reflect the user's actual use case. Ask clarifying questions to understand the business context before generating. + +## Core Principles + +### 1. Data Must Be Interesting + +Synthetic data should reveal patterns humans can see in dashboards and ML models can learn from: + +- **Visible trends** — Revenue growth, seasonal spikes, degradation over time +- **Actionable segments** — Clear differences between customer tiers, regions, product categories +- **Anomalies to detect** — Fraud patterns, equipment failures, churn signals +- **Correlations to discover** — Higher tier = more spend, faster resolution = better CSAT + +**Anti-pattern:** Uniform random data with no story — useless for demos and ML. + +### 2. Non-Uniform Distributions + +Real data is never uniformly distributed. Use appropriate distributions: + +| Distribution | When to Use | Examples | +|--------------|-------------|----------| +| **Log-normal** | Monetary values, sizes | Order amounts, salaries, file sizes | +| **Pareto (80/20)** | Popularity, wealth | 20% of customers = 80% of revenue | +| **Exponential** | Time between events | Support resolution time, session duration | +| **Weighted categorical** | Skewed categories | Status (70% complete, 5% failed), tiers | + +```python +# Log-normal for amounts (long tail, always positive) +amount = np.random.lognormal(mean=5.5, sigma=0.8) # ~$245 median + +# Pareto for power-law (few large, many small) +value = (np.random.pareto(a=1.5) + 1) * base_value + +# Exponential for time-to-event +hours = np.random.exponential(scale=24) # avg 24h, skewed right +``` + +### 3. Row Coherence + +Attributes within a row must make business sense together. Generate correlated attributes in a single UDF for example: + +| If This... | Then This... | +|------------|--------------| +| Enterprise tier | Higher order amounts, more activity, priority support | +| Critical priority | Faster resolution, more interactions | +| Older equipment | Higher failure rate, more anomalies | +| Large transaction + unusual hour | Higher fraud probability | +| Fast resolution | Higher CSAT score | + +```python +@F.pandas_udf("struct") +def generate_coherent_ticket(tiers: pd.Series) -> pd.DataFrame: + """All attributes correlate logically within each row.""" + results = [] + for tier in tiers: + # Priority depends on tier + priority = "Critical" if tier == "Enterprise" and random() < 0.3 else "Medium" + # Resolution depends on priority + resolution = np.random.exponential(4 if priority == "Critical" else 36) + # CSAT depends on resolution + csat = 5 if resolution < 4 else (3 if resolution < 24 else 2) + results.append({"priority": priority, "resolution_hours": resolution, "csat": csat}) + return pd.DataFrame(results) +``` + +### 4. The 80/20 Rule + +Apply power-law distributions where appropriate: + +- **20% of customers** generate 80% of orders/revenue +- **20% of products** account for 80% of sales +- **20% of support agents** handle 80% of tickets + +Implementation: Use weighted sampling when assigning FKs, not uniform random. + +### 5. Time-Based Patterns + +Most data has temporal patterns: + +- **Weekday vs weekend** — B2B drops on weekends, B2C peaks +- **Business hours** — Support tickets cluster 9am-5pm +- **Seasonality** — Q4 retail spike, summer travel peak +- **Trends** — Growth over time, degradation curves + +```python +def get_volume_multiplier(date): + multiplier = 1.0 + if date.weekday() >= 5: multiplier *= 0.6 # Weekend drop + if date.month in [11, 12]: multiplier *= 1.5 # Holiday spike + return multiplier +``` + +### 6. ML-Ready Data + +If data will train ML models, ensure: + +- **Signal exists** — The patterns you want the model to learn are present +- **Noise is realistic** — Not too clean (overfitting) or too noisy (unlearnable) +- **Class balance** — Fraud at 0.1-1%, not 50/50 (unrealistic) +- **Temporal validity** — Train/test split respects time (no future leakage) + +## Referential Integrity + +Generate master tables first, write to Delta, then join for FKs: + +```python +# 1. Generate and write master table +customers_df.write.mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.customers") + +# 2. Read back for FK joins (NOT cache - unsupported on serverless) +customer_lookup = spark.table(f"{CATALOG}.{SCHEMA}.customers") + +# 3. Generate child table with valid FKs via join +orders_df = spark.range(N_ORDERS).select( + (F.abs(F.hash(F.col("id"))) % N_CUSTOMERS).alias("customer_idx") +) +orders_with_fk = orders_df.join(customer_lookup, on="customer_idx") +``` + +## Data Volume + +Generate enough rows so patterns survive aggregation: + +| Analysis Type | Minimum Rows | Rationale | +|---------------|--------------|-----------| +| Daily dashboard | 50-100/day | Trends visible after weekly rollup | +| Category comparison | 500+ per category | Statistical significance | +| ML training | 10K-100K+ | Enough signal for model learning | +| Customer-level | 5-20 events/customer | Individual patterns visible | + +**Rule of thumb:** If you'll GROUP BY a column, ensure each group has 100+ rows. + +--- + +## Remember + +These are guiding principles, not templates. Real implementations should: +- Reflect the user's specific business domain and terminology +- Use realistic parameter values (research typical ranges for the industry) +- Include edge cases relevant to the use case (returns, cancellations, failures) +- Have more complex correlations than shown in examples above +- **Never use flat/uniform distributions** — categories, tiers, regions, statuses should always be skewed (e.g., 60/30/10 not 33/33/33) diff --git a/.claude/skills/databricks-synthetic-data-gen/references/2-troubleshooting.md b/.claude/skills/databricks-synthetic-data-gen/references/2-troubleshooting.md new file mode 100644 index 0000000..420b350 --- /dev/null +++ b/.claude/skills/databricks-synthetic-data-gen/references/2-troubleshooting.md @@ -0,0 +1,324 @@ +# Troubleshooting Guide + +Common issues and solutions for synthetic data generation. + +## Environment Issues + +### ModuleNotFoundError: faker (or other library) + +**Problem:** Dependencies not available in execution environment. + +**Solutions by execution mode:** + +| Mode | Solution | +|------|----------| +| **DB Connect 16.4+** | Use `DatabricksEnv().withDependencies("faker", "pandas", ...)` | +| **Older DB Connect with Serverless** | Create job with `environments` parameter | +| **Databricks Runtime** | Use Databricks CLI to install `faker holidays` | +| **Classic cluster** | Use Databricks CLI to install libraries. `databricks libraries install --json '{"cluster_id": "", "libraries": [{"pypi": {"package": "faker"}}, {"pypi": {"package": "holidays"}}]}'` | + +```python +# For DB Connect 16.4+ +from databricks.connect import DatabricksSession, DatabricksEnv + +env = DatabricksEnv().withDependencies("faker", "pandas", "numpy", "holidays") +spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() +``` + +### DatabricksEnv not found + +**Problem:** Using older databricks-connect version. + +**Solution:** Upgrade to 16.4+ or use job-based approach: + +```bash +# Upgrade (prefer uv, fall back to pip) +uv pip install "databricks-connect>=16.4,<17.4" +# or: pip install "databricks-connect>=16.4,<17.4" + +# Or use job with environments parameter instead +``` + +### serverless_compute_id error + +**Problem:** Missing serverless configuration. + +**Solution:** Add to `~/.databrickscfg`: + +```ini +[DEFAULT] +host = https://your-workspace.cloud.databricks.com/ +serverless_compute_id = auto +auth_type = databricks-cli +``` + +--- + +## Execution Issues + +### CRITICAL: cache() and persist() NOT supported on serverless + +**Problem:** Using `.cache()` or `.persist()` on serverless compute fails with: +``` +AnalysisException: [NOT_SUPPORTED_WITH_SERVERLESS] PERSIST TABLE is not supported on serverless compute. +``` + +**Why this happens:** Serverless compute does not support caching DataFrames in memory. This is a fundamental limitation of the serverless architecture. + +**Solution:** Write master tables to Delta first, then read them back for FK joins: + +```python +# BAD - will fail on serverless +customers_df = spark.range(0, N_CUSTOMERS)... +customers_df.cache() # ❌ FAILS: "PERSIST TABLE is not supported on serverless compute" + +# GOOD - write to Delta, then read back +customers_df = spark.range(0, N_CUSTOMERS)... +customers_df.write.mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.customers") +customer_lookup = spark.table(f"{CATALOG}.{SCHEMA}.customers") # ✓ Read from Delta +``` + +**Best practice for referential integrity:** +1. Generate master table (e.g., customers) +2. Write to Delta table +3. Read back for FK lookup joins +4. Generate child tables (e.g., orders, tickets) with valid FKs +5. Write child tables to Delta + +--- + +### Serverless job fails to start + +**Possible causes:** +1. Workspace doesn't have serverless enabled +2. Unity Catalog permissions missing +3. Invalid environment configuration + +**Solutions:** +```python +# Verify serverless is available +# Try creating a simple job first to test + +# Check Unity Catalog permissions +spark.sql("SELECT current_catalog(), current_schema()") +``` + +### Classic cluster startup slow (3-8 minutes) + +**Problem:** Clusters take time to start. + +**Solution:** Switch to serverless: + +```python +# Instead of: +# spark = DatabricksSession.builder.clusterId("xxx").getOrCreate() + +# Use: +spark = DatabricksSession.builder.serverless(True).getOrCreate() +``` + +### "Either base environment or version must be provided" + +**Problem:** Missing `client` in job environment spec. + +**Solution:** Add `"client": "4"` to the spec: + +```python +{ + "environments": [{ + "environment_key": "datagen_env", + "spec": { + "client": "4", # Required! + "dependencies": ["faker", "numpy", "pandas"] + } + }] +} +``` + +--- + +## Data Generation Issues + +### AttributeError: 'function' object has no attribute 'partitionBy' + +**Problem:** Using `F.window` instead of `Window` for analytical window functions. + +```python +# WRONG - F.window is for time-based tumbling/sliding windows (streaming) +window_spec = F.window.partitionBy("account_id").orderBy("contact_id") +# Error: AttributeError: 'function' object has no attribute 'partitionBy' + +# CORRECT - Window is for analytical window specifications +from pyspark.sql.window import Window +window_spec = Window.partitionBy("account_id").orderBy("contact_id") +``` + +**When to use Window:** For analytical functions like `row_number()`, `rank()`, `lead()`, `lag()`: + +```python +from pyspark.sql.window import Window + +# Mark first contact per account as primary +window_spec = Window.partitionBy("account_id").orderBy("contact_id") +contacts_df = contacts_df.withColumn( + "is_primary", + F.row_number().over(window_spec) == 1 +) +``` + +--- + +### Faker UDF is slow + +**Problem:** Single-row UDFs don't parallelize well. + +**Solution:** Use `pandas_udf` for batch processing: + +```python +# SLOW - scalar UDF +@F.udf(returnType=StringType()) +def slow_fake_name(): + return Faker().name() + +# FAST - pandas UDF (batch processing) +@F.pandas_udf(StringType()) +def fast_fake_name(ids: pd.Series) -> pd.Series: + fake = Faker() + return pd.Series([fake.name() for _ in range(len(ids))]) +``` + +### Out of memory with large data + +**Problem:** Not enough partitions for data size. + +**Solution:** Increase partitions: + +```python +# For large datasets (1M+ rows) +customers_df = spark.range(0, N_CUSTOMERS, numPartitions=64) # Increase from default +``` + +| Data Size | Recommended Partitions | +|-----------|----------------------| +| < 100K | 8 | +| 100K - 500K | 16 | +| 500K - 1M | 32 | +| 1M+ | 64+ | + +### Context corrupted on classic cluster + +**Problem:** Stale execution context. + +**Solution:** Create fresh context (omit context_id), reinstall libraries: + +```python +# Don't reuse context_id if you see strange errors +# Let it create a new context +``` + +### Referential integrity violations + +**Problem:** Foreign keys reference non-existent parent records. + +**Solution:** Write master table to Delta first, then read back for FK joins: + +```python +# 1. Generate and WRITE master table (do NOT use cache with serverless!) +customers_df = spark.range(0, N_CUSTOMERS)... +customers_df.write.mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.customers") + +# 2. Read back for FK lookups +customer_lookup = spark.table(f"{CATALOG}.{SCHEMA}.customers").select("customer_id", "tier") + +# 3. Generate child table with valid FKs +orders_df = spark.range(0, N_ORDERS).join( + customer_lookup, + on=, + how="left" +) +``` + +> **WARNING:** Do NOT use `.cache()` or `.persist()` with serverless compute. See the dedicated section above. + +--- + +## Data Quality Issues + +### Uniform distributions (unrealistic) + +**Problem:** All customers have similar order counts, amounts are evenly distributed. + +**Solution:** Use non-linear distributions: + +```python +# BAD - uniform +amounts = np.random.uniform(10, 1000, N) + +# GOOD - log-normal (realistic) +amounts = np.random.lognormal(mean=5, sigma=0.8, N) +``` + +### Missing time-based patterns + +**Problem:** Data doesn't reflect weekday/weekend or seasonal patterns. + +**Solution:** Add multipliers: + +```python +import holidays + +US_HOLIDAYS = holidays.US(years=[2024, 2025]) + +def get_multiplier(date): + mult = 1.0 + if date.weekday() >= 5: # Weekend + mult *= 0.6 + if date in US_HOLIDAYS: + mult *= 0.3 + return mult +``` + +### Incoherent row attributes + +**Problem:** Enterprise customer has low-value orders, critical ticket has slow resolution. + +**Solution:** Correlate attributes: + +```python +# Priority based on tier +if tier == 'Enterprise': + priority = np.random.choice(['Critical', 'High'], p=[0.4, 0.6]) +else: + priority = np.random.choice(['Medium', 'Low'], p=[0.6, 0.4]) + +# Resolution based on priority +resolution_scale = {'Critical': 4, 'High': 12, 'Medium': 36, 'Low': 72} +resolution_hours = np.random.exponential(scale=resolution_scale[priority]) +``` + +--- + +## Validation Steps + +After generation, verify your data: + +```python +# 1. Check row counts +print(f"Customers: {customers_df.count():,}") +print(f"Orders: {orders_df.count():,}") + +# 2. Verify distributions +customers_df.groupBy("tier").count().show() +orders_df.describe("amount").show() + +# 3. Check referential integrity +orphans = orders_df.join( + customers_df, + orders_df.customer_id == customers_df.customer_id, + "left_anti" +) +print(f"Orphan orders: {orphans.count()}") + +# 4. Verify date range +orders_df.select(F.min("order_date"), F.max("order_date")).show() +``` diff --git a/.claude/skills/databricks-synthetic-data-gen/scripts/generate_synthetic_data.py b/.claude/skills/databricks-synthetic-data-gen/scripts/generate_synthetic_data.py new file mode 100644 index 0000000..b9f953f --- /dev/null +++ b/.claude/skills/databricks-synthetic-data-gen/scripts/generate_synthetic_data.py @@ -0,0 +1,390 @@ +"""Generate synthetic data using Spark + Faker + Pandas UDFs. + +This is the recommended approach for ALL data generation tasks: +- Scales from thousands to millions of rows +- Parallel execution via Spark +- Direct write to Unity Catalog +- Works with serverless and classic compute + +Auto-detects environment and uses: +- DatabricksEnv with managed dependencies if databricks-connect >= 16.4 (local) +- Standard session if running on Databricks Runtime or older databricks-connect +""" +import sys +import os +from pyspark.sql import functions as F +from pyspark.sql.window import Window +from pyspark.sql.types import StringType, DoubleType, StructType, StructField, IntegerType +import numpy as np +import pandas as pd +from datetime import datetime, timedelta + +# ============================================================================= +# CONFIGURATION +# ============================================================================= +# Compute - Serverless strongly recommended +USE_SERVERLESS = True # Set to False and provide CLUSTER_ID for classic compute +CLUSTER_ID = None # Only used if USE_SERVERLESS=False + +# Storage - Update these for your environment +CATALOG = "" # REQUIRED: replace with your catalog +SCHEMA = "" # REQUIRED: replace with your schema +VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" + +# Data sizes +N_CUSTOMERS = 10_000 +N_ORDERS = 50_000 +PARTITIONS = 16 # Adjust: 8 for <100K, 32 for 1M+ + +# Date range - last 6 months from today +END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) +START_DATE = END_DATE - timedelta(days=180) + +# Write mode - "overwrite" for one-time, "append" for incremental +WRITE_MODE = "overwrite" + +# Bad data injection for testing data quality rules +INJECT_BAD_DATA = False # Set to True to inject bad data +BAD_DATA_CONFIG = { + "null_rate": 0.02, # 2% nulls in required fields + "outlier_rate": 0.01, # 1% impossible values + "orphan_fk_rate": 0.01, # 1% orphan foreign keys +} + +# Reproducibility +SEED = 42 + +# Tier distribution: Free 60%, Pro 30%, Enterprise 10% +TIER_PROBS = [0.6, 0.3, 0.1] + +# Region distribution +REGION_PROBS = [0.4, 0.25, 0.2, 0.15] + +# ============================================================================= +# ENVIRONMENT DETECTION AND SESSION CREATION +# ============================================================================= + +def is_databricks_runtime(): + """Check if running on Databricks Runtime vs locally.""" + return "DATABRICKS_RUNTIME_VERSION" in os.environ + +def get_databricks_connect_version(): + """Get databricks-connect version as (major, minor) tuple or None.""" + try: + import importlib.metadata + version_str = importlib.metadata.version('databricks-connect') + parts = version_str.split('.') + return (int(parts[0]), int(parts[1])) + except Exception: + return None + +# Detect environment +on_runtime = is_databricks_runtime() +db_version = get_databricks_connect_version() + +print("=" * 80) +print("ENVIRONMENT DETECTION") +print("=" * 80) +print(f"Running on Databricks Runtime: {on_runtime}") +if db_version: + print(f"databricks-connect version: {db_version[0]}.{db_version[1]}") +else: + print("databricks-connect: not available") + +# Use DatabricksEnv with managed dependencies if: +# - Running locally (not on Databricks Runtime) +# - databricks-connect >= 16.4 +use_managed_deps = (not on_runtime) and db_version and db_version >= (16, 4) + +if use_managed_deps: + print("Using DatabricksEnv with managed dependencies") + print("=" * 80) + from databricks.connect import DatabricksSession, DatabricksEnv + + env = DatabricksEnv().withDependencies("faker", "pandas", "numpy", "holidays") + + if USE_SERVERLESS: + spark = DatabricksSession.builder.withEnvironment(env).serverless(True).getOrCreate() + print("Connected to serverless compute with managed dependencies!") + else: + if not CLUSTER_ID: + raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") + spark = DatabricksSession.builder.withEnvironment(env).clusterId(CLUSTER_ID).getOrCreate() + print(f"Connected to cluster with managed dependencies!") +else: + print("Using standard session (dependencies must be pre-installed)") + print("=" * 80) + + # Check that UDF dependencies are available + print("\nChecking UDF dependencies...") + missing_deps = [] + + try: + from faker import Faker + print(" faker: OK") + except ImportError: + missing_deps.append("faker") + print(" faker: MISSING") + + try: + import pandas as pd + print(" pandas: OK") + except ImportError: + missing_deps.append("pandas") + print(" pandas: MISSING") + + if missing_deps: + print("\n" + "=" * 80) + print("ERROR: Missing dependencies for UDFs") + print("=" * 80) + print(f"Missing: {', '.join(missing_deps)}") + if on_runtime: + print('\nSolution: Install libraries via Databricks CLI:') + print(' databricks libraries install --json \'{"cluster_id": "", "libraries": [{"pypi": {"package": "faker"}}, {"pypi": {"package": "holidays"}}]}\'') + else: + print("\nSolution: Upgrade to databricks-connect >= 16.4 for managed deps") + print(" Or create a job with environment settings") + print("=" * 80) + sys.exit(1) + + print("\nAll dependencies available") + print("=" * 80) + + from databricks.connect import DatabricksSession + + if USE_SERVERLESS: + spark = DatabricksSession.builder.serverless(True).getOrCreate() + print("Connected to serverless compute") + else: + if not CLUSTER_ID: + raise ValueError("CLUSTER_ID must be set when USE_SERVERLESS=False") + spark = DatabricksSession.builder.clusterId(CLUSTER_ID).getOrCreate() + print(f"Connected to cluster ") + +# Import Faker for UDF definitions +from faker import Faker + +# ============================================================================= +# DEFINE PANDAS UDFs FOR FAKER DATA +# ============================================================================= + +@F.pandas_udf(StringType()) +def fake_name(ids: pd.Series) -> pd.Series: + """Generate realistic person names.""" + fake = Faker() + Faker.seed(SEED) + return pd.Series([fake.name() for _ in range(len(ids))]) + +@F.pandas_udf(StringType()) +def fake_company(ids: pd.Series) -> pd.Series: + """Generate realistic company names.""" + fake = Faker() + Faker.seed(SEED) + return pd.Series([fake.company() for _ in range(len(ids))]) + +@F.pandas_udf(StringType()) +def fake_address(ids: pd.Series) -> pd.Series: + """Generate realistic addresses.""" + fake = Faker() + Faker.seed(SEED) + return pd.Series([fake.address().replace('\n', ', ') for _ in range(len(ids))]) + +@F.pandas_udf(StringType()) +def fake_email(names: pd.Series) -> pd.Series: + """Generate email based on name.""" + emails = [] + for name in names: + if name: + domain = name.lower().replace(" ", ".").replace(",", "")[:20] + emails.append(f"{domain}@example.com") + else: + emails.append("unknown@example.com") + return pd.Series(emails) + +@F.pandas_udf(DoubleType()) +def generate_lognormal_amount(tiers: pd.Series) -> pd.Series: + """Generate amount based on tier using log-normal distribution.""" + np.random.seed(SEED) + amounts = [] + for tier in tiers: + if tier == "Enterprise": + amounts.append(float(np.random.lognormal(mean=7.5, sigma=0.8))) # ~$1800 avg + elif tier == "Pro": + amounts.append(float(np.random.lognormal(mean=5.5, sigma=0.7))) # ~$245 avg + else: + amounts.append(float(np.random.lognormal(mean=4.0, sigma=0.6))) # ~$55 avg + return pd.Series(amounts) + +# ============================================================================= +# CREATE INFRASTRUCTURE +# ============================================================================= +print("\nCreating infrastructure...") +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") +spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") +print(f"Infrastructure ready: {VOLUME_PATH}") + +# ============================================================================= +# GENERATE CUSTOMERS (Master Table) +# ============================================================================= +print(f"\nGenerating {N_CUSTOMERS:,} customers...") + +customers_df = ( + spark.range(0, N_CUSTOMERS, numPartitions=PARTITIONS) + .select( + F.concat(F.lit("CUST-"), F.lpad(F.col("id").cast("string"), 5, "0")).alias("customer_id"), + fake_name(F.col("id")).alias("name"), + fake_company(F.col("id")).alias("company"), + fake_address(F.col("id")).alias("address"), + # Tier distribution: Free 60%, Pro 30%, Enterprise 10% + F.when(F.rand(SEED) < TIER_PROBS[0], "Free") + .when(F.rand(SEED) < TIER_PROBS[0] + TIER_PROBS[1], "Pro") + .otherwise("Enterprise").alias("tier"), + # Region distribution + F.when(F.rand(SEED) < REGION_PROBS[0], "North") + .when(F.rand(SEED) < REGION_PROBS[0] + REGION_PROBS[1], "South") + .when(F.rand(SEED) < REGION_PROBS[0] + REGION_PROBS[1] + REGION_PROBS[2], "East") + .otherwise("West").alias("region"), + # Created date (within last 2 years before start date) + F.date_sub(F.lit(START_DATE.date()), (F.rand(SEED) * 730).cast("int")).alias("created_at"), + ) +) + +# Add tier-based ARR and email +customers_df = ( + customers_df + .withColumn("arr", F.round(generate_lognormal_amount(F.col("tier")), 2)) + .withColumn("email", fake_email(F.col("name"))) +) + +# Save customers +customers_df.write.mode(WRITE_MODE).parquet(f"{VOLUME_PATH}/customers") +print(f" Saved customers to {VOLUME_PATH}/customers") + +# Show tier distribution +print("\n Tier distribution:") +customers_df.groupBy("tier").count().orderBy("tier").show() + +# ============================================================================= +# GENERATE ORDERS (Child Table with Referential Integrity) +# ============================================================================= +print(f"\nGenerating {N_ORDERS:,} orders with referential integrity...") + +# Write customer lookup to temp Delta table (no .cache() on serverless!) +customers_tmp_table = f"{CATALOG}.{SCHEMA}._tmp_customers_lookup" +customers_df.select("customer_id", "tier").write.mode("overwrite").saveAsTable(customers_tmp_table) +customer_lookup = spark.table(customers_tmp_table) + +# Generate orders base +orders_df = ( + spark.range(0, N_ORDERS, numPartitions=PARTITIONS) + .select( + F.concat(F.lit("ORD-"), F.lpad(F.col("id").cast("string"), 6, "0")).alias("order_id"), + # Generate customer_idx for FK join (hash-based distribution) + (F.abs(F.hash(F.col("id"), F.lit(SEED))) % N_CUSTOMERS).alias("customer_idx"), + # Order status + F.when(F.rand(SEED) < 0.65, "delivered") + .when(F.rand(SEED) < 0.80, "shipped") + .when(F.rand(SEED) < 0.90, "processing") + .when(F.rand(SEED) < 0.95, "pending") + .otherwise("cancelled").alias("status"), + # Order date within date range + F.date_add(F.lit(START_DATE.date()), (F.rand(SEED) * 180).cast("int")).alias("order_date"), + ) +) + +# Add customer_idx to lookup for join +customer_lookup_with_idx = customer_lookup.withColumn( + "customer_idx", + (F.row_number().over(Window.orderBy(F.monotonically_increasing_id())) - 1).cast("int") +) + +# Join to get customer_id and tier as foreign key +orders_with_fk = ( + orders_df + .join(customer_lookup_with_idx, on="customer_idx", how="left") + .drop("customer_idx") +) + +# Add tier-based amount +orders_with_fk = orders_with_fk.withColumn( + "amount", + F.round(generate_lognormal_amount(F.col("tier")), 2) +) + +# ============================================================================= +# INJECT BAD DATA (OPTIONAL) +# ============================================================================= +if INJECT_BAD_DATA: + print("\nInjecting bad data for quality testing...") + + # Calculate counts + null_count = int(N_ORDERS * BAD_DATA_CONFIG["null_rate"]) + outlier_count = int(N_ORDERS * BAD_DATA_CONFIG["outlier_rate"]) + orphan_count = int(N_ORDERS * BAD_DATA_CONFIG["orphan_fk_rate"]) + + # Add bad data flags + orders_with_fk = orders_with_fk.withColumn( + "row_num", + F.row_number().over(Window.orderBy(F.monotonically_increasing_id())) + ) + + # Inject nulls in customer_id for first null_count rows + orders_with_fk = orders_with_fk.withColumn( + "customer_id", + F.when(F.col("row_num") <= null_count, None).otherwise(F.col("customer_id")) + ) + + # Inject negative amounts for next outlier_count rows + orders_with_fk = orders_with_fk.withColumn( + "amount", + F.when( + (F.col("row_num") > null_count) & (F.col("row_num") <= null_count + outlier_count), + F.lit(-999.99) + ).otherwise(F.col("amount")) + ) + + # Inject orphan FKs for next orphan_count rows + orders_with_fk = orders_with_fk.withColumn( + "customer_id", + F.when( + (F.col("row_num") > null_count + outlier_count) & + (F.col("row_num") <= null_count + outlier_count + orphan_count), + F.lit("CUST-NONEXISTENT") + ).otherwise(F.col("customer_id")) + ) + + orders_with_fk = orders_with_fk.drop("row_num") + + print(f" Injected {null_count} null customer_ids") + print(f" Injected {outlier_count} negative amounts") + print(f" Injected {orphan_count} orphan foreign keys") + +# Drop tier column (not needed in final output) +orders_final = orders_with_fk.drop("tier") + +# Save orders +orders_final.write.mode(WRITE_MODE).parquet(f"{VOLUME_PATH}/orders") +print(f" Saved orders to {VOLUME_PATH}/orders") + +# Show status distribution +print("\n Status distribution:") +orders_final.groupBy("status").count().orderBy("status").show() + +# ============================================================================= +# CLEANUP AND SUMMARY +# ============================================================================= +spark.sql(f"DROP TABLE IF EXISTS {customers_tmp_table}") + +print("\n" + "=" * 80) +print("GENERATION COMPLETE") +print("=" * 80) +print(f"Catalog: {CATALOG}") +print(f"Schema: {SCHEMA}") +print(f"Volume: {VOLUME_PATH}") +print(f"\nGenerated data:") +print(f" - customers: {N_CUSTOMERS:,} rows") +print(f" - orders: {N_ORDERS:,} rows") +if INJECT_BAD_DATA: + print(f" - Bad data injected: nulls, outliers, orphan FKs") +print(f"\nDate range: {START_DATE.date()} to {END_DATE.date()}") +print("=" * 80) diff --git a/.claude/skills/databricks-synthetic-data-generation/SKILL.md b/.claude/skills/databricks-synthetic-data-generation/SKILL.md deleted file mode 100644 index ce2a17c..0000000 --- a/.claude/skills/databricks-synthetic-data-generation/SKILL.md +++ /dev/null @@ -1,660 +0,0 @@ ---- -name: databricks-synthetic-data-generation -description: "Generate realistic synthetic data using Faker and Spark, with non-linear distributions, integrity constraints, and save to Databricks. Use when creating test data, demo datasets, or synthetic tables." ---- - -# Synthetic Data Generation - -Generate realistic, story-driven synthetic data for Databricks using Python with Faker and Spark. - -## Common Libraries - -These libraries are useful for generating realistic synthetic data: - -- **faker**: Generates realistic names, addresses, emails, companies, dates, etc. -- **holidays**: Provides country-specific holiday calendars for realistic date patterns - -These are typically NOT pre-installed on Databricks. Install them using `execute_databricks_command` tool: -- `code`: "%pip install faker holidays" - -Save the returned `cluster_id` and `context_id` for subsequent calls. - -## Workflow - -1. **Write Python code to a local file** in the project (e.g., `scripts/generate_data.py`) -2. **Execute on Databricks** using the `run_python_file_on_databricks` MCP tool -3. **If execution fails**: Edit the local file to fix the error, then re-execute -4. **Reuse the context** for follow-up executions by passing the returned `cluster_id` and `context_id` - -**Always work with local files first, then execute.** This makes debugging easier - you can see and edit the code. - -### Context Reuse Pattern - -The first execution auto-selects a running cluster and creates an execution context. **Reuse this context for follow-up calls** - it's much faster (~1s vs ~15s) and shares variables/imports: - -**First execution** - use `run_python_file_on_databricks` tool: -- `file_path`: "scripts/generate_data.py" - -Returns: `{ success, output, error, cluster_id, context_id, ... }` - -Save `cluster_id` and `context_id` for follow-up calls. - -**If execution fails:** -1. Read the error from the result -2. Edit the local Python file to fix the issue -3. Re-execute with same context using `run_python_file_on_databricks` tool: - - `file_path`: "scripts/generate_data.py" - - `cluster_id`: "" - - `context_id`: "" - -**Follow-up executions** reuse the context (faster, shares state): -- `file_path`: "scripts/validate_data.py" -- `cluster_id`: "" -- `context_id`: "" - -### Handling Failures - -When execution fails: -1. Read the error from the result -2. **Edit the local Python file** to fix the issue -3. Re-execute using the same `cluster_id` and `context_id` (faster, keeps installed libraries) -4. If the context is corrupted, omit `context_id` to create a fresh one - -### Installing Libraries - -Databricks provides Spark, pandas, numpy, and common data libraries by default. **Only install a library if you get an import error.** - -Use `execute_databricks_command` tool: -- `code`: "%pip install faker" -- `cluster_id`: "" -- `context_id`: "" - -The library is immediately available in the same context. - -**Note:** Keeping the same `context_id` means installed libraries persist across calls. - -## Storage Destination - -### Ask for Schema Name - -By default, use the `ai_dev_kit` catalog. Ask the user which schema to use: - -> "I'll save the data to `ai_dev_kit.`. What schema name would you like to use? (You can also specify a different catalog if needed.)" - -If the user provides just a schema name, use `ai_dev_kit.{schema}`. If they provide `catalog.schema`, use that instead. - -### Create Infrastructure in the Script - -Always create the catalog, schema, and volume **inside the Python script** using `spark.sql()`. Do NOT make separate MCP SQL calls - it's much slower. - -The `spark` variable is available by default on Databricks clusters. - -```python -# ============================================================================= -# CREATE INFRASTRUCTURE (inside the Python script) -# ============================================================================= -spark.sql(f"CREATE CATALOG IF NOT EXISTS {CATALOG}") -spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") -spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") -``` - -### Save to Volume as Raw Data (Never Tables) - -**Always save data to a Volume as parquet files, never directly to tables** (unless the user explicitly requests tables). This is the input for the downstream Spark Declarative Pipeline (SDP) that will handle bronze/silver/gold layers. - -```python -VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" - -# Save as parquet files (raw data) -spark.createDataFrame(customers_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") -spark.createDataFrame(orders_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/orders") -spark.createDataFrame(tickets_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/tickets") -``` - -## Raw Data Only - No Pre-Aggregated Fields (Unless Instructed Otherwise) - -**By default, generate raw, transactional data only.** Do not create fields that represent sums, totals, averages, or counts. - -- One row = one event/transaction/record -- No columns like `total_orders`, `sum_revenue`, `avg_csat`, `order_count` -- Each row has its own individual values, not rollups - -**Why?** A Spark Declarative Pipeline (SDP) will typically be built after data generation to: -- Ingest raw data (bronze layer) -- Clean and validate (silver layer) -- Aggregate and compute metrics (gold layer) - -The synthetic data is the **source** for this pipeline. Aggregations happen downstream. - -**Note:** If the user specifically requests aggregated fields or summary tables, follow their instructions. - -```python -# GOOD - Raw transactional data -# Customer table: one row per customer, no aggregated fields -customers_data.append({ - "customer_id": cid, - "name": fake.company(), - "tier": "Enterprise", - "region": "North", -}) - -# Order table: one row per order -orders_data.append({ - "order_id": f"ORD-{i:06d}", - "customer_id": cid, - "amount": 150.00, # This order's amount - "order_date": "2024-10-15", -}) - -# BAD - Don't add pre-aggregated fields -# customers_data.append({ -# "customer_id": cid, -# "total_orders": 47, # NO - this is an aggregation -# "total_revenue": 12500.00, # NO - this is a sum -# "avg_order_value": 265.95, # NO - this is an average -# }) -``` - -## Temporality and Data Volume - -### Date Range: Last 6 Months from Today - -**Always generate data for the last ~6 months ending at the current date.** This ensures: -- Data feels current and relevant for demos -- Recent patterns are visible in dashboards -- Downstream aggregations (daily/weekly/monthly) have enough history - -```python -from datetime import datetime, timedelta - -# Dynamic date range - last 6 months from today -END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) -START_DATE = END_DATE - timedelta(days=180) - -# Place special events within this range (e.g., incident 3 weeks ago) -INCIDENT_END = END_DATE - timedelta(days=21) -INCIDENT_START = INCIDENT_END - timedelta(days=10) -``` - -### Data Volume for Aggregation - -Generate enough data so patterns remain visible after downstream aggregation (SDP pipelines often aggregate by day/week/region/category). Rules of thumb: - -| Grain | Minimum Records | Rationale | -|-------|-----------------|-----------| -| Daily time series | 50-100/day | See trends after weekly rollup | -| Per category | 500+ per category | Statistical significance | -| Per customer | 5-20 events/customer | Enough for customer-level analysis | -| Total rows | 10K-50K minimum | Patterns survive GROUP BY | - -```python -# Example: 8000 tickets over 180 days = ~44/day average -# After weekly aggregation: ~310 records per week per category -# After monthly by region: still enough to see patterns -N_TICKETS = 8000 -N_CUSTOMERS = 2500 # Each has ~3 tickets on average -N_ORDERS = 25000 # ~10 orders per customer average -``` - -## Script Structure - -Always structure scripts with configuration variables at the top: - -```python -"""Generate synthetic data for [use case].""" -import numpy as np -import pandas as pd -from datetime import datetime, timedelta -from faker import Faker -import holidays -from pyspark.sql import SparkSession - -# ============================================================================= -# CONFIGURATION - Edit these values -# ============================================================================= -CATALOG = "my_catalog" -SCHEMA = "my_schema" -VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" - -# Data sizes - enough for aggregation patterns to survive -N_CUSTOMERS = 2500 -N_ORDERS = 25000 -N_TICKETS = 8000 - -# Date range - last 6 months from today -END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) -START_DATE = END_DATE - timedelta(days=180) - -# Special events (within the date range) -INCIDENT_END = END_DATE - timedelta(days=21) -INCIDENT_START = INCIDENT_END - timedelta(days=10) - -# Holiday calendar for realistic patterns -US_HOLIDAYS = holidays.US(years=[START_DATE.year, END_DATE.year]) - -# Reproducibility -SEED = 42 - -# ============================================================================= -# SETUP -# ============================================================================= -np.random.seed(SEED) -Faker.seed(SEED) -fake = Faker() -spark = SparkSession.builder.getOrCreate() - -# ... rest of script -``` - -## Key Principles - -### 1. Use Pandas for Generation, Spark for Saving - -Generate data with pandas (faster, easier), convert to Spark for saving: - -```python -import pandas as pd - -# Generate with pandas -customers_pdf = pd.DataFrame({ - "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], - "name": [fake.company() for _ in range(N_CUSTOMERS)], - "tier": np.random.choice(['Free', 'Pro', 'Enterprise'], N_CUSTOMERS, p=[0.6, 0.3, 0.1]), - "region": np.random.choice(['North', 'South', 'East', 'West'], N_CUSTOMERS, p=[0.4, 0.25, 0.2, 0.15]), - "created_at": [fake.date_between(start_date='-2y', end_date='-6m') for _ in range(N_CUSTOMERS)], -}) - -# Convert to Spark and save -customers_df = spark.createDataFrame(customers_pdf) -customers_df.write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") -``` - -### 2. Iterate on DataFrames for Referential Integrity - -Generate master tables first, then iterate on them to create related tables with matching IDs: - -```python -# 1. Generate customers (master table) -customers_pdf = pd.DataFrame({ - "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], - "tier": np.random.choice(['Free', 'Pro', 'Enterprise'], N_CUSTOMERS, p=[0.6, 0.3, 0.1]), - # ... -}) - -# 2. Create lookup for foreign key generation -customer_ids = customers_pdf["customer_id"].tolist() -customer_tier_map = dict(zip(customers_pdf["customer_id"], customers_pdf["tier"])) - -# Weight by tier - Enterprise customers generate more orders -tier_weights = customers_pdf["tier"].map({'Enterprise': 5.0, 'Pro': 2.0, 'Free': 1.0}) -customer_weights = (tier_weights / tier_weights.sum()).tolist() - -# 3. Generate orders with valid foreign keys and tier-based logic -orders_data = [] -for i in range(N_ORDERS): - cid = np.random.choice(customer_ids, p=customer_weights) - tier = customer_tier_map[cid] - - # Amount depends on tier - if tier == 'Enterprise': - amount = np.random.lognormal(7, 0.8) - elif tier == 'Pro': - amount = np.random.lognormal(5, 0.7) - else: - amount = np.random.lognormal(3.5, 0.6) - - orders_data.append({ - "order_id": f"ORD-{i:06d}", - "customer_id": cid, - "amount": round(amount, 2), - "order_date": fake.date_between(start_date=START_DATE, end_date=END_DATE), - }) - -orders_pdf = pd.DataFrame(orders_data) - -# 4. Generate tickets that reference both customers and orders -order_ids = orders_pdf["order_id"].tolist() -tickets_data = [] -for i in range(N_TICKETS): - cid = np.random.choice(customer_ids, p=customer_weights) - oid = np.random.choice(order_ids) # Or None for general inquiry - - tickets_data.append({ - "ticket_id": f"TKT-{i:06d}", - "customer_id": cid, - "order_id": oid if np.random.random() > 0.3 else None, - # ... - }) - -tickets_pdf = pd.DataFrame(tickets_data) -``` - -### 3. Non-Linear Distributions - -**Never use uniform distributions** - real data is rarely uniform: - -```python -# BAD - Uniform (unrealistic) -prices = np.random.uniform(10, 1000, size=N_ORDERS) - -# GOOD - Log-normal (realistic for prices, salaries, order amounts) -prices = np.random.lognormal(mean=4.5, sigma=0.8, size=N_ORDERS) - -# GOOD - Pareto/power law (popularity, wealth, page views) -popularity = (np.random.pareto(a=2.5, size=N_PRODUCTS) + 1) * 10 - -# GOOD - Exponential (time between events, resolution time) -resolution_hours = np.random.exponential(scale=24, size=N_TICKETS) - -# GOOD - Weighted categorical -regions = np.random.choice( - ['North', 'South', 'East', 'West'], - size=N_CUSTOMERS, - p=[0.40, 0.25, 0.20, 0.15] -) -``` - -### 4. Time-Based Patterns - -Add weekday/weekend effects, holidays, seasonality, and event spikes: - -```python -import holidays - -# Load holiday calendar -US_HOLIDAYS = holidays.US(years=[START_DATE.year, END_DATE.year]) - -def get_daily_multiplier(date): - """Calculate volume multiplier for a given date.""" - multiplier = 1.0 - - # Weekend drop - if date.weekday() >= 5: - multiplier *= 0.6 - - # Holiday drop (even lower than weekends) - if date in US_HOLIDAYS: - multiplier *= 0.3 - - # Q4 seasonality (higher in Oct-Dec) - multiplier *= 1 + 0.15 * (date.month - 6) / 6 - - # Incident spike - if INCIDENT_START <= date <= INCIDENT_END: - multiplier *= 3.0 - - # Random noise - multiplier *= np.random.normal(1, 0.1) - - return max(0.1, multiplier) - -# Distribute tickets across dates with realistic patterns -date_range = pd.date_range(START_DATE, END_DATE, freq='D') -daily_volumes = [int(BASE_DAILY_TICKETS * get_daily_multiplier(d)) for d in date_range] -``` - -### 5. Row Coherence - -Attributes within a row should correlate logically: - -```python -def generate_ticket(customer_id, tier, date): - """Generate a coherent ticket where attributes correlate.""" - - # Priority correlates with tier - if tier == 'Enterprise': - priority = np.random.choice(['Critical', 'High', 'Medium'], p=[0.3, 0.5, 0.2]) - else: - priority = np.random.choice(['Critical', 'High', 'Medium', 'Low'], p=[0.05, 0.2, 0.45, 0.3]) - - # Resolution time correlates with priority - resolution_scale = {'Critical': 4, 'High': 12, 'Medium': 36, 'Low': 72} - resolution_hours = np.random.exponential(scale=resolution_scale[priority]) - - # CSAT correlates with resolution time - if resolution_hours < 4: - csat = np.random.choice([4, 5], p=[0.3, 0.7]) - elif resolution_hours < 24: - csat = np.random.choice([3, 4, 5], p=[0.2, 0.5, 0.3]) - else: - csat = np.random.choice([1, 2, 3, 4], p=[0.1, 0.3, 0.4, 0.2]) - - return { - "customer_id": customer_id, - "priority": priority, - "resolution_hours": round(resolution_hours, 1), - "csat_score": csat, - "created_at": date, - } -``` - -## Complete Example - -Save as `scripts/generate_data.py`: - -```python -"""Generate synthetic customer, order, and ticket data.""" -import numpy as np -import pandas as pd -from datetime import datetime, timedelta -from faker import Faker -import holidays -from pyspark.sql import SparkSession - -# ============================================================================= -# CONFIGURATION -# ============================================================================= -CATALOG = "my_catalog" -SCHEMA = "my_schema" -VOLUME_PATH = f"/Volumes/{CATALOG}/{SCHEMA}/raw_data" - -N_CUSTOMERS = 2500 -N_ORDERS = 25000 -N_TICKETS = 8000 - -# Date range - last 6 months from today -END_DATE = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) -START_DATE = END_DATE - timedelta(days=180) - -# Special events (within the date range) -INCIDENT_END = END_DATE - timedelta(days=21) -INCIDENT_START = INCIDENT_END - timedelta(days=10) - -# Holiday calendar -US_HOLIDAYS = holidays.US(years=[START_DATE.year, END_DATE.year]) - -SEED = 42 - -# ============================================================================= -# SETUP -# ============================================================================= -np.random.seed(SEED) -Faker.seed(SEED) -fake = Faker() -spark = SparkSession.builder.getOrCreate() - -# ============================================================================= -# CREATE INFRASTRUCTURE -# ============================================================================= -print(f"Creating catalog/schema/volume if needed...") -spark.sql(f"CREATE CATALOG IF NOT EXISTS {CATALOG}") -spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") -spark.sql(f"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.raw_data") - -print(f"Generating: {N_CUSTOMERS:,} customers, {N_ORDERS:,} orders, {N_TICKETS:,} tickets") - -# ============================================================================= -# 1. CUSTOMERS (Master Table) -# ============================================================================= -print("Generating customers...") - -customers_pdf = pd.DataFrame({ - "customer_id": [f"CUST-{i:05d}" for i in range(N_CUSTOMERS)], - "name": [fake.company() for _ in range(N_CUSTOMERS)], - "tier": np.random.choice(['Free', 'Pro', 'Enterprise'], N_CUSTOMERS, p=[0.6, 0.3, 0.1]), - "region": np.random.choice(['North', 'South', 'East', 'West'], N_CUSTOMERS, p=[0.4, 0.25, 0.2, 0.15]), -}) - -# ARR correlates with tier -customers_pdf["arr"] = customers_pdf["tier"].apply( - lambda t: round(np.random.lognormal(11, 0.5), 2) if t == 'Enterprise' - else round(np.random.lognormal(8, 0.6), 2) if t == 'Pro' else 0 -) - -# Lookups for foreign keys -customer_ids = customers_pdf["customer_id"].tolist() -customer_tier_map = dict(zip(customers_pdf["customer_id"], customers_pdf["tier"])) -tier_weights = customers_pdf["tier"].map({'Enterprise': 5.0, 'Pro': 2.0, 'Free': 1.0}) -customer_weights = (tier_weights / tier_weights.sum()).tolist() - -print(f" Created {len(customers_pdf):,} customers") - -# ============================================================================= -# 2. ORDERS (References Customers) -# ============================================================================= -print("Generating orders...") - -orders_data = [] -for i in range(N_ORDERS): - cid = np.random.choice(customer_ids, p=customer_weights) - tier = customer_tier_map[cid] - amount = np.random.lognormal(7 if tier == 'Enterprise' else 5 if tier == 'Pro' else 3.5, 0.7) - - orders_data.append({ - "order_id": f"ORD-{i:06d}", - "customer_id": cid, - "amount": round(amount, 2), - "status": np.random.choice(['completed', 'pending', 'cancelled'], p=[0.85, 0.10, 0.05]), - "order_date": fake.date_between(start_date=START_DATE, end_date=END_DATE), - }) - -orders_pdf = pd.DataFrame(orders_data) -print(f" Created {len(orders_pdf):,} orders") - -# ============================================================================= -# 3. TICKETS (References Customers, with incident spike) -# ============================================================================= -print("Generating tickets...") - -def get_daily_volume(date, base=25): - vol = base * (0.6 if date.weekday() >= 5 else 1.0) - if date in US_HOLIDAYS: - vol *= 0.3 # Even lower on holidays - if INCIDENT_START <= date <= INCIDENT_END: - vol *= 3.0 - return int(vol * np.random.normal(1, 0.15)) - -# Distribute tickets across dates -tickets_data = [] -ticket_idx = 0 -for day in pd.date_range(START_DATE, END_DATE): - daily_count = get_daily_volume(day.to_pydatetime()) - is_incident = INCIDENT_START <= day.to_pydatetime() <= INCIDENT_END - - for _ in range(daily_count): - if ticket_idx >= N_TICKETS: - break - - cid = np.random.choice(customer_ids, p=customer_weights) - tier = customer_tier_map[cid] - - # Category - Auth dominates during incident - if is_incident: - category = np.random.choice(['Auth', 'Network', 'Billing', 'Account'], p=[0.65, 0.15, 0.1, 0.1]) - else: - category = np.random.choice(['Auth', 'Network', 'Billing', 'Account'], p=[0.25, 0.30, 0.25, 0.20]) - - # Priority correlates with tier - priority = np.random.choice(['Critical', 'High', 'Medium'], p=[0.3, 0.5, 0.2]) if tier == 'Enterprise' \ - else np.random.choice(['Critical', 'High', 'Medium', 'Low'], p=[0.05, 0.2, 0.45, 0.3]) - - # Resolution time correlates with priority - res_scale = {'Critical': 4, 'High': 12, 'Medium': 36, 'Low': 72} - resolution = np.random.exponential(scale=res_scale[priority]) - - # CSAT degrades during incident for Auth - if is_incident and category == 'Auth': - csat = np.random.choice([1, 2, 3, 4, 5], p=[0.15, 0.25, 0.35, 0.2, 0.05]) - else: - csat = 5 if resolution < 4 else (4 if resolution < 12 else np.random.choice([2, 3, 4], p=[0.2, 0.5, 0.3])) - - tickets_data.append({ - "ticket_id": f"TKT-{ticket_idx:06d}", - "customer_id": cid, - "category": category, - "priority": priority, - "resolution_hours": round(resolution, 1), - "csat_score": csat, - "created_at": day.strftime("%Y-%m-%d"), - }) - ticket_idx += 1 - - if ticket_idx >= N_TICKETS: - break - -tickets_pdf = pd.DataFrame(tickets_data) -print(f" Created {len(tickets_pdf):,} tickets") - -# ============================================================================= -# 4. SAVE TO VOLUME -# ============================================================================= -print(f"\nSaving to {VOLUME_PATH}...") - -spark.createDataFrame(customers_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/customers") -spark.createDataFrame(orders_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/orders") -spark.createDataFrame(tickets_pdf).write.mode("overwrite").parquet(f"{VOLUME_PATH}/tickets") - -print("Done!") - -# ============================================================================= -# 5. VALIDATION -# ============================================================================= -print("\n=== VALIDATION ===") -print(f"Tier distribution: {customers_pdf['tier'].value_counts(normalize=True).to_dict()}") -print(f"Avg order by tier: {orders_pdf.merge(customers_pdf[['customer_id', 'tier']]).groupby('tier')['amount'].mean().to_dict()}") - -incident_tickets = tickets_pdf[tickets_pdf['created_at'].between( - INCIDENT_START.strftime("%Y-%m-%d"), INCIDENT_END.strftime("%Y-%m-%d") -)] -print(f"Incident period tickets: {len(incident_tickets):,} ({len(incident_tickets)/len(tickets_pdf)*100:.1f}%)") -print(f"Incident Auth %: {(incident_tickets['category'] == 'Auth').mean()*100:.1f}%") -``` - -Execute using `run_python_file_on_databricks` tool: -- `file_path`: "scripts/generate_data.py" - -If it fails, edit the file and re-run with the same `cluster_id` and `context_id`. - -### Validate Generated Data - -After successful execution, use `get_volume_folder_details` tool to verify the generated data: -- `volume_path`: "my_catalog/my_schema/raw_data/customers" -- `format`: "parquet" -- `table_stat_level`: "SIMPLE" - -This returns schema, row counts, and column statistics to confirm the data was written correctly. - -## Best Practices - -1. **Ask for schema**: Default to `ai_dev_kit` catalog, ask user for schema name -2. **Create infrastructure**: Use `CREATE CATALOG/SCHEMA/VOLUME IF NOT EXISTS` -3. **Raw data only**: No `total_x`, `sum_x`, `avg_x` fields - SDP pipeline computes those -4. **Save to Volume, not tables**: Write parquet to `/Volumes/{catalog}/{schema}/raw_data/` -5. **Configuration at top**: All sizes, dates, and paths as variables -6. **Dynamic dates**: Use `datetime.now() - timedelta(days=180)` for last 6 months -7. **Pandas for generation**: Faster and easier than Spark for row-by-row logic -8. **Master tables first**: Generate customers, then orders reference customer_ids -9. **Weighted sampling**: Enterprise customers generate more activity -10. **Distributions**: Log-normal for values, exponential for times, weighted categorical -11. **Time patterns**: Weekday/weekend, holidays, seasonality, event spikes -12. **Row coherence**: Priority affects resolution time affects CSAT -13. **Volume for aggregation**: 10K-50K rows minimum so patterns survive GROUP BY -14. **Always use files**: Write to local file, execute, edit if error, re-execute -15. **Context reuse**: Pass `cluster_id` and `context_id` for faster iterations -16. **Libraries**: Install `faker` and `holidays` first; most others are pre-installed - -## Related Skills - -- **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - for building bronze/silver/gold pipelines on top of generated data -- **[databricks-aibi-dashboards](../databricks-aibi-dashboards/SKILL.md)** - for visualizing the generated data in dashboards -- **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - for managing catalogs, schemas, and volumes where data is stored diff --git a/.claude/skills/databricks-unity-catalog/6-volumes.md b/.claude/skills/databricks-unity-catalog/6-volumes.md index 1eae49a..497b609 100644 --- a/.claude/skills/databricks-unity-catalog/6-volumes.md +++ b/.claude/skills/databricks-unity-catalog/6-volumes.md @@ -39,69 +39,16 @@ All volume operations use the path format: ## MCP Tools -### List Files in Volume - -```python -# List files and directories -list_volume_files( - volume_path="/Volumes/main/default/my_volume/data/" -) -# Returns: [{"name": "file.csv", "path": "...", "is_directory": false, "file_size": 1024, "last_modified": "..."}] -``` - -### Upload File to Volume - -```python -# Upload a local file -upload_to_volume( - local_path="/tmp/data.csv", - volume_path="/Volumes/main/default/my_volume/data.csv", - overwrite=True -) -# Returns: {"local_path": "...", "volume_path": "...", "success": true} -``` - -### Download File from Volume - -```python -# Download to local path -download_from_volume( - volume_path="/Volumes/main/default/my_volume/data.csv", - local_path="/tmp/downloaded.csv", - overwrite=True -) -# Returns: {"volume_path": "...", "local_path": "...", "success": true} -``` - -### Create Directory - -```python -# Create directory (creates parents like mkdir -p) -create_volume_directory( - volume_path="/Volumes/main/default/my_volume/data/2024/01" -) -# Returns: {"volume_path": "...", "success": true} -``` - -### Delete File - -```python -# Delete a file -delete_volume_file( - volume_path="/Volumes/main/default/my_volume/old_data.csv" -) -# Returns: {"volume_path": "...", "success": true} -``` - -### Get File Info - -```python -# Get file metadata -get_volume_file_info( - volume_path="/Volumes/main/default/my_volume/data.csv" -) -# Returns: {"name": "data.csv", "file_size": 1024, "last_modified": "...", "success": true} -``` +| Tool | Usage | +|------|-------| +| `list_volume_files` | `list_volume_files(volume_path="/Volumes/catalog/schema/volume/path/")` | +| `get_volume_folder_details` | `get_volume_folder_details(volume_path="catalog/schema/volume/path", format="parquet")` - schema, row counts, stats | +| `upload_to_volume` | `upload_to_volume(local_path="/tmp/data/*", volume_path="/Volumes/.../dest")` - supports files, folders, globs | +| `download_from_volume` | `download_from_volume(volume_path="/Volumes/.../file.csv", local_path="/tmp/file.csv")` | +| `create_volume_directory` | `create_volume_directory(volume_path="/Volumes/.../new_folder")` - creates parents like `mkdir -p` | +| `delete_volume_file` | `delete_volume_file(volume_path="/Volumes/.../file.csv")` | +| `delete_volume_directory` | `delete_volume_directory(volume_path="/Volumes/.../folder")` - directory must be empty | +| `get_volume_file_info` | `get_volume_file_info(volume_path="/Volumes/.../file.csv")` - returns size, modified date | --- diff --git a/.claude/skills/databricks-unity-catalog/7-data-profiling.md b/.claude/skills/databricks-unity-catalog/7-data-profiling.md new file mode 100644 index 0000000..23a2b62 --- /dev/null +++ b/.claude/skills/databricks-unity-catalog/7-data-profiling.md @@ -0,0 +1,309 @@ +# Data Profiling (formerly Lakehouse Monitoring) + +Comprehensive reference for Data Profiling: create quality monitors on Unity Catalog tables to track data profiles, detect drift, and monitor ML model performance. + +## Overview + +Data profiling automatically computes statistical profiles and drift metrics for tables over time. When you create a monitor, Databricks generates two output Delta tables (profile metrics + drift metrics) and an optional dashboard. + +| Component | Description | +|-----------|-------------| +| **Monitor** | Configuration attached to a UC table | +| **Profile Metrics Table** | Summary statistics computed per column | +| **Drift Metrics Table** | Statistical drift compared to baseline or previous time window | +| **Dashboard** | Auto-generated visualization of metrics | + +### Requirements + +- Unity Catalog enabled workspace +- Databricks SQL access +- Privileges: `USE CATALOG`, `USE SCHEMA`, `SELECT`, and `MANAGE` on the table +- Only Delta tables supported (managed, external, views, materialized views, streaming tables) + +--- + +## Profile Types + +| Type | Use Case | Key Params | Limitations | +|------|----------|------------|-------------| +| **Snapshot** | General-purpose tables without time column | None required | Max 4TB table size | +| **TimeSeries** | Tables with a timestamp column | `timestamp_column`, `granularities` | Last 30 days only | +| **InferenceLog** | ML model monitoring | `timestamp_column`, `granularities`, `model_id_column`, `problem_type`, `prediction_column` | Last 30 days only | + +### Granularities (for TimeSeries and InferenceLog) + +Supported `AggregationGranularity` values: `AGGREGATION_GRANULARITY_5_MINUTES`, `AGGREGATION_GRANULARITY_30_MINUTES`, `AGGREGATION_GRANULARITY_1_HOUR`, `AGGREGATION_GRANULARITY_1_DAY`, `AGGREGATION_GRANULARITY_1_WEEK` – `AGGREGATION_GRANULARITY_4_WEEKS`, `AGGREGATION_GRANULARITY_1_MONTH`, `AGGREGATION_GRANULARITY_1_YEAR` + +--- + +## MCP Tools + +Use the `manage_uc_monitors` tool for all monitor operations: + +| Action | Description | +|--------|-------------| +| `create` | Create a quality monitor on a table | +| `get` | Get monitor details and status | +| `run_refresh` | Trigger a metric refresh | +| `list_refreshes` | List refresh history | +| `delete` | Delete the monitor (assets are not deleted) | + +### Create a Monitor + +> **Note:** The MCP tool currently only creates **snapshot** monitors. For TimeSeries or InferenceLog monitors, use the Python SDK directly (see below). + +```python +manage_uc_monitors( + action="create", + table_name="catalog.schema.my_table", + output_schema_name="catalog.schema", +) +``` + +### Get Monitor Status + +```python +manage_uc_monitors( + action="get", + table_name="catalog.schema.my_table", +) +``` + +### Trigger a Refresh + +```python +manage_uc_monitors( + action="run_refresh", + table_name="catalog.schema.my_table", +) +``` + +### Delete a Monitor + +```python +manage_uc_monitors( + action="delete", + table_name="catalog.schema.my_table", +) +``` + +--- + +## Python SDK Examples + +**Doc:** https://databricks-sdk-py.readthedocs.io/en/stable/workspace/dataquality/data_quality.html + +The new SDK provides full control over all profile types via `w.data_quality`. + +### Create Snapshot Monitor + +```python +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.dataquality import ( + Monitor, DataProfilingConfig, SnapshotConfig, +) + +w = WorkspaceClient() + +# Look up UUIDs — the new API uses object_id and output_schema_id (both UUIDs) +table_info = w.tables.get("catalog.schema.my_table") +schema_info = w.schemas.get(f"{table_info.catalog_name}.{table_info.schema_name}") + +monitor = w.data_quality.create_monitor( + monitor=Monitor( + object_type="table", + object_id=table_info.table_id, + data_profiling_config=DataProfilingConfig( + assets_dir="/Workspace/Users/user@example.com/monitoring/my_table", + output_schema_id=schema_info.schema_id, + snapshot=SnapshotConfig(), + ), + ), +) +print(f"Monitor status: {monitor.data_profiling_config.status}") +``` + +### Create TimeSeries Monitor + +```python +from databricks.sdk.service.dataquality import ( + Monitor, DataProfilingConfig, TimeSeriesConfig, AggregationGranularity, +) + +table_info = w.tables.get("catalog.schema.events") +schema_info = w.schemas.get(f"{table_info.catalog_name}.{table_info.schema_name}") + +monitor = w.data_quality.create_monitor( + monitor=Monitor( + object_type="table", + object_id=table_info.table_id, + data_profiling_config=DataProfilingConfig( + assets_dir="/Workspace/Users/user@example.com/monitoring/events", + output_schema_id=schema_info.schema_id, + time_series=TimeSeriesConfig( + timestamp_column="event_timestamp", + granularities=[AggregationGranularity.AGGREGATION_GRANULARITY_1_DAY], + ), + ), + ), +) +``` + +### Create InferenceLog Monitor + +```python +from databricks.sdk.service.dataquality import ( + Monitor, DataProfilingConfig, InferenceLogConfig, + AggregationGranularity, InferenceProblemType, +) + +table_info = w.tables.get("catalog.schema.model_predictions") +schema_info = w.schemas.get(f"{table_info.catalog_name}.{table_info.schema_name}") + +monitor = w.data_quality.create_monitor( + monitor=Monitor( + object_type="table", + object_id=table_info.table_id, + data_profiling_config=DataProfilingConfig( + assets_dir="/Workspace/Users/user@example.com/monitoring/predictions", + output_schema_id=schema_info.schema_id, + inference_log=InferenceLogConfig( + timestamp_column="prediction_timestamp", + granularities=[AggregationGranularity.AGGREGATION_GRANULARITY_1_HOUR], + model_id_column="model_version", + problem_type=InferenceProblemType.INFERENCE_PROBLEM_TYPE_CLASSIFICATION, + prediction_column="prediction", + label_column="label", + ), + ), + ), +) +``` + +### Schedule a Monitor + +```python +from databricks.sdk.service.dataquality import ( + Monitor, DataProfilingConfig, SnapshotConfig, CronSchedule, +) + +table_info = w.tables.get("catalog.schema.my_table") +schema_info = w.schemas.get(f"{table_info.catalog_name}.{table_info.schema_name}") + +monitor = w.data_quality.create_monitor( + monitor=Monitor( + object_type="table", + object_id=table_info.table_id, + data_profiling_config=DataProfilingConfig( + assets_dir="/Workspace/Users/user@example.com/monitoring/my_table", + output_schema_id=schema_info.schema_id, + snapshot=SnapshotConfig(), + schedule=CronSchedule( + quartz_cron_expression="0 0 12 * * ?", # Daily at noon + timezone_id="UTC", + ), + ), + ), +) +``` + +### Get, Refresh, and Delete + +```python +# Get monitor details +monitor = w.data_quality.get_monitor( + object_type="table", + object_id=table_info.table_id, +) + +# Trigger refresh +from databricks.sdk.service.dataquality import Refresh + +refresh = w.data_quality.create_refresh( + object_type="table", + object_id=table_info.table_id, + refresh=Refresh( + object_type="table", + object_id=table_info.table_id, + ), +) + +# Delete monitor (does not delete output tables or dashboard) +w.data_quality.delete_monitor( + object_type="table", + object_id=table_info.table_id, +) +``` + +--- + +## Anomaly Detection + +Anomaly detection is enabled at the **schema level**, not per table. Once enabled, Databricks automatically scans all tables in the schema at the same frequency they are updated. + +```python +from databricks.sdk.service.dataquality import Monitor, AnomalyDetectionConfig + +schema_info = w.schemas.get("catalog.schema") + +monitor = w.data_quality.create_monitor( + monitor=Monitor( + object_type="schema", + object_id=schema_info.schema_id, + anomaly_detection_config=AnomalyDetectionConfig(), + ), +) +``` + +> **Note:** Anomaly detection requires `MANAGE SCHEMA` or `MANAGE CATALOG` privileges and serverless compute enabled on the workspace. + +--- + +## Output Tables + +When a monitor is created, two metric tables are generated in the specified output schema: + +| Table | Naming Convention | Contents | +|-------|-------------------|----------| +| **Profile Metrics** | `{table_name}_profile_metrics` | Per-column statistics (nulls, min, max, mean, distinct count, etc.) | +| **Drift Metrics** | `{table_name}_drift_metrics` | Statistical tests comparing current vs. baseline or previous window | + +### Query Output Tables + +```sql +-- View latest profile metrics +SELECT * +FROM catalog.schema.my_table_profile_metrics +ORDER BY window_end DESC +LIMIT 100; + +-- View latest drift metrics +SELECT * +FROM catalog.schema.my_table_drift_metrics +ORDER BY window_end DESC +LIMIT 100; +``` + +--- + +## Common Issues + +| Issue | Cause | Solution | +|-------|-------|----------| +| `FEATURE_NOT_ENABLED` | Data profiling not enabled on workspace | Contact workspace admin to enable the feature | +| `PERMISSION_DENIED` | Missing `MANAGE` privilege on the table | Grant `MANAGE` on the table to your user/group | +| Monitor refresh stuck in `PENDING` | No SQL warehouse available | Ensure a SQL warehouse is running or set `warehouse_id` | +| Profile metrics table empty | Refresh has not completed yet | Check refresh state with `list_refreshes`; wait for `SUCCESS` | +| Snapshot monitor on large table fails | Table exceeds 4TB limit | Switch to TimeSeries profile type instead | +| TimeSeries shows limited data | Only processes last 30 days | Expected behavior; contact account team to adjust | + +--- + +> **Note:** Data profiling was formerly known as Lakehouse Monitoring. The legacy SDK accessor +> `w.lakehouse_monitors` and the MCP tool `manage_uc_monitors` still use the previous API. + +## Resources + +- [Data Quality Monitoring Documentation](https://docs.databricks.com/aws/en/data-quality-monitoring/) +- [Data Quality SDK Reference](https://databricks-sdk-py.readthedocs.io/en/stable/workspace/dataquality/data_quality.html) +- [Legacy Lakehouse Monitors SDK Reference](https://databricks-sdk-py.readthedocs.io/en/stable/workspace/catalog/lakehouse_monitors.html) diff --git a/.claude/skills/databricks-unity-catalog/SKILL.md b/.claude/skills/databricks-unity-catalog/SKILL.md index 9b77fed..2e3d05f 100644 --- a/.claude/skills/databricks-unity-catalog/SKILL.md +++ b/.claude/skills/databricks-unity-catalog/SKILL.md @@ -17,6 +17,7 @@ Use this skill when: - Tracking **compute resources** (cluster usage, warehouse metrics) - Reviewing **job execution** (run history, success rates, failures) - Analyzing **query performance** (slow queries, warehouse utilization) +- Profiling **data quality** (data profiling, drift detection, metric tables) ## Reference Files @@ -24,30 +25,19 @@ Use this skill when: |-------|------|-------------| | System Tables | [5-system-tables.md](5-system-tables.md) | Lineage, audit, billing, compute, jobs, query history | | Volumes | [6-volumes.md](6-volumes.md) | Volume file operations, permissions, best practices | +| Data Profiling | [7-data-profiling.md](7-data-profiling.md) | Data profiling, drift detection, profile metrics | ## Quick Start ### Volume File Operations (MCP Tools) -```python -# List files in a volume -list_volume_files(volume_path="/Volumes/catalog/schema/volume/folder/") - -# Upload file to volume -upload_to_volume( - local_path="/tmp/data.csv", - volume_path="/Volumes/catalog/schema/volume/data.csv" -) - -# Download file from volume -download_from_volume( - volume_path="/Volumes/catalog/schema/volume/data.csv", - local_path="/tmp/downloaded.csv" -) - -# Create directory -create_volume_directory(volume_path="/Volumes/catalog/schema/volume/new_folder") -``` +| Tool | Usage | +|------|-------| +| `list_volume_files` | `list_volume_files(volume_path="/Volumes/catalog/schema/volume/path/")` | +| `get_volume_folder_details` | `get_volume_folder_details(volume_path="catalog/schema/volume/path", format="parquet")` - schema, row counts, stats | +| `upload_to_volume` | `upload_to_volume(local_path="/tmp/data/*", volume_path="/Volumes/.../dest")` | +| `download_from_volume` | `download_from_volume(volume_path="/Volumes/.../file.csv", local_path="/tmp/file.csv")` | +| `create_volume_directory` | `create_volume_directory(volume_path="/Volumes/.../new_folder")` | ### Enable System Tables Access @@ -108,7 +98,7 @@ mcp__databricks__execute_sql( - **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - for pipelines that write to Unity Catalog tables - **[databricks-jobs](../databricks-jobs/SKILL.md)** - for job execution data visible in system tables -- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - for generating data stored in Unity Catalog Volumes +- **[databricks-synthetic-data-gen](../databricks-synthetic-data-gen/SKILL.md)** - for generating data stored in Unity Catalog Volumes - **[databricks-aibi-dashboards](../databricks-aibi-dashboards/SKILL.md)** - for building dashboards on top of Unity Catalog data ## Resources diff --git a/.claude/skills/databricks-unstructured-pdf-generation/SKILL.md b/.claude/skills/databricks-unstructured-pdf-generation/SKILL.md index 7666f21..92322fd 100644 --- a/.claude/skills/databricks-unstructured-pdf-generation/SKILL.md +++ b/.claude/skills/databricks-unstructured-pdf-generation/SKILL.md @@ -1,194 +1,337 @@ --- name: databricks-unstructured-pdf-generation -description: "Generate synthetic PDF documents for RAG and unstructured data use cases. Use when creating test PDFs, demo documents, or evaluation datasets for retrieval systems." +description: "Generate PDF documents from HTML and upload to Unity Catalog volumes. Use for creating test PDFs, demo documents, reports, or evaluation datasets." --- -# Unstructured PDF Generation +# PDF Generation from HTML -Generate realistic synthetic PDF documents using LLM for RAG (Retrieval-Augmented Generation) and unstructured data use cases. +Convert HTML content to PDF documents and upload them to Unity Catalog Volumes. ## Overview -This skill uses the `generate_pdf_documents` MCP tool to create professional PDF documents with: -- LLM-generated content based on your description -- Accompanying JSON files with questions and evaluation guidelines (for RAG testing) -- Automatic upload to Unity Catalog Volumes +The `generate_and_upload_pdf` MCP tool converts HTML to PDF and uploads to a Unity Catalog Volume. You (the LLM) generate the HTML content, and the tool handles conversion and upload. -## Quick Start - -Use the `generate_pdf_documents` MCP tool: -- `catalog`: "my_catalog" -- `schema`: "my_schema" -- `description`: "Technical documentation for a cloud infrastructure platform including setup guides, troubleshooting procedures, and API references." -- `count`: 10 - -This generates 10 PDF documents and saves them to `/Volumes/my_catalog/my_schema/raw_data/pdf_documents/` (using default volume and folder). - -### With Custom Location - -Use the `generate_pdf_documents` MCP tool: -- `catalog`: "my_catalog" -- `schema`: "my_schema" -- `description`: "HR policy documents..." -- `count`: 10 -- `volume`: "custom_volume" -- `folder`: "hr_policies" -- `overwrite_folder`: true - -## Parameters - -| Parameter | Type | Required | Default | Description | -|-----------|------|----------|---------|-------------| -| `catalog` | string | Yes | - | Unity Catalog name | -| `schema` | string | Yes | - | Schema name | -| `description` | string | Yes | - | Detailed description of what PDFs should contain | -| `count` | int | Yes | - | Number of PDFs to generate | -| `volume` | string | No | `raw_data` | Volume name (created if not exists) | -| `folder` | string | No | `pdf_documents` | Folder within volume for output files | -| `doc_size` | string | No | `MEDIUM` | Document size: `SMALL` (~1 page), `MEDIUM` (~5 pages), `LARGE` (~10+ pages) | -| `overwrite_folder` | bool | No | `false` | If true, deletes existing folder contents first | - -### Document Size Guide +## Tool Signature -- **SMALL**: ~1 page, concise content. Best for quick demos or testing. -- **MEDIUM**: ~4-6 pages, comprehensive coverage. Good balance for most use cases. -- **LARGE**: ~10+ pages, exhaustive documentation. Use for thorough RAG evaluation. - -## Output Files - -For each document, the tool creates two files: - -1. **PDF file** (`.pdf`): The generated document -2. **JSON file** (`.json`): Metadata for RAG evaluation - -### JSON Structure +``` +generate_and_upload_pdf( + html_content: str, # Complete HTML document + filename: str, # PDF filename (e.g., "report.pdf") + catalog: str, # Unity Catalog name + schema: str, # Schema name + volume: str = "raw_data", # Volume name (default: "raw_data") + folder: str = None, # Optional subfolder +) +``` +**Returns:** ```json { - "title": "API Authentication Guide", - "category": "Technical", - "pdf_path": "/Volumes/catalog/schema/volume/folder/doc_001.pdf", - "question": "What authentication methods are supported by the API?", - "guideline": "Answer should mention OAuth 2.0, API keys, and JWT tokens with their use cases." + "success": true, + "volume_path": "/Volumes/catalog/schema/volume/filename.pdf", + "error": null } ``` -## Common Patterns +## Quick Start -### Pattern 1: HR Policy Documents +Generate a simple PDF: -Use the `generate_pdf_documents` MCP tool: -- `catalog`: "ai_dev_kit" -- `schema`: "hr_demo" -- `description`: "HR policy documents for a technology company including employee handbook, leave policies, performance review procedures, benefits guide, and workplace conduct guidelines." -- `count`: 15 -- `folder`: "hr_policies" -- `overwrite_folder`: true +``` +generate_and_upload_pdf( + html_content=''' + + + + + +

Quarterly Report Q1 2024

+
+

Executive Summary

+

Revenue increased 15% year-over-year...

+
+ +''', + filename="q1_report.pdf", + catalog="my_catalog", + schema="my_schema" +) +``` -### Pattern 2: Technical Documentation +## Performance: Generate Multiple PDFs in Parallel -Use the `generate_pdf_documents` MCP tool: -- `catalog`: "ai_dev_kit" -- `schema`: "tech_docs" -- `description`: "Technical documentation for a SaaS analytics platform including installation guides, API references, troubleshooting procedures, security best practices, and integration tutorials." -- `count`: 20 -- `folder`: "product_docs" -- `overwrite_folder`: true +**IMPORTANT**: PDF generation and upload can take 2-5 seconds per document. When generating multiple PDFs, **call the tool in parallel** to maximize throughput. -### Pattern 3: Financial Reports +### Example: Generate 5 PDFs in Parallel -Use the `generate_pdf_documents` MCP tool: -- `catalog`: "ai_dev_kit" -- `schema`: "finance_demo" -- `description`: "Financial documents for a retail company including quarterly reports, expense policies, budget guidelines, and audit procedures." -- `count`: 12 -- `folder`: "reports" -- `overwrite_folder`: true +Make 5 simultaneous `generate_and_upload_pdf` calls: -### Pattern 4: Training Materials +``` +# Call 1 +generate_and_upload_pdf( + html_content="...Employee Handbook content...", + filename="employee_handbook.pdf", + catalog="hr_catalog", schema="policies", folder="2024" +) + +# Call 2 (parallel) +generate_and_upload_pdf( + html_content="...Leave Policy content...", + filename="leave_policy.pdf", + catalog="hr_catalog", schema="policies", folder="2024" +) + +# Call 3 (parallel) +generate_and_upload_pdf( + html_content="...Code of Conduct content...", + filename="code_of_conduct.pdf", + catalog="hr_catalog", schema="policies", folder="2024" +) + +# Call 4 (parallel) +generate_and_upload_pdf( + html_content="...Benefits Guide content...", + filename="benefits_guide.pdf", + catalog="hr_catalog", schema="policies", folder="2024" +) + +# Call 5 (parallel) +generate_and_upload_pdf( + html_content="...Remote Work Policy content...", + filename="remote_work_policy.pdf", + catalog="hr_catalog", schema="policies", folder="2024" +) +``` -Use the `generate_pdf_documents` MCP tool: -- `catalog`: "ai_dev_kit" -- `schema`: "training" -- `description`: "Training materials for new software developers including onboarding guides, coding standards, code review procedures, and deployment workflows." -- `count`: 8 -- `folder`: "courses" -- `overwrite_folder`: true +By calling these in parallel (not sequentially), 5 PDFs that would take 15-25 seconds sequentially complete in 3-5 seconds total. -## Workflow +## HTML Best Practices -1. **Ask for destination**: Default to `ai_dev_kit` catalog, ask user for schema name -2. **Get description**: Ask what kind of documents they need -3. **Generate PDFs**: Call `generate_pdf_documents` MCP tool with appropriate parameters -4. **Verify output**: Check the volume path for generated files +### Use Complete HTML5 Structure -## Best Practices +Always include the full HTML structure: -1. **Detailed descriptions**: The more specific your description, the better the generated content - - BAD: "Generate some HR documents" - - GOOD: "HR policy documents for a technology company including employee handbook covering remote work policies, leave policies with PTO and sick leave details, performance review procedures with quarterly and annual cycles, and workplace conduct guidelines" +```html + + + + + + + + + +``` -2. **Appropriate count**: - - For demos: 5-10 documents - - For RAG testing: 15-30 documents - - For comprehensive evaluation: 50+ documents +### CSS Features Supported + +PlutoPrint supports modern CSS3: +- Flexbox and Grid layouts +- CSS variables (`--var-name`) +- Web fonts (system fonts recommended) +- Colors, backgrounds, borders +- Tables with styling + +### CSS to Avoid + +- Animations and transitions (static PDF) +- Interactive elements (forms, hover effects) +- External resources (images via URL) - use embedded base64 if needed + +### Professional Document Template + +```html + + + + + + +

Document Title

+ +

Section 1

+

Content here...

+ +
+ Important: Key information highlighted here. +
+ +

Data Table

+
+ + +
Column 1Column 2Column 3
DataDataData
+ +
+ Generated on 2024-01-15 | Confidential +
+ + +``` -3. **Folder organization**: Use descriptive folder names that indicate content type - - `hr_policies/` - - `technical_docs/` - - `training_materials/` +## Common Patterns -4. **Use overwrite_folder**: Set to `true` when regenerating to ensure clean state +### Pattern 1: Technical Documentation -## Integration with RAG Pipelines +Generate API documentation, user guides, or technical specs: -The generated JSON files are designed for RAG evaluation: +``` +generate_and_upload_pdf( + html_content=''' + + + +

API Reference

+
+ GET /api/v1/users +

Returns a list of all users.

+
+

Request Headers

+ 
Authorization: Bearer {token}
+Content-Type: application/json
+ +''', + filename="api_reference.pdf", + catalog="docs_catalog", + schema="api_docs" +) +``` -1. **Ingest PDFs**: Use the PDF files as source documents for your vector database -2. **Test retrieval**: Use the `question` field to query your RAG system -3. **Evaluate answers**: Use the `guideline` field to assess if the RAG response is correct +### Pattern 2: Business Reports -Example evaluation workflow: -```python -# Load questions from JSON files -questions = load_json_files(f"/Volumes/{catalog}/{schema}/{volume}/{folder}/*.json") +``` +generate_and_upload_pdf( + html_content=''' + + + +

Q1 2024 Performance Report

+
+
$2.4M
+
Revenue
+
+
+
+15%
+
Growth
+
+ +''', + filename="q1_2024_report.pdf", + catalog="finance", + schema="reports", + folder="quarterly" +) +``` -for q in questions: - # Query RAG system - response = rag_system.query(q["question"]) +### Pattern 3: HR Policies - # Evaluate using guideline - is_correct = evaluate_response(response, q["guideline"]) +``` +generate_and_upload_pdf( + html_content=''' + + + +

Employee Leave Policy

+

Effective: January 1, 2024

+ +
+

1. Annual Leave

+

All full-time employees are entitled to 20 days of paid annual leave per calendar year.

+
+ +
+ Note: Leave requests must be submitted at least 2 weeks in advance. +
+ +''', + filename="leave_policy.pdf", + catalog="hr_catalog", + schema="policies" +) ``` -## Environment Configuration +## Workflow for Multiple Documents -The tool requires LLM configuration via environment variables: +When asked to generate multiple PDFs: -```bash -# Databricks Foundation Models (default) -LLM_PROVIDER=DATABRICKS -DATABRICKS_MODEL=databricks-meta-llama-3-3-70b-instruct +1. **Plan the documents**: Determine titles, content structure for each +2. **Generate HTML for each**: Create complete HTML documents +3. **Call tool in parallel**: Make multiple simultaneous `generate_and_upload_pdf` calls +4. **Report results**: Summarize successful uploads and any errors -# Or Azure OpenAI -LLM_PROVIDER=AZURE -AZURE_OPENAI_ENDPOINT=https://your-resource.cognitiveservices.azure.com/ -AZURE_OPENAI_API_KEY=your-api-key -AZURE_OPENAI_DEPLOYMENT=gpt-4o -``` +## Prerequisites + +- Unity Catalog schema must exist +- Volume must exist (default: `raw_data`) +- User must have WRITE permission on the volume -## Common Issues +## Troubleshooting | Issue | Solution | |-------|----------| -| **"No LLM endpoint configured"** | Set `DATABRICKS_MODEL` or `AZURE_OPENAI_DEPLOYMENT` environment variable | -| **"Volume does not exist"** | The tool creates volumes automatically; ensure you have CREATE VOLUME permission | -| **"PDF generation timeout"** | Reduce `count` or check LLM endpoint availability | -| **Low quality content** | Provide more detailed `description` with specific topics and document types | - -## Related Skills - -- **[databricks-agent-bricks](../databricks-agent-bricks/SKILL.md)** - Create Knowledge Assistants that ingest the generated PDFs -- **[databricks-vector-search](../databricks-vector-search/SKILL.md)** - Index generated documents for semantic search and RAG -- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - Generate structured tabular data (complement to unstructured PDFs) -- **[databricks-mlflow-evaluation](../databricks-mlflow-evaluation/SKILL.md)** - Evaluate RAG systems using the generated question/guideline pairs +| "Volume does not exist" | Create the volume first or use an existing one | +| "Schema does not exist" | Create the schema or check the name | +| PDF looks wrong | Check HTML/CSS syntax, use supported CSS features | +| Slow generation | Call multiple PDFs in parallel, not sequentially | diff --git a/.claude/skills/databricks-vector-search/SKILL.md b/.claude/skills/databricks-vector-search/SKILL.md index 276cab3..72068ec 100644 --- a/.claude/skills/databricks-vector-search/SKILL.md +++ b/.claude/skills/databricks-vector-search/SKILL.md @@ -31,8 +31,8 @@ Databricks Vector Search provides managed vector similarity search with automati | Type | Latency | Capacity | Cost | Best For | |------|---------|----------|------|----------| -| **Standard** | ~50-100ms | 320M vectors (768 dim) | Higher | Real-time, low-latency | -| **Storage-Optimized** | ~250ms | 1B+ vectors (768 dim) | 7x lower | Large-scale, cost-sensitive | +| **Standard** | 20-50ms | 320M vectors (768 dim) | Higher | Real-time, low-latency | +| **Storage-Optimized** | 300-500ms | 1B+ vectors (768 dim) | 7x lower | Large-scale, cost-sensitive | ## Index Types @@ -184,13 +184,15 @@ results = w.vector_search_indexes.query_index( ### Hybrid Search (Semantic + Keyword) +Hybrid search combines vector similarity (ANN) with BM25 keyword scoring. Use it when queries contain exact terms that must match — SKUs, error codes, proper nouns, or technical terminology — where pure semantic search might miss keyword-specific results. See [search-modes.md](search-modes.md) for detailed guidance on choosing between ANN and hybrid search. + ```python # Combines vector similarity with keyword matching results = w.vector_search_indexes.query_index( index_name="catalog.schema.my_index", columns=["id", "content"], - query_text="machine learning algorithms", - query_type="hybrid", # Enable hybrid search + query_text="SPARK-12345 executor memory error", + query_type="HYBRID", num_results=10 ) ``` @@ -212,20 +214,26 @@ results = w.vector_search_indexes.query_index( ### Storage-Optimized Filters (SQL-like) +Storage-Optimized endpoints use SQL-like filter syntax via the `databricks-vectorsearch` package's `filters` parameter (accepts a string): + ```python -# filter_string uses SQL-like syntax -results = w.vector_search_indexes.query_index( - index_name="catalog.schema.my_index", - columns=["id", "content"], +from databricks.vector_search.client import VectorSearchClient + +vsc = VectorSearchClient() +index = vsc.get_index(endpoint_name="my-storage-endpoint", index_name="catalog.schema.my_index") + +# SQL-like filter syntax for storage-optimized endpoints +results = index.similarity_search( query_text="machine learning", + columns=["id", "content"], num_results=10, - filter_string="category = 'ai' AND status IN ('active', 'pending')" + filters="category = 'ai' AND status IN ('active', 'pending')" ) # More filter examples -filter_string="price > 100 AND price < 500" -filter_string="department LIKE 'eng%'" -filter_string="created_at >= '2024-01-01'" +# filters="price > 100 AND price < 500" +# filters="department LIKE 'eng%'" +# filters="created_at >= '2024-01-01'" ``` ### Trigger Index Sync @@ -249,7 +257,12 @@ scan_result = w.vector_search_indexes.scan_index( ## Reference Files -- [index-types.md](index-types.md) - Detailed comparison of index types and creation patterns +| Topic | File | Description | +|-------|------|-------------| +| Index Types | [index-types.md](index-types.md) | Detailed comparison of Delta Sync (managed/self-managed) vs Direct Access | +| End-to-End RAG | [end-to-end-rag.md](end-to-end-rag.md) | Complete walkthrough: source table → endpoint → index → query → agent integration | +| Search Modes | [search-modes.md](search-modes.md) | When to use semantic (ANN) vs hybrid search, decision guide | +| Operations | [troubleshooting-and-operations.md](troubleshooting-and-operations.md) | Monitoring, cost optimization, capacity planning, migration | ## CLI Quick Reference @@ -285,19 +298,20 @@ databricks vector-search indexes delete-index \ |-------|----------| | **Index sync slow** | Use Storage-Optimized endpoints (20x faster indexing) | | **Query latency high** | Use Standard endpoint for <100ms latency | -| **filters_json not working** | Storage-Optimized uses `filter_string` (SQL syntax) | +| **filters_json not working** | Storage-Optimized uses SQL-like string filters via `databricks-vectorsearch` package's `filters` parameter | | **Embedding dimension mismatch** | Ensure query and index dimensions match | | **Index not updating** | Check pipeline_type; use sync_index() for TRIGGERED | | **Out of capacity** | Upgrade to Storage-Optimized (1B+ vectors) | +| **`query_vector` truncated by MCP tool** | MCP tool calls serialize arrays as JSON and can truncate large vectors (e.g. 1024-dim). Use `query_text` instead (for managed embedding indexes), or use the Databricks SDK/CLI to pass raw vectors | ## Embedding Models Databricks provides built-in embedding models: -| Model | Dimensions | Use Case | -|-------|------------|----------| -| `databricks-gte-large-en` | 1024 | English text, high quality | -| `databricks-bge-large-en` | 1024 | English text, general | +| Model | Dimensions | Context Window | Use Case | +|-------|------------|----------------|----------| +| `databricks-gte-large-en` | 1024 | 8192 tokens | English text, high quality | +| `databricks-bge-large-en` | 1024 | 512 tokens | English text, general purpose | ```python # Use with managed embeddings @@ -311,42 +325,118 @@ embedding_source_columns=[ ## MCP Tools -The following MCP tools are available for managing Vector Search infrastructure. These are **management tools** for creating and configuring endpoints/indexes. For agent-runtime querying, use the Databricks managed Vector Search MCP server or `VectorSearchRetrieverTool`. +The following MCP tools are available for managing Vector Search infrastructure. For a full end-to-end walkthrough, see [end-to-end-rag.md](end-to-end-rag.md). + +### manage_vs_endpoint - Endpoint Management + +| Action | Description | Required Params | +|--------|-------------|-----------------| +| `create_or_update` | Create endpoint (STANDARD or STORAGE_OPTIMIZED). Idempotent | name | +| `get` | Get endpoint details | name | +| `list` | List all endpoints | (none) | +| `delete` | Delete endpoint (indexes must be deleted first) | name | + +```python +# Create or update an endpoint +result = manage_vs_endpoint(action="create_or_update", name="my-vs-endpoint", endpoint_type="STANDARD") +# Returns {"name": "my-vs-endpoint", "endpoint_type": "STANDARD", "created": True} + +# List all endpoints +endpoints = manage_vs_endpoint(action="list") + +# Get specific endpoint +endpoint = manage_vs_endpoint(action="get", name="my-vs-endpoint") +``` + +### manage_vs_index - Index Management + +| Action | Description | Required Params | +|--------|-------------|-----------------| +| `create_or_update` | Create index. Idempotent, auto-triggers sync for DELTA_SYNC | name, endpoint_name, primary_key | +| `get` | Get index details | name | +| `list` | List indexes. Optional endpoint_name filter | (none) | +| `delete` | Delete index | name | + +```python +# Create a Delta Sync index with managed embeddings +result = manage_vs_index( + action="create_or_update", + name="catalog.schema.my_index", + endpoint_name="my-vs-endpoint", + primary_key="id", + index_type="DELTA_SYNC", + delta_sync_index_spec={ + "source_table": "catalog.schema.docs", + "embedding_source_columns": [{"name": "content", "embedding_model_endpoint_name": "databricks-gte-large-en"}], + "pipeline_type": "TRIGGERED" + } +) + +# Get a specific index +index = manage_vs_index(action="get", name="catalog.schema.my_index") + +# List all indexes on an endpoint +indexes = manage_vs_index(action="list", endpoint_name="my-vs-endpoint") + +# List all indexes across all endpoints +all_indexes = manage_vs_index(action="list") +``` + +### query_vs_index - Query (Hot Path) + +Query index with `query_text`, `query_vector`, or hybrid (`query_type="HYBRID"`). Prefer `query_text` over `query_vector` — MCP tool calls can truncate large embedding arrays (1024-dim). + +```python +# Query an index +results = query_vs_index( + index_name="catalog.schema.my_index", + columns=["id", "content"], + query_text="machine learning best practices", + num_results=5 +) -### Endpoint Management +# Hybrid search (combines vector + keyword) +results = query_vs_index( + index_name="catalog.schema.my_index", + columns=["id", "content"], + query_text="SPARK-12345 memory error", + query_type="HYBRID", + num_results=10 +) +``` -| Tool | Description | -|------|-------------| -| `create_vs_endpoint` | Create a Vector Search endpoint (STANDARD or STORAGE_OPTIMIZED) | -| `get_vs_endpoint` | Get endpoint status and details | -| `list_vs_endpoints` | List all endpoints in the workspace | -| `delete_vs_endpoint` | Delete an endpoint (indexes must be deleted first) | +### manage_vs_data - Data Operations -### Index Management +| Action | Description | Required Params | +|--------|-------------|-----------------| +| `upsert` | Insert/update records | index_name, inputs_json | +| `delete` | Delete by primary key | index_name, primary_keys | +| `scan` | Scan index contents | index_name | +| `sync` | Trigger sync for TRIGGERED indexes | index_name | -| Tool | Description | -|------|-------------| -| `create_vs_index` | Create a Delta Sync or Direct Access index | -| `get_vs_index` | Get index status and configuration | -| `list_vs_indexes` | List all indexes on an endpoint | -| `delete_vs_index` | Delete an index | -| `sync_vs_index` | Trigger sync for TRIGGERED pipeline indexes | +```python +# Upsert data into a Direct Access index +manage_vs_data( + action="upsert", + index_name="catalog.schema.my_index", + inputs_json=[{"id": "doc1", "content": "...", "embedding": [0.1, 0.2, ...]}] +) -### Query and Data +# Trigger manual sync for a TRIGGERED pipeline index +manage_vs_data(action="sync", index_name="catalog.schema.my_index") -| Tool | Description | -|------|-------------| -| `query_vs_index` | Query index with text, vector, or hybrid search (for testing) | -| `upsert_vs_data` | Upsert vectors into a Direct Access index | -| `delete_vs_data` | Delete vectors from a Direct Access index | -| `scan_vs_index` | Scan/export index entries (for debugging) | +# Scan index contents +manage_vs_data(action="scan", index_name="catalog.schema.my_index", num_results=100) +``` ## Notes -- **Storage-Optimized is newer** - Better for most use cases unless you need <100ms latency -- **Delta Sync recommended** - Easier than Direct Access for most scenarios -- **Hybrid search** - Available for both Delta Sync and Direct Access indexes -- **Management vs runtime** - MCP tools above handle lifecycle management; for agent tool-calling at runtime, use the Databricks managed Vector Search MCP server +- **Storage-Optimized is newer** — better for most use cases unless you need <100ms latency +- **Delta Sync recommended** — easier than Direct Access for most scenarios +- **Hybrid search** — available for both Delta Sync and Direct Access indexes +- **`columns_to_sync` matters** — only synced columns are available in query results; include all columns you need +- **Filter syntax differs by endpoint** — Standard uses dict-format filters, Storage-Optimized uses SQL-like string filters. Use the `databricks-vectorsearch` package's `filters` parameter which accepts both formats +- **Management vs runtime** — MCP tools above handle lifecycle management; for agent tool-calling at runtime, use `VectorSearchRetrieverTool` or the Databricks managed Vector Search MCP server ## Related Skills diff --git a/.claude/skills/databricks-vector-search/end-to-end-rag.md b/.claude/skills/databricks-vector-search/end-to-end-rag.md new file mode 100644 index 0000000..a3808d1 --- /dev/null +++ b/.claude/skills/databricks-vector-search/end-to-end-rag.md @@ -0,0 +1,241 @@ +# End-to-End RAG with Vector Search + +Build a complete Retrieval-Augmented Generation pipeline: prepare documents, create a vector index, query it, and wire it into an agent. + +## MCP Tools Used + +| Tool | Step | +|------|------| +| `execute_sql` | Create source table, insert documents | +| `manage_vs_endpoint(action="create")` | Create compute endpoint | +| `manage_vs_index(action="create")` | Create Delta Sync index with managed embeddings | +| `manage_vs_index(action="sync")` | Trigger index sync | +| `manage_vs_index(action="get")` | Check index status | +| `query_vs_index` | Test similarity search | + +--- + +## Step 1: Prepare Source Table + +The source Delta table needs a primary key column and a text column to embed. + +```sql +CREATE TABLE IF NOT EXISTS catalog.schema.knowledge_base ( + doc_id STRING, + title STRING, + content STRING, + category STRING, + updated_at TIMESTAMP DEFAULT current_timestamp() +); + +INSERT INTO catalog.schema.knowledge_base VALUES +('doc-001', 'Getting Started', 'Databricks is a unified analytics platform...', 'overview', current_timestamp()), +('doc-002', 'Unity Catalog', 'Unity Catalog provides centralized governance...', 'governance', current_timestamp()), +('doc-003', 'Delta Lake', 'Delta Lake is an open-source storage layer...', 'storage', current_timestamp()); +``` + +Or via MCP: + +```python +execute_sql(sql_query=""" + CREATE TABLE IF NOT EXISTS catalog.schema.knowledge_base ( + doc_id STRING, + title STRING, + content STRING, + category STRING, + updated_at TIMESTAMP DEFAULT current_timestamp() + ) +""") +``` + +## Step 2: Create Vector Search Endpoint + +```python +manage_vs_endpoint( + action="create", + name="my-rag-endpoint", + endpoint_type="STORAGE_OPTIMIZED" +) +``` + +Endpoint creation is asynchronous. Check status: + +```python +manage_vs_endpoint(action="get", name="my-rag-endpoint") +# Wait for state: "ONLINE" +``` + +## Step 3: Create Delta Sync Index + +```python +manage_vs_index( + action="create", + name="catalog.schema.knowledge_base_index", + endpoint_name="my-rag-endpoint", + primary_key="doc_id", + index_type="DELTA_SYNC", + delta_sync_index_spec={ + "source_table": "catalog.schema.knowledge_base", + "embedding_source_columns": [ + { + "name": "content", + "embedding_model_endpoint_name": "databricks-gte-large-en" + } + ], + "pipeline_type": "TRIGGERED", + "columns_to_sync": ["doc_id", "title", "content", "category"] + } +) +``` + +Key decisions: +- **`embedding_source_columns`**: Databricks computes embeddings automatically from the `content` column +- **`pipeline_type`**: `TRIGGERED` for manual sync (cheaper), `CONTINUOUS` for auto-sync on table changes +- **`columns_to_sync`**: Only sync columns you need in query results (reduces storage and improves performance) + +## Step 4: Sync and Verify + +```python +# Trigger initial sync +manage_vs_index(action="sync", index_name="catalog.schema.knowledge_base_index") + +# Check status +manage_vs_index(action="get", index_name="catalog.schema.knowledge_base_index") +# Wait for state: "ONLINE" +``` + +## Step 5: Query the Index + +```python +# Semantic search +query_vs_index( + index_name="catalog.schema.knowledge_base_index", + columns=["doc_id", "title", "content", "category"], + query_text="How do I govern my data?", + num_results=3 +) +``` + +### With Filters + +The filter syntax depends on the endpoint type used when creating the index. + +```python +# Storage-Optimized endpoint (used in this walkthrough): SQL-like filter syntax +query_vs_index( + index_name="catalog.schema.knowledge_base_index", + columns=["doc_id", "title", "content"], + query_text="How do I govern my data?", + num_results=3, + filters="category = 'governance'" +) + +# Standard endpoint (if you created a Standard endpoint instead): JSON filters_json +query_vs_index( + index_name="catalog.schema.my_standard_index", + columns=["doc_id", "title", "content"], + query_text="How do I govern my data?", + num_results=3, + filters_json='{"category": "governance"}' +) +``` + +### Hybrid Search (Vector + Keyword) + +```python +query_vs_index( + index_name="catalog.schema.knowledge_base_index", + columns=["doc_id", "title", "content"], + query_text="Delta Lake ACID transactions", + num_results=5, + query_type="HYBRID" +) +``` + +--- + +## Step 6: Use in an Agent + +### As a Tool in a ChatAgent + +Use `VectorSearchRetrieverTool` to wire the index into an agent deployed on Model Serving: + +```python +from databricks.agents import ChatAgent +from databricks.agents.tools import VectorSearchRetrieverTool +from databricks.sdk import WorkspaceClient + +# Define the retriever tool +retriever_tool = VectorSearchRetrieverTool( + index_name="catalog.schema.knowledge_base_index", + columns=["doc_id", "title", "content"], + num_results=3, +) + +class RAGAgent(ChatAgent): + def __init__(self): + self.w = WorkspaceClient() + + def predict(self, messages, context=None): + query = messages[-1].content + + results = self.w.vector_search_indexes.query_index( + index_name="catalog.schema.knowledge_base_index", + columns=["title", "content"], + query_text=query, + num_results=3, + ) + + context_docs = "\n\n".join( + f"**{row[0]}**: {row[1]}" + for row in results.result.data_array + ) + + response = self.w.serving_endpoints.query( + name="databricks-meta-llama-3-3-70b-instruct", + messages=[ + {"role": "system", "content": f"Answer using this context:\n{context_docs}"}, + {"role": "user", "content": query}, + ], + ) + + return {"content": response.choices[0].message.content} +``` + +--- + +## Updating the Index + +### Add New Documents + +```sql +INSERT INTO catalog.schema.knowledge_base VALUES +('doc-004', 'MLflow', 'MLflow is an open-source platform for ML lifecycle...', 'ml', current_timestamp()); +``` + +Then sync: + +```python +manage_vs_index(action="sync", index_name="catalog.schema.knowledge_base_index") +``` + +### Delete Documents + +```sql +DELETE FROM catalog.schema.knowledge_base WHERE doc_id = 'doc-001'; +``` + +Then sync — the index automatically handles deletions via Delta change data feed. + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| **Index stuck in PROVISIONING** | Endpoint may still be creating. Check `manage_vs_endpoint(action="get")` first | +| **Query returns no results** | Index may not be synced yet. Run `manage_vs_index(action="sync")` and wait for ONLINE state | +| **"Column not found in index"** | Column must be in `columns_to_sync`. Recreate index with the column included | +| **Embeddings not computed** | Ensure `embedding_model_endpoint_name` is a valid serving endpoint | +| **Stale results after table update** | For TRIGGERED pipelines, you must call `manage_vs_index(action="sync")` manually | +| **Filter not working** | Standard endpoints use dict-format filters (`filters_json`), Storage-Optimized use SQL-like string filters (`filters`) | diff --git a/.claude/skills/databricks-vector-search/search-modes.md b/.claude/skills/databricks-vector-search/search-modes.md new file mode 100644 index 0000000..58092af --- /dev/null +++ b/.claude/skills/databricks-vector-search/search-modes.md @@ -0,0 +1,142 @@ +# Vector Search Modes + +Databricks Vector Search supports three search modes: **ANN** (semantic, default), **HYBRID** (semantic + keyword), and **FULL_TEXT** (keyword only, beta). ANN and HYBRID work with Delta Sync and Direct Access indexes. + +## Semantic Search (ANN) + +ANN (Approximate Nearest Neighbor) is the default search mode. It finds documents by vector similarity — matching the *meaning* of your query against stored embeddings. + +### When to use + +- Conceptual or meaning-based queries ("How do I handle errors in my pipeline?") +- Paraphrased input where exact terms may not appear in the documents +- Multilingual scenarios where query and document languages may differ +- General-purpose RAG retrieval + +### Example + +```python +# ANN is the default — no query_type parameter needed +results = w.vector_search_indexes.query_index( + index_name="catalog.schema.my_index", + columns=["id", "content"], + query_text="How do I handle errors in my pipeline?", + num_results=5 +) +``` + +## Hybrid Search + +Hybrid search combines vector similarity (ANN) with BM25 keyword scoring. It retrieves documents that are both semantically similar *and* contain matching keywords, then merges the results. + +### When to use + +- Queries containing exact terms that must appear: SKUs, product codes, error codes, acronyms +- Proper nouns — company names, people, specific technologies +- Technical documentation where terminology precision matters +- Mixed-intent queries combining concepts with specific terms + +### Example + +```python +results = w.vector_search_indexes.query_index( + index_name="catalog.schema.my_index", + columns=["id", "content"], + query_text="SPARK-12345 executor memory error", + query_type="HYBRID", + num_results=10 +) +``` + +## Decision Guide + +| Mode | Best for | Trade-off | Choose when | +|------|----------|-----------|-------------| +| **ANN** (default) | Conceptual queries, paraphrases, meaning-based search | Fastest; may miss exact keyword matches | You want documents *about* a topic regardless of exact wording | +| **HYBRID** | Exact terms, codes, proper nouns, mixed-intent queries | ~2x resource usage vs ANN; max 200 results | Your queries contain specific identifiers or technical terms that must appear in results | +| **FULL_TEXT** (beta) | Pure keyword search without vector embeddings | No semantic understanding; max 200 results | You need keyword matching only, without vector similarity | + +**Start with ANN.** Switch to HYBRID if you notice relevant documents being missed because they don't share vocabulary with the query. + +## Combining Search Modes with Filters + +Both search modes support filters. The filter syntax depends on your endpoint type: + +- **Standard endpoints** → `filters` as dict (or `filters_json` as JSON string via `databricks-sdk`) +- **Storage-Optimized endpoints** → `filters` as SQL-like string (via `databricks-vectorsearch` package) + +### Standard endpoint with hybrid search + +```python +results = w.vector_search_indexes.query_index( + index_name="catalog.schema.my_index", + columns=["id", "content", "category"], + query_text="SPARK-12345 executor memory error", + query_type="HYBRID", + num_results=10, + filters_json='{"category": "troubleshooting", "status": ["open", "in_progress"]}' +) +``` + +### Storage-Optimized endpoint with hybrid search + +```python +from databricks.vector_search.client import VectorSearchClient + +vsc = VectorSearchClient() +index = vsc.get_index(endpoint_name="my-storage-endpoint", index_name="catalog.schema.my_index") + +results = index.similarity_search( + query_text="SPARK-12345 executor memory error", + columns=["id", "content", "category"], + query_type="hybrid", + num_results=10, + filters="category = 'troubleshooting' AND status IN ('open', 'in_progress')" +) +``` + +## Using with Pre-Computed Embeddings + +If you compute embeddings yourself, use `query_vector` instead of `query_text` for ANN search: + +```python +# ANN with pre-computed embedding (default) +results = w.vector_search_indexes.query_index( + index_name="catalog.schema.my_index", + columns=["id", "content"], + query_vector=[0.1, 0.2, 0.3, ...], # Your embedding vector + num_results=10 +) +``` + +For **hybrid search with self-managed embeddings** (indexes without an associated model endpoint), you must provide **both** `query_vector` and `query_text`. The vector is used for the ANN component and the text for the BM25 keyword component: + +```python +# HYBRID with self-managed embeddings — requires both vector AND text +results = w.vector_search_indexes.query_index( + index_name="catalog.schema.my_index", + columns=["id", "content"], + query_vector=[0.1, 0.2, 0.3, ...], # For ANN similarity + query_text="executor memory error", # For BM25 keyword matching + query_type="HYBRID", + num_results=10 +) +``` + +**Notes:** +- For **ANN** queries: provide either `query_text` or `query_vector`, not both. +- For **HYBRID** queries on **managed embedding indexes**: provide only `query_text` (the system handles both components). +- For **HYBRID** queries on **self-managed indexes without a model endpoint**: provide both `query_vector` and `query_text`. +- When using `query_text` alone, the index must have an associated embedding model (managed embeddings or `embedding_model_endpoint_name` on a Direct Access index). + +## Parameter Reference + +| Parameter | Type | Package | Description | +|-----------|------|---------|-------------| +| `query_text` | `str` | Both | Text query — requires embedding model on the index | +| `query_vector` | `list[float]` | Both | Pre-computed embedding vector | +| `query_type` | `str` | Both | `"ANN"` (default) or `"HYBRID"` or `"FULL_TEXT"` (beta) | +| `columns` | `list[str]` | Both | Column names to return in results | +| `num_results` | `int` | Both | Number of results (default: 10 in `databricks-sdk`, 5 in `databricks-vectorsearch`) | +| `filters_json` | `str` | `databricks-sdk` | JSON dict filter string (Standard endpoints) | +| `filters` | `str` or `dict` | `databricks-vectorsearch` | Dict for Standard, SQL-like string for Storage-Optimized | diff --git a/.claude/skills/databricks-vector-search/troubleshooting-and-operations.md b/.claude/skills/databricks-vector-search/troubleshooting-and-operations.md new file mode 100644 index 0000000..7dc4b8c --- /dev/null +++ b/.claude/skills/databricks-vector-search/troubleshooting-and-operations.md @@ -0,0 +1,177 @@ +# Vector Search Troubleshooting & Operations + +Operational guidance for monitoring, cost optimization, capacity planning, and migration of Databricks Vector Search resources. + +## Monitoring Endpoint Status + +Use `manage_vs_endpoint(action="get")` (MCP tool) or `w.vector_search_endpoints.get_endpoint()` (SDK) to check endpoint health. + +### Endpoint fields + +| Field | Description | +|-------|-------------| +| `state` | `ONLINE`, `PROVISIONING`, `OFFLINE`, `YELLOW_STATE`, `RED_STATE`, `DELETED` | +| `message` | Human-readable status or error message | +| `endpoint_type` | `STANDARD` or `STORAGE_OPTIMIZED` | +| `num_indexes` | Number of indexes hosted on this endpoint | +| `creation_timestamp` | When the endpoint was created | +| `last_updated_timestamp` | When the endpoint was last modified | + +### Example + +```python +endpoint = w.vector_search_endpoints.get_endpoint(endpoint_name="my-endpoint") +print(f"State: {endpoint.endpoint_status.state.value}") +print(f"Indexes: {endpoint.num_indexes}") +``` + +**What to do per state:** +- `PROVISIONING` → Wait. Endpoint creation is asynchronous and can take several minutes. +- `ONLINE` → Ready to serve queries and host indexes. +- `OFFLINE` → Check the `message` field for error details. May require recreation. +- `YELLOW_STATE` → Endpoint is degraded but still serving. Investigate the `message` field. +- `RED_STATE` → Endpoint is unhealthy. Check `message` for details; may need support intervention. + +## Monitoring Index Status + +Use `manage_vs_index(action="get")` (MCP tool) or `w.vector_search_indexes.get_index()` (SDK) to check index health. + +### Index fields + +| Field | Description | +|-------|-------------| +| `status.ready` | Boolean — `True` when ready for queries, `False` when provisioning/syncing | +| `status.message` | Status details or error information | +| `status.index_url` | URL to access the index in the Databricks UI | +| `status.indexed_row_count` | Number of rows currently indexed | +| `delta_sync_index_spec.pipeline_id` | DLT pipeline ID (Delta Sync indexes only) — useful for debugging sync issues | +| `index_type` | `DELTA_SYNC` or `DIRECT_ACCESS` | + +### Example + +```python +index = w.vector_search_indexes.get_index(index_name="catalog.schema.my_index") +if index.status.ready: + print("Index is ONLINE") +else: + print(f"Index is NOT_READY: {index.status.message}") +``` + +## Pipeline Type Trade-offs + +Delta Sync indexes use a DLT pipeline to sync data from the source Delta table. The pipeline type determines sync behavior: + +| Pipeline Type | Behavior | Cost | Best for | +|---------------|----------|------|----------| +| **TRIGGERED** | Manual sync via `manage_vs_index(action="sync")` | Lower — runs only when triggered | Batch updates, periodic refreshes, cost-sensitive workloads | +| **CONTINUOUS** | Auto-syncs on source table changes | Higher — always running | Real-time freshness, applications needing up-to-date results | + +### Triggering a sync + +```python +# For TRIGGERED pipelines only +w.vector_search_indexes.sync_index(index_name="catalog.schema.my_index") +# Check sync progress with get_index() +``` + +**Tip:** CONTINUOUS pipelines cannot be synced manually — they sync automatically. Calling `sync_index()` on a CONTINUOUS index will raise an error. + +## Cost Optimization + +### Endpoint type selection + +| Factor | Standard | Storage-Optimized | +|--------|----------|-------------------| +| Query latency | 20-50ms | 300-500ms | +| Cost | Higher | ~7x lower | +| Max capacity | 320M vectors (768 dim) | 1B+ vectors (768 dim) | +| Indexing speed | Slower | 20x faster | + +**Recommendation:** Start with Storage-Optimized unless you need sub-100ms latency. It handles most RAG workloads well. + +### Reducing storage costs + +- Use `columns_to_sync` to limit which columns are synced to the index. Only synced columns are available in query results, so include only what you need. +- Choose TRIGGERED pipelines for batch workloads to avoid continuous compute costs. + +```python +# Only sync the columns you actually need in query results +delta_sync_index_spec={ + "source_table": "catalog.schema.documents", + "embedding_source_columns": [ + {"name": "content", "embedding_model_endpoint_name": "databricks-gte-large-en"} + ], + "pipeline_type": "TRIGGERED", + "columns_to_sync": ["id", "content", "title"] # Exclude large unused columns +} +``` + +## Capacity Planning + +| Endpoint Type | Max Vectors (768 dim) | Guidance | +|---------------|----------------------|----------| +| Standard | ~320M | Suitable for most production workloads under 300M documents | +| Storage-Optimized | 1B+ | Large-scale corpora, enterprise knowledge bases | + +**Estimating needs:** +- One document typically maps to one vector (or multiple if chunked) +- If chunking at ~512 tokens, expect 2-5 vectors per page of text +- Monitor `num_indexes` on your endpoint to understand utilization + +## Migration Patterns + +### Changing endpoint type + +Endpoints are **immutable after creation** — you cannot change the type (Standard ↔ Storage-Optimized) of an existing endpoint. To migrate: + +1. **Create a new endpoint** with the desired type +2. **Recreate indexes** on the new endpoint pointing to the same source tables +3. **Wait for sync** to complete (check index state) +4. **Update applications** to query the new index names +5. **Delete old indexes**, then delete the old endpoint + +```python +# Step 1: Create new endpoint +w.vector_search_endpoints.create_endpoint( + name="my-endpoint-storage-optimized", + endpoint_type="STORAGE_OPTIMIZED" +) + +# Step 2: Recreate index on new endpoint (same source table) +w.vector_search_indexes.create_index( + name="catalog.schema.my_index_v2", + endpoint_name="my-endpoint-storage-optimized", + primary_key="id", + index_type="DELTA_SYNC", + delta_sync_index_spec={ + "source_table": "catalog.schema.documents", + "embedding_source_columns": [ + {"name": "content", "embedding_model_endpoint_name": "databricks-gte-large-en"} + ], + "pipeline_type": "TRIGGERED" + } +) + +# Step 3: Trigger sync and wait for ONLINE state +w.vector_search_indexes.sync_index(index_name="catalog.schema.my_index_v2") + +# Step 4: Update your application to use "catalog.schema.my_index_v2" +# Step 5: Clean up old resources +w.vector_search_indexes.delete_index(index_name="catalog.schema.my_index") +w.vector_search_endpoints.delete_endpoint(endpoint_name="my-endpoint") +``` + +## Expanded Troubleshooting + +| Issue | Likely Cause | Solution | +|-------|-------------|----------| +| **Index stuck in NOT_READY** | Sync pipeline failed or source table issue | Check `message` field via `manage_vs_index(action="get")`. Inspect the DLT pipeline using `pipeline_id`. | +| **Embedding dimension mismatch** | Query vector dimensions ≠ index dimensions | Ensure your embedding model output matches the `embedding_dimension` in the index spec. | +| **Permission errors on create** | Missing Unity Catalog privileges | User needs `CREATE TABLE` on the schema and `USE CATALOG`/`USE SCHEMA` privileges. | +| **Index returns NOT_FOUND** | Wrong name format or index deleted | Index names must be fully qualified: `catalog.schema.index_name`. | +| **Sync not running (TRIGGERED)** | Sync not triggered after source update | Call `manage_vs_index(action="sync")` or `w.vector_search_indexes.sync_index()` after updating source data. | +| **Endpoint NOT_FOUND** | Endpoint name typo or deleted | List all endpoints with `manage_vs_endpoint(action="list")` to verify available endpoints. | +| **Query returns empty results** | Index not yet synced, or filters too restrictive | Check index state is ONLINE. Verify `columns_to_sync` includes queried columns. Test without filters first. | +| **filters_json has no effect** | Using wrong filter syntax for endpoint type | Standard endpoints use dict-format filters (`filters_json` in SDK, `filters` as dict in `databricks-vectorsearch`). Storage-Optimized endpoints use SQL-like string filters (`filters` as str in `databricks-vectorsearch`). | +| **Quota or capacity errors** | Too many indexes or vectors | Check `num_indexes` on endpoint. Consider Storage-Optimized for higher capacity. | +| **Upsert fails on Delta Sync** | Cannot upsert to Delta Sync indexes | Upsert/delete operations only work on Direct Access indexes. Delta Sync indexes update via their source table. | diff --git a/.claude/skills/databricks-zerobus-ingest/1-setup-and-authentication.md b/.claude/skills/databricks-zerobus-ingest/1-setup-and-authentication.md index 10b07f6..31dfd1b 100644 --- a/.claude/skills/databricks-zerobus-ingest/1-setup-and-authentication.md +++ b/.claude/skills/databricks-zerobus-ingest/1-setup-and-authentication.md @@ -76,6 +76,8 @@ GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.my_events TO `=1.0.0 ``` Or with a virtual environment: ```bash -uv pip install databricks-zerobus-ingest-sdk +uv pip install databricks-zerobus-ingest-sdk>=1.0.0 ``` +**Note:** The Zerobus SDK cannot be pip-installed on Databricks serverless compute. Use classic compute clusters, or use the [Zerobus REST API](https://docs.databricks.com/aws/en/ingestion/zerobus-rest-api) (Beta) for notebook-based ingestion without the SDK. + ### Java (8+) Maven: diff --git a/.claude/skills/databricks-zerobus-ingest/2-python-client.md b/.claude/skills/databricks-zerobus-ingest/2-python-client.md index ac95cd4..64c6f8b 100644 --- a/.claude/skills/databricks-zerobus-ingest/2-python-client.md +++ b/.claude/skills/databricks-zerobus-ingest/2-python-client.md @@ -11,12 +11,14 @@ Python SDK patterns for Zerobus Ingest: synchronous and asynchronous APIs, JSON from zerobus.sdk.sync import ZerobusSdk # Asynchronous API (equivalent capabilities) -from zerobus.sdk.asyncio import ZerobusSdk as AsyncZerobusSdk +from zerobus.sdk.aio import ZerobusSdk as AsyncZerobusSdk # Shared types (used by both sync and async) from zerobus.sdk.shared import ( RecordType, - IngestRecordResponse, + AckCallback, + ZerobusException, + NonRetriableException, StreamConfigurationOptions, TableProperties, ) @@ -49,8 +51,8 @@ stream = sdk.create_stream(client_id, client_secret, table_props, options) try: for i in range(100): record = {"device_name": f"sensor-{i}", "temp": 22, "humidity": 55} - ack = stream.ingest_record(record) - ack.wait_for_ack() # Block until durably written + offset = stream.ingest_record_offset(record) + stream.wait_for_offset(offset) # Block until durably written finally: stream.close() ``` --> @@ -90,8 +92,8 @@ try: temp=22, humidity=55, ) - ack = stream.ingest_record(record) - ack.wait_for_ack() + offset = stream.ingest_record_offset(record) + stream.wait_for_offset(offset) finally: stream.close() ``` @@ -100,17 +102,21 @@ finally: ## ACK Callback (Asynchronous Acknowledgment) -Instead of blocking on each ACK, register a callback for background durability confirmation: +Instead of blocking on each ACK, register an `AckCallback` subclass for background durability confirmation: ```python -from zerobus.sdk.shared import IngestRecordResponse, StreamConfigurationOptions, RecordType +from zerobus.sdk.shared import AckCallback, StreamConfigurationOptions, RecordType -def on_ack(response: IngestRecordResponse) -> None: - print(f"Durable up to offset: {response.durability_ack_up_to_offset}") +class MyAckHandler(AckCallback): + def on_ack(self, offset: int) -> None: + print(f"Durable up to offset: {offset}") + + def on_error(self, offset: int, message: str) -> None: + print(f"Error at offset {offset}: {message}") options = StreamConfigurationOptions( record_type=RecordType.JSON, - ack_callback=on_ack, + ack_callback=MyAckHandler(), ) # Create stream with callback @@ -119,7 +125,7 @@ stream = sdk.create_stream(client_id, client_secret, table_props, options) try: for i in range(1000): record = {"device_name": f"sensor-{i}", "temp": 22, "humidity": 55} - stream.ingest_record(record) # Non-blocking, ACKs arrive via callback + stream.ingest_record_nowait(record) # Fire-and-forget, ACKs arrive via callback stream.flush() # Ensure all buffered records are sent finally: stream.close() @@ -135,12 +141,12 @@ A production-ready wrapper with retry logic, reconnection, and both JSON and Pro import os import time import logging -from typing import Optional, Callable +from typing import Optional from zerobus.sdk.sync import ZerobusSdk from zerobus.sdk.shared import ( RecordType, - IngestRecordResponse, + AckCallback, StreamConfigurationOptions, TableProperties, ) @@ -159,7 +165,7 @@ class ZerobusClient: client_id: str, client_secret: str, record_type: RecordType = RecordType.JSON, - ack_callback: Optional[Callable[[IngestRecordResponse], None]] = None, + ack_callback: Optional[AckCallback] = None, proto_descriptor=None, ): self.server_endpoint = server_endpoint @@ -199,8 +205,8 @@ class ZerobusClient: try: if self.stream is None: self.init_stream() - ack = self.stream.ingest_record(payload) - ack.wait_for_ack() + offset = self.stream.ingest_record_offset(payload) + self.stream.wait_for_offset(offset) return True except Exception as e: err = str(e).lower() @@ -275,7 +281,7 @@ The SDK provides an equivalent async API for use with `asyncio`: ```python import asyncio -from zerobus.sdk.asyncio import ZerobusSdk as AsyncZerobusSdk +from zerobus.sdk.aio import ZerobusSdk as AsyncZerobusSdk from zerobus.sdk.shared import RecordType, StreamConfigurationOptions, TableProperties @@ -289,8 +295,8 @@ async def ingest_async(): try: for i in range(100): record = {"device_name": f"sensor-{i}", "temp": 22, "humidity": 55} - ack = await stream.ingest_record(record) - await ack.wait_for_ack() + offset = await stream.ingest_record_offset(record) + await stream.wait_for_offset(offset) finally: await stream.close() @@ -304,7 +310,7 @@ asyncio.run(ingest_async()) ## Batch Pattern -For higher throughput, send records without blocking on each ACK and flush at the end: +For higher throughput, use `ingest_record_nowait` (fire-and-forget) or batch methods, and flush at the end: ```python with ZerobusClient( @@ -314,10 +320,39 @@ with ZerobusClient( client_id=os.environ["DATABRICKS_CLIENT_ID"], client_secret=os.environ["DATABRICKS_CLIENT_SECRET"], record_type=RecordType.JSON, - ack_callback=lambda resp: None, # Discard individual ACKs ) as client: for i in range(10_000): record = {"device_name": f"sensor-{i}", "temp": 22, "humidity": 55} - client.stream.ingest_record(record) # Non-blocking + client.stream.ingest_record_nowait(record) # Fire-and-forget # flush() and close() called automatically by context manager ``` + +For true batch ingestion, use the batch variants: + +```python +records = [ + {"device_name": f"sensor-{i}", "temp": 22, "humidity": 55} + for i in range(10_000) +] +# Fire-and-forget batch +stream.ingest_records_nowait(records) +stream.flush() + +# Or with offset tracking +offset = stream.ingest_records_offset(records) +stream.wait_for_offset(offset) +``` + +--- + +## Ingestion Method Comparison + +| Method | Returns | Blocks? | Best For | +|--------|---------|---------|----------| +| `ingest_record_offset(record)` | offset | No (enqueues) | Single record with durability tracking | +| `ingest_record_nowait(record)` | None | No | Max single-record throughput | +| `ingest_records_offset(records)` | last offset | No (enqueues) | Batch with durability tracking | +| `ingest_records_nowait(records)` | None | No | Max batch throughput | +| `wait_for_offset(offset)` | None | Yes (until ACK) | Durability confirmation | +| `flush()` | None | Yes (until sent) | Ensure all buffered records are sent | +| `ingest_record(record)` | RecordAcknowledgment | No | Primary method in SDK v1.1.0+; pass `json.dumps(record)` for JSON | diff --git a/.claude/skills/databricks-zerobus-ingest/3-multilanguage-clients.md b/.claude/skills/databricks-zerobus-ingest/3-multilanguage-clients.md index 217398c..4eba101 100644 --- a/.claude/skills/databricks-zerobus-ingest/3-multilanguage-clients.md +++ b/.claude/skills/databricks-zerobus-ingest/3-multilanguage-clients.md @@ -51,7 +51,8 @@ public class ZerobusProducer { .setTemp(22) .setHumidity(55) .build(); - stream.ingestRecord(record).join(); + long offset = stream.ingestRecordOffset(record); + stream.waitForOffset(offset); } } finally { stream.close(); @@ -126,12 +127,12 @@ func main() { record := fmt.Sprintf( `{"device_name": "sensor-%d", "temp": 22, "humidity": 55}`, i, ) - ack, err := stream.IngestRecord(record) + offset, err := stream.IngestRecordOffset(record) if err != nil { log.Printf("Ingest failed for record %d: %v", i, err) continue } - ack.Await() + stream.WaitForOffset(offset) } stream.Flush() @@ -187,7 +188,8 @@ const stream = await sdk.createStream( try { for (let i = 0; i < 100; i++) { const record = { device_name: `sensor-${i}`, temp: 22, humidity: 55 }; - await stream.ingestRecord(record); + const offset = await stream.ingestRecordOffset(record); + await stream.waitForOffset(offset); } await stream.flush(); } finally { @@ -207,7 +209,8 @@ async function ingestWithRetry( ): Promise { for (let attempt = 0; attempt < maxRetries; attempt++) { try { - await stream.ingestRecord(record); + const offset = await stream.ingestRecordOffset(record); + await stream.waitForOffset(offset); return true; } catch (error) { console.warn(`Attempt ${attempt + 1}/${maxRetries} failed:`, error); @@ -268,8 +271,8 @@ async fn main() -> Result<(), Box> { r#"{{"device_name": "sensor-{}", "temp": 22, "humidity": 55}}"#, i ); - let ack = stream.ingest_record(record.into_bytes()).await?; - ack.await?; + let offset = stream.ingest_record_offset(record.into_bytes()).await?; + stream.wait_for_offset(offset).await?; } stream.close().await?; @@ -296,8 +299,8 @@ let mut stream = sdk // Ingest serialized protobuf bytes let record_bytes = my_proto_message.encode_to_vec(); -let ack = stream.ingest_record(record_bytes).await?; -ack.await?; +let offset = stream.ingest_record_offset(record_bytes).await?; +stream.wait_for_offset(offset).await?; ``` --- @@ -310,5 +313,5 @@ ack.await?; | Package | `databricks-zerobus-ingest-sdk` | `com.databricks:zerobus-ingest-sdk` | `github.com/databricks/zerobus-sdk-go` | `@databricks/zerobus-ingest-sdk` | `databricks-zerobus-ingest-sdk` | | Default serialization | JSON | Protobuf | JSON | JSON | JSON | | Async API | Yes (separate module) | CompletableFuture | Goroutines | Native async/await | Tokio async/await | -| ACK pattern | `ack.wait_for_ack()` or callback | `.join()` | `ack.Await()` | Implicit in `await` | `ack.await?` | +| ACK pattern | `wait_for_offset(offset)` or `AckCallback` | `waitForOffset(offset)` | `WaitForOffset(offset)` | `await waitForOffset(offset)` | `wait_for_offset(offset).await?` | | Proto generation | `python -m zerobus.tools.generate_proto` | JAR CLI tool | External `protoc` | External `protoc` | External `protoc` | diff --git a/.claude/skills/databricks-zerobus-ingest/5-operations-and-limits.md b/.claude/skills/databricks-zerobus-ingest/5-operations-and-limits.md index 7b8cb2b..004774d 100644 --- a/.claude/skills/databricks-zerobus-ingest/5-operations-and-limits.md +++ b/.claude/skills/databricks-zerobus-ingest/5-operations-and-limits.md @@ -12,40 +12,44 @@ Every ingested record returns a durability acknowledgment. An ACK indicates that | Strategy | When to Use | Trade-off | |----------|-------------|-----------| -| **Sync block per record** | Low-volume, strict ordering | Simplest; lower throughput | -| **ACK callback** | High-volume producers | Higher throughput; more complex | -| **Periodic flush** | Batch-oriented workloads | Best throughput; eventual consistency | +| **`ingest_record_offset` + `wait_for_offset`** | Low-volume, strict ordering | Simplest; lower throughput | +| **`ingest_record_nowait` + `AckCallback`** | High-volume producers | Higher throughput; more complex | +| **`ingest_record_nowait` + periodic `flush`** | Batch-oriented workloads | Best throughput; eventual consistency | ### Sync Block (Python) ```python -ack = stream.ingest_record(record) -ack.wait_for_ack() # Blocks until durable +offset = stream.ingest_record_offset(record) +stream.wait_for_offset(offset) # Blocks until durable ``` ### ACK Callback (Python) ```python -from zerobus.sdk.shared import IngestRecordResponse +from zerobus.sdk.shared import AckCallback -last_acked_offset = 0 +class MyAckHandler(AckCallback): + def __init__(self): + self.last_acked_offset = 0 -def on_ack(response: IngestRecordResponse) -> None: - global last_acked_offset - last_acked_offset = response.durability_ack_up_to_offset + def on_ack(self, offset: int) -> None: + self.last_acked_offset = offset + + def on_error(self, offset: int, message: str) -> None: + print(f"Error at offset {offset}: {message}") options = StreamConfigurationOptions( record_type=RecordType.JSON, - ack_callback=on_ack, + ack_callback=MyAckHandler(), ) ``` ### Flush-Based ```python -# Send many records without blocking +# Send many records without blocking (fire-and-forget) for record in batch: - stream.ingest_record(record) + stream.ingest_record_nowait(record) # Flush ensures all buffered records are sent stream.flush() @@ -89,8 +93,8 @@ def ingest_with_retry(stream_factory, record, max_retries=5): for attempt in range(max_retries): try: - ack = stream.ingest_record(record) - ack.wait_for_ack() + offset = stream.ingest_record_offset(record) + stream.wait_for_offset(offset) return stream # Return the (possibly new) stream except Exception as e: err = str(e).lower() diff --git a/.claude/skills/databricks-zerobus-ingest/SKILL.md b/.claude/skills/databricks-zerobus-ingest/SKILL.md index d29dc00..22f90c5 100644 --- a/.claude/skills/databricks-zerobus-ingest/SKILL.md +++ b/.claude/skills/databricks-zerobus-ingest/SKILL.md @@ -7,7 +7,7 @@ description: "Build Zerobus Ingest clients for near real-time data ingestion int Build clients that ingest data directly into Databricks Delta tables via the Zerobus gRPC API. -**Status:** Public Preview (currently free; Databricks plans to introduce charges in the future) +**Status:** GA (Generally Available since February 2026; billed under Lakeflow Jobs Serverless SKU) **Documentation:** - [Zerobus Overview](https://docs.databricks.com/aws/en/ingestion/zerobus-overview) @@ -37,7 +37,7 @@ Zerobus Ingest is a serverless connector that enables direct, record-by-record d | Schema generation from UC table | Any | Protobuf | [4-protobuf-schema.md](4-protobuf-schema.md) | | Retry / reconnection logic | Any | Any | [5-operations-and-limits.md](5-operations-and-limits.md) | -If not speficfied, default to python. +If not specified, default to python. --- @@ -46,9 +46,9 @@ If not speficfied, default to python. These libraries are essential for ZeroBus data ingestion: - **databricks-sdk>=0.85.0**: Databricks workspace client for authentication and metadata -- **databricks-zerobus-ingest-sdk>=0.2.0**: ZeroBus SDK for high-performance streaming ingestion +- **databricks-zerobus-ingest-sdk>=1.0.0**: ZeroBus SDK for high-performance streaming ingestion - **grpcio-tools** -These are typically NOT pre-installed on Databricks. Install them using `execute_databricks_command` tool: +These are typically NOT pre-installed on Databricks. Install them using `execute_code` tool: - `code`: "%pip install databricks-sdk>=VERSION databricks-zerobus-ingest-sdk>=VERSION" Save the returned `cluster_id` and `context_id` for subsequent calls. @@ -85,6 +85,7 @@ See [1-setup-and-authentication.md](1-setup-and-authentication.md) for complete ## Minimal Python Example (JSON) ```python +import json from zerobus.sdk.sync import ZerobusSdk from zerobus.sdk.shared import RecordType, StreamConfigurationOptions, TableProperties @@ -95,8 +96,8 @@ table_props = TableProperties(table_name) stream = sdk.create_stream(client_id, client_secret, table_props, options) try: record = {"device_name": "sensor-1", "temp": 22, "humidity": 55} - ack = stream.ingest_record(record) - ack.wait_for_ack() + stream.ingest_record(json.dumps(record)) + stream.flush() finally: stream.close() ``` @@ -115,22 +116,24 @@ finally: --- -You must always follow all the steps in the Workslfow +You must always follow all the steps in the Workflow ## Workflow 0. **Display the plan of your execution** 1. **Determinate the type of client** -2. **Get schema** Always use 4-protobuf-schema.md. Execute using the `run_python_file_on_databricks` MCP tool -3. **Write Python code to a local file follow the instructions in the relevant guide to ingest with zerobus** in the project (e.g., `scripts/zerobus_ingest.py`). -4. **Execute on Databricks** using the `run_python_file_on_databricks` MCP tool +2. **Get schema** Always use 4-protobuf-schema.md. Execute using the `execute_code` MCP tool +3. **Write Python code to a local file follow the instructions in the relevant guide to ingest with zerobus** in the project (e.g., `scripts/zerobus_ingest.py`). +4. **Execute on Databricks** using the `execute_code` MCP tool (with `file_path` parameter) 5. **If execution fails**: Edit the local file to fix the error, then re-execute 6. **Reuse the context** for follow-up executions by passing the returned `cluster_id` and `context_id` --- ## Important -- Never install local packages +- Never install local packages - Always validate MCP server requirement before execution +- **Serverless limitation**: The Zerobus SDK cannot pip-install on serverless compute. Use classic compute clusters, or use the [Zerobus REST API](https://docs.databricks.com/aws/en/ingestion/zerobus-rest-api) (Beta) for notebook-based ingestion without the SDK. +- **Explicit table grants**: Service principals need explicit `MODIFY` and `SELECT` grants on the target table. Schema-level inherited permissions may not be sufficient for the `authorization_details` OAuth flow. --- @@ -138,7 +141,7 @@ You must always follow all the steps in the Workslfow The first execution auto-selects a running cluster and creates an execution context. **Reuse this context for follow-up calls** - it's much faster (~1s vs ~15s) and shares variables/imports: -**First execution** - use `run_python_file_on_databricks` tool: +**First execution** - use `execute_code` tool: - `file_path`: "scripts/zerobus_ingest.py" Returns: `{ success, output, error, cluster_id, context_id, ... }` @@ -148,7 +151,7 @@ Save `cluster_id` and `context_id` for follow-up calls. **If execution fails:** 1. Read the error from the result 2. Edit the local Python file to fix the issue -3. Re-execute with same context using `run_python_file_on_databricks` tool: +3. Re-execute with same context using `execute_code` tool: - `file_path`: "scripts/zerobus_ingest.py" - `cluster_id`: "" - `context_id`: "" @@ -172,8 +175,8 @@ When execution fails: Databricks provides Spark, pandas, numpy, and common data libraries by default. **Only install a library if you get an import error.** -Use `execute_databricks_command` tool: -- `code`: "%pip install databricks-zerobus-ingest-sdk>=0.2.0" +Use `execute_code` tool: +- `code`: "%pip install databricks-zerobus-ingest-sdk>=1.0.0" - `cluster_id`: "" - `context_id`: "" @@ -193,7 +196,7 @@ The timestamp generation must use microseconds for Databricks. - **gRPC + Protobuf**: Zerobus uses gRPC as its transport protocol. Any application that can communicate via gRPC and construct Protobuf messages can produce to Zerobus. - **JSON or Protobuf serialization**: JSON for quick starts; Protobuf for type safety, forward compatibility, and performance. - **At-least-once delivery**: The connector provides at-least-once guarantees. Design consumers to handle duplicates. -- **Durability ACKs**: Each ingested record returns an ACK confirming durable write. ACKs indicate all records up to that offset have been durably written. +- **Durability ACKs**: Each ingested record returns a `RecordAcknowledgment`. Use `flush()` to ensure all buffered records are durably written, or use `wait_for_offset(offset)` for offset-based tracking. - **No table management**: Zerobus does not create or alter tables. You must pre-create your target table and manage schema evolution yourself. - **Single-AZ durability**: The service runs in a single availability zone. Plan for potential zone outages. @@ -210,6 +213,8 @@ The timestamp generation must use microseconds for Databricks. | **Throughput limits hit** | Max 100 MB/s and 15,000 rows/s per stream. Open multiple streams or contact Databricks. | | **Region not supported** | Check supported regions in [5-operations-and-limits.md](5-operations-and-limits.md). | | **Table not found** | Ensure table is a managed Delta table in a supported region with correct three-part name. | +| **SDK install fails on serverless** | The Zerobus SDK cannot be pip-installed on serverless compute. Use classic compute clusters or the REST API (Beta) from notebooks. | +| **Error 4024 / authorization_details** | Service principal lacks explicit table-level grants. Grant `MODIFY` and `SELECT` directly on the target table — schema-level inherited grants may be insufficient. | --- @@ -218,7 +223,7 @@ The timestamp generation must use microseconds for Databricks. - **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - General SDK patterns and WorkspaceClient for table/schema management - **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - Downstream pipeline processing of ingested data - **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - Managing catalogs, schemas, and tables that Zerobus writes to -- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - Generate test data to feed into Zerobus producers +- **[databricks-synthetic-data-gen](../databricks-synthetic-data-gen/SKILL.md)** - Generate test data to feed into Zerobus producers - **[databricks-config](../databricks-config/SKILL.md)** - Profile and authentication setup ## Resources diff --git a/.claude/skills/flutter-add-integration-test/SKILL.md b/.claude/skills/flutter-add-integration-test/SKILL.md new file mode 100644 index 0000000..60902f1 --- /dev/null +++ b/.claude/skills/flutter-add-integration-test/SKILL.md @@ -0,0 +1,163 @@ +--- +name: flutter-add-integration-test +description: Configures Flutter Driver for app interaction and converts MCP actions into permanent integration tests. Use when adding integration testing to a project, exploring UI components via MCP, or automating user flows with the integration_test package. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Tue, 21 Apr 2026 18:29:20 GMT +--- +# Implementing Flutter Integration Tests + +## Contents +- [Project Setup and Dependencies](#project-setup-and-dependencies) +- [Interactive Exploration via MCP](#interactive-exploration-via-mcp) +- [Test Authoring Guidelines](#test-authoring-guidelines) +- [Execution and Profiling](#execution-and-profiling) +- [Workflow: End-to-End Integration Testing](#workflow-end-to-end-integration-testing) +- [Examples](#examples) + +## Project Setup and Dependencies + +Configure the project to support integration testing and Flutter Driver extensions. + +1. Add required development dependencies to `pubspec.yaml`: + ```bash + flutter pub add 'dev:integration_test:{"sdk":"flutter"}' + flutter pub add 'dev:flutter_test:{"sdk":"flutter"}' + ``` +2. Enable the Flutter Driver extension in your application entry point (typically `lib/main.dart` or a dedicated `lib/main_test.dart`): + - Import `package:flutter_driver/driver_extension.dart`. + - Call `enableFlutterDriverExtension();` before `runApp()`. +3. Add `Key` parameters (e.g., `ValueKey('login_button')`) to critical widgets in the application code to ensure reliable targeting during tests. + +## Interactive Exploration via MCP + +Use the Dart/Flutter MCP server tools to interactively explore and manipulate the application state before writing static tests. + +- **Launch**: Execute `launch_app` with `target: "lib/main_test.dart"` to start the application and acquire the DTD URI. +- **Inspect**: Execute `get_widget_tree` to discover available `Key`s, `Text` nodes, and widget `Type`s. +- **Interact**: Execute `tap`, `enter_text`, and `scroll` to simulate user flows. +- **Wait**: Always execute `waitFor` or verify state with `get_health` when navigating or triggering animations. +- **Troubleshoot Unmounted Widgets**: If a widget is not found in the tree, it may be lazily loaded in a `SliverList` or `ListView`. Execute `scroll` or `scrollIntoView` to force the widget to mount before interacting with it. + +## Test Authoring Guidelines + +Structure integration tests using the `flutter_test` API paradigm. + +- Create a dedicated `integration_test/` directory at the project root. +- Name all test files using the `_test.dart` convention. +- Initialize the binding by calling `IntegrationTestWidgetsFlutterBinding.ensureInitialized();` at the start of `main()`. +- Load the application UI using `await tester.pumpWidget(MyApp());`. +- Trigger frames and wait for animations to complete using `await tester.pumpAndSettle();` after interactions like `tester.tap()`. +- Assert widget visibility using `expect(find.byKey(ValueKey('foo')), findsOneWidget);` or `findsNothing`. +- Scroll to specific off-screen widgets using `await tester.scrollUntilVisible(itemFinder, 500.0, scrollable: listFinder);`. + +**Conditional Logic for Legacy `flutter_driver`:** +- If maintaining or migrating legacy `flutter_driver` tests, use `driver.waitFor()`, `driver.waitForAbsent()`, `driver.tap()`, and `driver.scroll()` instead of the `WidgetTester` APIs. + +## Execution and Profiling + +Execute tests using the `flutter drive` command. Require a host driver script located in `test_driver/integration_test.dart` that calls `integrationDriver()`. + +**Conditional Execution Targets:** +- **If testing on Chrome:** Launch `chromedriver --port=4444` in a separate terminal, then run: + `flutter drive --driver=test_driver/integration_test.dart --target=integration_test/app_test.dart -d chrome` +- **If testing headless web:** Run with `-d web-server`. +- **If testing on Android (Local):** Run `flutter drive --driver=test_driver/integration_test.dart --target=integration_test/app_test.dart`. +- **If testing on Firebase Test Lab (Android):** + 1. Build debug APK: `flutter build apk --debug` + 2. Build test APK: `./gradlew app:assembleAndroidTest` + 3. Upload both APKs to the Firebase Test Lab console. + +## Workflow: End-to-End Integration Testing + +Copy and follow this checklist to implement and verify integration tests. + +- [ ] **Task Progress: Setup** + - [ ] Add `integration_test` and `flutter_test` to `pubspec.yaml`. + - [ ] Inject `enableFlutterDriverExtension()` into the app entry point. + - [ ] Assign `ValueKey`s to target widgets. +- [ ] **Task Progress: Exploration** + - [ ] Run `launch_app` via MCP. + - [ ] Map the widget tree using `get_widget_tree`. + - [ ] Validate interaction paths using MCP tools (`tap`, `enter_text`). +- [ ] **Task Progress: Authoring** + - [ ] Create `integration_test/app_test.dart`. + - [ ] Write test cases using `WidgetTester` APIs. + - [ ] Create `test_driver/integration_test.dart` with `integrationDriver()`. +- [ ] **Task Progress: Execution & Feedback Loop** + - [ ] Run `flutter drive --driver=test_driver/integration_test.dart --target=integration_test/app_test.dart`. + - [ ] **Feedback Loop**: Review test output -> If `PumpAndSettleTimedOutException` occurs, check for infinite animations -> If widget not found, add `scrollUntilVisible` -> Re-run test until passing. + +## Examples + +### Standard Integration Test (`integration_test/app_test.dart`) + +```dart +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; +import 'package:my_app/main.dart'; + +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + group('End-to-end test', () { + testWidgets('tap on the floating action button, verify counter', (tester) async { + // Load app widget. + await tester.pumpWidget(const MyApp()); + + // Verify the counter starts at 0. + expect(find.text('0'), findsOneWidget); + + // Find the floating action button to tap on. + final fab = find.byKey(const ValueKey('increment')); + + // Emulate a tap on the floating action button. + await tester.tap(fab); + + // Trigger a frame and wait for animations. + await tester.pumpAndSettle(); + + // Verify the counter increments by 1. + expect(find.text('1'), findsOneWidget); + }); + }); +} +``` + +### Host Driver Script (`test_driver/integration_test.dart`) + +```dart +import 'package:integration_test/integration_test_driver.dart'; + +Future main() => integrationDriver(); +``` + +### Performance Profiling Driver Script (`test_driver/perf_driver.dart`) + +Use this driver script if you wrap your test actions in `binding.traceAction()` to capture performance metrics. + +```dart +import 'package:flutter_driver/flutter_driver.dart' as driver; +import 'package:integration_test/integration_test_driver.dart'; + +Future main() { + return integrationDriver( + responseDataCallback: (data) async { + if (data != null) { + final timeline = driver.Timeline.fromJson( + data['scrolling_timeline'] as Map, + ); + + final summary = driver.TimelineSummary.summarize(timeline); + + await summary.writeTimelineToFile( + 'scrolling_timeline', + pretty: true, + includeSummary: true, + ); + } + }, + ); +} +``` diff --git a/.claude/skills/flutter-add-widget-preview/SKILL.md b/.claude/skills/flutter-add-widget-preview/SKILL.md new file mode 100644 index 0000000..6ba6894 --- /dev/null +++ b/.claude/skills/flutter-add-widget-preview/SKILL.md @@ -0,0 +1,145 @@ +--- +name: flutter-add-widget-preview +description: Adds interactive widget previews to the project using the previews.dart system. Use when creating new UI components or updating existing screens to ensure consistent design and interactive testing. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Tue, 21 Apr 2026 20:05:23 GMT +--- +# Previewing Flutter Widgets + +## Contents +- [Preview Guidelines](#preview-guidelines) +- [Handling Limitations](#handling-limitations) +- [Workflows](#workflows) +- [Examples](#examples) + +## Preview Guidelines + +Use the Flutter Widget Previewer to render widgets in real-time, isolated from the full application context. + +- **Target Elements:** Apply the `@Preview` annotation to top-level functions, static methods within a class, or public widget constructors/factories that have no required arguments and return a `Widget` or `WidgetBuilder`. +- **Imports:** Always import `package:flutter/widget_previews.dart` to access the preview annotations. +- **Custom Annotations:** Extend the `Preview` class to create custom annotations that inject common properties (e.g., themes, wrappers) across multiple widgets. +- **Multiple Configurations:** Apply multiple `@Preview` annotations to a single target to generate multiple preview instances. Alternatively, extend `MultiPreview` to encapsulate common multi-preview configurations. +- **Runtime Transformations:** Override the `transform()` method in custom `Preview` or `MultiPreview` classes to modify preview configurations dynamically at runtime (e.g., generating names based on dynamic values, which is impossible in a `const` context). + +## Handling Limitations + +Adhere to the following constraints when authoring previewable widgets, as the Widget Previewer runs in a web environment: + +- **No Native APIs:** Do not use native plugins or APIs from `dart:io` or `dart:ffi`. Widgets with transitive dependencies on `dart:io` or `dart:ffi` will throw exceptions upon invocation. Use conditional imports to mock or bypass these in preview mode. +- **Asset Paths:** Use package-based paths for assets loaded via `dart:ui` `fromAsset` APIs (e.g., `packages/my_package_name/assets/my_image.png` instead of `assets/my_image.png`). +- **Public Callbacks:** Ensure all callback arguments provided to preview annotations are public and constant to satisfy code generation requirements. +- **Constraints:** Apply explicit constraints using the `size` parameter in the `@Preview` annotation if your widget is unconstrained, as the previewer defaults to constraining them to approximately half the viewport. + +## Workflows + +### Creating a Widget Preview +Copy and track this checklist when implementing a new widget preview: + +- [ ] Import `package:flutter/widget_previews.dart`. +- [ ] Identify a valid target (top-level function, static method, or parameter-less public constructor). +- [ ] Apply the `@Preview` annotation to the target. +- [ ] Configure preview parameters (`name`, `group`, `size`, `theme`, `brightness`, etc.) as needed. +- [ ] If applying the same configuration to multiple widgets, extract the configuration into a custom class extending `Preview`. + +### Interacting with Previews +Follow the appropriate conditional workflow to launch and interact with the Widget Previewer: + +**If using a supported IDE (Android Studio, IntelliJ, VS Code with Flutter 3.38+):** +1. Launch the IDE. The Widget Previewer starts automatically. +2. Open the "Flutter Widget Preview" tab in the sidebar. +3. Toggle "Filter previews by selected file" at the bottom left if you want to view previews outside the currently active file. + +**If using the Command Line:** +1. Navigate to the Flutter project's root directory. +2. Run `flutter widget-preview start`. +3. View the automatically opened Chrome environment. + +**Feedback Loop: Preview Iteration** +1. Modify the widget code or preview configuration. +2. Observe the automatic update in the Widget Previewer. +3. If global state (e.g., static initializers) was modified: Click the global hot restart button at the bottom right. +4. If only the local widget state needs resetting: Click the individual hot restart button on the specific preview card. +5. Review errors in the IDE/CLI console -> fix -> repeat. + +## Examples + +### Basic Preview +```dart +import 'package:flutter/widget_previews.dart'; +import 'package:flutter/material.dart'; + +@Preview(name: 'My Sample Text', group: 'Typography') +Widget mySampleText() { + return const Text('Hello, World!'); +} +``` + +### Custom Preview with Runtime Transformation +```dart +import 'package:flutter/widget_previews.dart'; +import 'package:flutter/material.dart'; + +final class TransformativePreview extends Preview { + const TransformativePreview({ + super.name, + super.group, + }); + + PreviewThemeData _themeBuilder() { + return PreviewThemeData( + materialLight: ThemeData.light(), + materialDark: ThemeData.dark(), + ); + } + + @override + Preview transform() { + final originalPreview = super.transform(); + final builder = originalPreview.toBuilder(); + + builder + ..name = 'Transformed - ${originalPreview.name}' + ..theme = _themeBuilder; + + return builder.toPreview(); + } +} + +@TransformativePreview(name: 'Custom Themed Button') +Widget myButton() => const ElevatedButton(onPressed: null, child: Text('Click')); +``` + +### MultiPreview Implementation +```dart +import 'package:flutter/widget_previews.dart'; +import 'package:flutter/material.dart'; + +/// Creates light and dark mode previews automatically. +final class MultiBrightnessPreview extends MultiPreview { + const MultiBrightnessPreview({required this.name}); + + final String name; + + @override + List get previews => const [ + Preview(brightness: Brightness.light), + Preview(brightness: Brightness.dark), + ]; + + @override + List transform() { + final previews = super.transform(); + return previews.map((preview) { + final builder = preview.toBuilder() + ..group = 'Brightness' + ..name = '$name - ${preview.brightness!.name}'; + return builder.toPreview(); + }).toList(); + } +} + +@MultiBrightnessPreview(name: 'Primary Card') +Widget cardPreview() => const Card(child: Padding(padding: EdgeInsets.all(8.0), child: Text('Content'))); +``` diff --git a/.claude/skills/flutter-add-widget-test/SKILL.md b/.claude/skills/flutter-add-widget-test/SKILL.md new file mode 100644 index 0000000..01ac7ac --- /dev/null +++ b/.claude/skills/flutter-add-widget-test/SKILL.md @@ -0,0 +1,154 @@ +--- +name: flutter-add-widget-test +description: Implement a component-level test using `WidgetTester` to verify UI rendering and user interactions (tapping, scrolling, entering text). Use when validating that a specific widget displays correct data and responds to events as expected. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Tue, 21 Apr 2026 21:15:41 GMT +--- +# Writing Flutter Widget Tests + +## Contents +- [Setup & Configuration](#setup--configuration) +- [Core Components](#core-components) +- [Workflow: Implementing a Widget Test](#workflow-implementing-a-widget-test) +- [Interaction & State Management](#interaction--state-management) +- [Examples](#examples) + +## Setup & Configuration + +Ensure the testing environment is properly configured before authoring widget tests. + +1. Add the `flutter_test` dependency to the `dev_dependencies` section of `pubspec.yaml`. +2. Place all test files in the `test/` directory at the root of the project. +3. Suffix all test file names with `_test.dart` (e.g., `widget_test.dart`). + +## Core Components + +Utilize the following `flutter_test` components to interact with and validate the widget tree: + +* **`WidgetTester`**: The primary interface for building and interacting with widgets in the test environment. Provided automatically by the `testWidgets()` function. +* **`Finder`**: Locates widgets in the test environment (e.g., `find.text('Submit')`, `find.byType(TextField)`, `find.byKey(Key('submit_btn'))`). +* **`Matcher`**: Verifies the presence or state of widgets located by a `Finder` (e.g., `findsOneWidget`, `findsNothing`, `findsNWidgets(2)`, `matchesGoldenFile`). + +## Workflow: Implementing a Widget Test + +Copy the following checklist to track progress when implementing a new widget test. + +### Task Progress +- [ ] **Step 1: Define the test.** Use `testWidgets('description', (WidgetTester tester) async { ... })`. +- [ ] **Step 2: Build the widget.** Call `await tester.pumpWidget(MyWidget())` to render the UI. Wrap the widget in a `MaterialApp` or `Directionality` widget if it requires inherited directional or theme data. +- [ ] **Step 3: Locate elements.** Instantiate `Finder` objects for the target widgets. +- [ ] **Step 4: Verify initial state.** Use `expect(finder, matcher)` to validate the initial render. +- [ ] **Step 5: Simulate interactions.** Execute gestures or inputs (e.g., `await tester.tap(buttonFinder)`). +- [ ] **Step 6: Rebuild the tree.** Call `await tester.pump()` or `await tester.pumpAndSettle()` to process state changes. +- [ ] **Step 7: Verify updated state.** Use `expect()` to validate the UI after the interaction. +- [ ] **Step 8: Run and validate.** Execute `flutter test test/your_test_file_test.dart`. +- [ ] **Step 9: Feedback Loop.** Review test output -> identify failing matchers -> adjust widget logic or test assertions -> re-run until passing. + +## Interaction & State Management + +Apply the following conditional logic based on the type of interaction or state change being tested: + +* **If testing static rendering:** Call `await tester.pumpWidget()` once, then immediately run `expect()` assertions. +* **If testing standard state changes (e.g., button taps):** + 1. Call `await tester.tap(finder)`. + 2. Call `await tester.pump()` to trigger a single frame rebuild. +* **If testing animations, transitions, or asynchronous UI updates:** + 1. Trigger the action (e.g., `await tester.drag(finder, Offset(500, 0))`). + 2. Call `await tester.pumpAndSettle()` to repeatedly pump frames until no more frames are scheduled (animation completes). +* **If testing text input:** Call `await tester.enterText(textFieldFinder, 'Input string')`. +* **If testing items in a dynamic or long list:** Call `await tester.scrollUntilVisible(itemFinder, 500.0, scrollable: listFinder)` to ensure the target widget is rendered before interacting with it. + +## Examples + +### High-Fidelity Widget Test Implementation + +**Target Widget (`lib/todo_list.dart`):** +```dart +import 'package:flutter/material.dart'; + +class TodoList extends StatefulWidget { + const TodoList({super.key}); + + @override + State createState() => _TodoListState(); +} + +class _TodoListState extends State { + final todos = []; + final controller = TextEditingController(); + + @override + Widget build(BuildContext context) { + return MaterialApp( + home: Scaffold( + body: Column( + children: [ + TextField(controller: controller), + Expanded( + child: ListView.builder( + itemCount: todos.length, + itemBuilder: (context, index) { + final todo = todos[index]; + return Dismissible( + key: Key('$todo$index'), + onDismissed: (_) => setState(() => todos.removeAt(index)), + child: ListTile(title: Text(todo)), + ); + }, + ), + ), + ], + ), + floatingActionButton: FloatingActionButton( + onPressed: () { + setState(() { + todos.add(controller.text); + controller.clear(); + }); + }, + child: const Icon(Icons.add), + ), + ), + ); + } +} +``` + +**Test Implementation (`test/todo_list_test.dart`):** +```dart +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:my_app/todo_list.dart'; + +void main() { + testWidgets('Add and remove a todo item', (WidgetTester tester) async { + // 1. Build the widget + await tester.pumpWidget(const TodoList()); + + // 2. Verify initial state + expect(find.byType(ListTile), findsNothing); + + // 3. Enter text into the TextField + await tester.enterText(find.byType(TextField), 'Buy groceries'); + + // 4. Tap the add button + await tester.tap(find.byType(FloatingActionButton)); + + // 5. Rebuild the widget to reflect the new state + await tester.pump(); + + // 6. Verify the item was added + expect(find.text('Buy groceries'), findsOneWidget); + + // 7. Swipe the item to dismiss it + await tester.drag(find.byType(Dismissible), const Offset(500, 0)); + + // 8. Build the widget until the dismiss animation ends + await tester.pumpAndSettle(); + + // 9. Verify the item was removed + expect(find.text('Buy groceries'), findsNothing); + }); +} +``` diff --git a/.claude/skills/flutter-apply-architecture-best-practices/SKILL.md b/.claude/skills/flutter-apply-architecture-best-practices/SKILL.md new file mode 100644 index 0000000..791994b --- /dev/null +++ b/.claude/skills/flutter-apply-architecture-best-practices/SKILL.md @@ -0,0 +1,162 @@ +--- +name: flutter-apply-architecture-best-practices +description: Architects a Flutter application using the recommended layered approach (UI, Logic, Data). Use when structuring a new project or refactoring for scalability. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Tue, 21 Apr 2026 20:11:20 GMT +--- +# Architecting Flutter Applications + +## Contents +- [Architectural Layers](#architectural-layers) +- [Project Structure](#project-structure) +- [Workflow: Implementing a New Feature](#workflow-implementing-a-new-feature) +- [Examples](#examples) + +## Architectural Layers + +Enforce strict Separation of Concerns by dividing the application into distinct layers. Never mix UI rendering with business logic or data fetching. + +### UI Layer (Presentation) +Implement the MVVM (Model-View-ViewModel) pattern to manage UI state and logic. +* **Views:** Write reusable, lean widgets. Restrict logic in Views to UI-specific operations (e.g., animations, layout constraints, simple routing). Pass all required data from the ViewModel. +* **ViewModels:** Manage UI state and handle user interactions. Extend `ChangeNotifier` (or use `Listenable`) to expose state. Expose immutable state snapshots to the View. Inject Repositories into ViewModels via the constructor. + +### Data Layer +Implement the Repository pattern to isolate data access logic and create a single source of truth. +* **Services:** Create stateless classes to wrap external APIs (HTTP clients, local databases, platform plugins). Return raw API models or `Result` wrappers. +* **Repositories:** Consume one or more Services. Transform raw API models into clean Domain Models. Handle caching, offline synchronization, and retry logic. Expose Domain Models to ViewModels. + +### Logic Layer (Domain - Optional) +* **Use Cases:** Implement this layer only if the application contains complex business logic that clutters the ViewModel, or if logic must be reused across multiple ViewModels. Extract this logic into dedicated Use Case (interactor) classes that sit between ViewModels and Repositories. + +## Project Structure + +Organize the codebase using a hybrid approach: group UI components by feature, and group Data/Domain components by type. + +```text +lib/ +├── data/ +│ ├── models/ # API models +│ ├── repositories/ # Repository implementations +│ └── services/ # API clients, local storage wrappers +├── domain/ +│ ├── models/ # Clean domain models +│ └── use_cases/ # Optional business logic classes +└── ui/ + ├── core/ # Shared widgets, themes, typography + └── features/ + └── [feature_name]/ + ├── view_models/ + └── views/ +``` + +## Workflow: Implementing a New Feature + +Follow this sequential workflow when adding a new feature to the application. Copy the checklist to track progress. + +### Task Progress +- [ ] **Step 1: Define Domain Models.** Create immutable data classes for the feature using `freezed` or `built_value`. +- [ ] **Step 2: Implement Services.** Create or update Service classes to handle external API communication. +- [ ] **Step 3: Implement Repositories.** Create the Repository to consume Services and return Domain Models. +- [ ] **Step 4: Apply Conditional Logic (Domain Layer).** + - *If the feature requires complex data transformation or cross-repository logic:* Create a Use Case class. + - *If the feature is a simple CRUD operation:* Skip to Step 5. +- [ ] **Step 5: Implement the ViewModel.** Create the ViewModel extending `ChangeNotifier`. Inject required Repositories/Use Cases. Expose immutable state and command methods. +- [ ] **Step 6: Implement the View.** Create the UI widget. Use `ListenableBuilder` or `AnimatedBuilder` to listen to ViewModel changes. +- [ ] **Step 7: Inject Dependencies.** Register the new Service, Repository, and ViewModel in the dependency injection container (e.g., `provider` or `get_it`). +- [ ] **Step 8: Run Validator.** Execute unit tests for the ViewModel and Repository. + - *Feedback Loop:* Run tests -> Review failures -> Fix logic -> Re-run until passing. + +## Examples + +### Data Layer: Service and Repository + +```dart +// 1. Service (Raw API interaction) +class ApiClient { + Future fetchUser(String id) async { + // HTTP GET implementation... + } +} + +// 2. Repository (Single source of truth, returns Domain Model) +class UserRepository { + UserRepository({required ApiClient apiClient}) : _apiClient = apiClient; + + final ApiClient _apiClient; + User? _cachedUser; + + Future getUser(String id) async { + if (_cachedUser != null) return _cachedUser!; + + final apiModel = await _apiClient.fetchUser(id); + _cachedUser = User(id: apiModel.id, name: apiModel.fullName); // Transform to Domain Model + return _cachedUser!; + } +} +``` + +### UI Layer: ViewModel and View + +```dart +// 3. ViewModel (State management and presentation logic) +class ProfileViewModel extends ChangeNotifier { + ProfileViewModel({required UserRepository userRepository}) + : _userRepository = userRepository; + + final UserRepository _userRepository; + + User? _user; + User? get user => _user; + + bool _isLoading = false; + bool get isLoading => _isLoading; + + Future loadProfile(String id) async { + _isLoading = true; + notifyListeners(); + + try { + _user = await _userRepository.getUser(id); + } finally { + _isLoading = false; + notifyListeners(); + } + } +} + +// 4. View (Dumb UI component) +class ProfileView extends StatelessWidget { + const ProfileView({super.key, required this.viewModel}); + + final ProfileViewModel viewModel; + + @override + Widget build(BuildContext context) { + return ListenableBuilder( + listenable: viewModel, + builder: (context, _) { + if (viewModel.isLoading) { + return const Center(child: CircularProgressIndicator()); + } + + final user = viewModel.user; + if (user == null) { + return const Center(child: Text('User not found')); + } + + return Column( + children: [ + Text(user.name), + ElevatedButton( + onPressed: () => viewModel.loadProfile(user.id), + child: const Text('Refresh'), + ), + ], + ); + }, + ); + } +} +``` diff --git a/.claude/skills/flutter-build-responsive-layout/SKILL.md b/.claude/skills/flutter-build-responsive-layout/SKILL.md new file mode 100644 index 0000000..b85bfd7 --- /dev/null +++ b/.claude/skills/flutter-build-responsive-layout/SKILL.md @@ -0,0 +1,139 @@ +--- +name: flutter-build-responsive-layout +description: Use `LayoutBuilder`, `MediaQuery`, or `Expanded/Flexible` to create a layout that adapts to different screen sizes. Use when you need the UI to look good on both mobile and tablet/desktop form factors. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Tue, 21 Apr 2026 20:17:40 GMT +--- +# Implementing Adaptive Layouts + +## Contents +- [Space Measurement Guidelines](#space-measurement-guidelines) +- [Widget Sizing and Constraints](#widget-sizing-and-constraints) +- [Device and Orientation Behaviors](#device-and-orientation-behaviors) +- [Workflow: Constructing an Adaptive Layout](#workflow-constructing-an-adaptive-layout) +- [Workflow: Optimizing for Large Screens](#workflow-optimizing-for-large-screens) +- [Examples](#examples) + +## Space Measurement Guidelines +Determine the available space accurately to ensure layouts adapt to the app window, not just the physical device. + +* **Use `MediaQuery.sizeOf(context)`** to get the size of the entire app window. +* **Use `LayoutBuilder`** to make layout decisions based on the parent widget's allocated space. Evaluate `constraints.maxWidth` to determine the appropriate widget tree to return. +* **Do not use `MediaQuery.orientationOf` or `OrientationBuilder`** near the top of the widget tree to switch layouts. Device orientation does not accurately reflect the available app window space. +* **Do not check for hardware types** (e.g., "phone" vs. "tablet"). Flutter apps run in resizable windows, multi-window modes, and picture-in-picture. Base all layout decisions strictly on available window space. + +## Widget Sizing and Constraints +Understand and apply Flutter's core layout rule: **Constraints go down. Sizes go up. Parent sets position.** + +* **Distribute Space:** Use `Expanded` and `Flexible` within `Row`, `Column`, or `Flex` widgets. + * Use `Expanded` to force a child to fill all remaining available space (equivalent to `Flexible` with `fit: FlexFit.tight` and a `flex` factor of 1.0). + * Use `Flexible` to allow a child to size itself up to a specific limit while still expanding/contracting. Use the `flex` factor to define the ratio of space consumption among siblings. +* **Constrain Width:** Prevent widgets from consuming all horizontal space on large screens. Wrap widgets like `GridView` or `ListView` in a `ConstrainedBox` or `Container` and define a `maxWidth` in the `BoxConstraints`. +* **Lazy Rendering:** Always use `ListView.builder` or `GridView.builder` when rendering lists with an unknown or large number of items. + +## Device and Orientation Behaviors +Ensure the app behaves correctly across all device form factors and input methods. + +* **Do not lock screen orientation.** Locking orientation causes severe layout issues on foldable devices, often resulting in letterboxing (the app centered with black borders). Android large format tiers require both portrait and landscape support. +* **Fallback for Locked Orientation:** If business requirements strictly mandate a locked orientation, use the `Display API` to retrieve physical screen dimensions instead of `MediaQuery`. `MediaQuery` fails to receive the larger window size in compatibility modes. +* **Support Multiple Inputs:** Implement support for basic mice, trackpads, and keyboard shortcuts. Ensure touch targets are appropriately sized and keyboard navigation is accessible. + +## Workflow: Constructing an Adaptive Layout + +Follow this workflow to implement a layout that adapts to the available `BoxConstraints`. + +**Task Progress:** +- [ ] Identify the target widget that requires adaptive behavior. +- [ ] Wrap the widget tree in a `LayoutBuilder`. +- [ ] Extract the `constraints.maxWidth` from the builder callback. +- [ ] Define an adaptive breakpoint (e.g., `largeScreenMinWidth = 600`). +- [ ] **If `maxWidth > largeScreenMinWidth`:** Return a large-screen layout (e.g., a `Row` placing a navigation sidebar and content area side-by-side). +- [ ] **If `maxWidth <= largeScreenMinWidth`:** Return a small-screen layout (e.g., a `Column` or standard navigation-style approach). +- [ ] Run validator -> resize the application window -> review layout transitions -> fix overflow errors. + +## Workflow: Optimizing for Large Screens + +Follow this workflow to prevent UI elements from stretching unnaturally on large displays. + +**Task Progress:** +- [ ] Identify full-width components (e.g., `ListView`, text blocks, forms). +- [ ] **If optimizing a list:** Convert `ListView.builder` to `GridView.builder` using `SliverGridDelegateWithMaxCrossAxisExtent` to automatically adjust column counts based on window size. +- [ ] **If optimizing a form or text block:** Wrap the component in a `ConstrainedBox`. +- [ ] Apply `BoxConstraints(maxWidth: [optimal_width])` to the `ConstrainedBox`. +- [ ] Wrap the `ConstrainedBox` in a `Center` widget to keep the constrained content centered on large screens. +- [ ] Run validator -> test on desktop/tablet target -> review horizontal stretching -> adjust `maxWidth` or grid extents. + +## Examples + +### Adaptive Layout using LayoutBuilder +Demonstrates switching between a mobile and desktop layout based on available width. + +```dart +import 'package:flutter/material.dart'; + +const double largeScreenMinWidth = 600.0; + +class AdaptiveLayout extends StatelessWidget { + const AdaptiveLayout({super.key}); + + @override + Widget build(BuildContext context) { + return LayoutBuilder( + builder: (context, constraints) { + if (constraints.maxWidth > largeScreenMinWidth) { + return _buildLargeScreenLayout(); + } else { + return _buildSmallScreenLayout(); + } + }, + ); + } + + Widget _buildLargeScreenLayout() { + return Row( + children: [ + const SizedBox(width: 250, child: Placeholder(color: Colors.blue)), + const VerticalDivider(width: 1), + Expanded(child: const Placeholder(color: Colors.green)), + ], + ); + } + + Widget _buildSmallScreenLayout() { + return const Placeholder(color: Colors.green); + } +} +``` + +### Constraining Width on Large Screens +Demonstrates preventing a widget from consuming all horizontal space. + +```dart +import 'package:flutter/material.dart'; + +class ConstrainedContent extends StatelessWidget { + const ConstrainedContent({super.key}); + + @override + Widget build(BuildContext context) { + return Scaffold( + body: Center( + child: ConstrainedBox( + constraints: const BoxConstraints( + maxWidth: 800.0, // Maximum width for readability + ), + child: ListView.builder( + itemCount: 50, + itemBuilder: (context, index) { + return ListTile( + title: Text('Item $index'), + ); + }, + ), + ), + ), + ); + } +} +``` diff --git a/.claude/skills/flutter-fix-layout-issues/SKILL.md b/.claude/skills/flutter-fix-layout-issues/SKILL.md new file mode 100644 index 0000000..3804a3c --- /dev/null +++ b/.claude/skills/flutter-fix-layout-issues/SKILL.md @@ -0,0 +1,130 @@ +--- +name: flutter-fix-layout-issues +description: Fixes Flutter layout errors (overflows, unbounded constraints) using Dart and Flutter MCP tools. Use when addressing "RenderFlex overflowed", "Vertical viewport was given unbounded height", or similar layout issues. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Tue, 21 Apr 2026 19:45:59 GMT +--- +# Resolving Flutter Layout Errors + +## Contents +- [Constraint Violation Diagnostics](#constraint-violation-diagnostics) +- [Layout Error Resolution Workflow](#layout-error-resolution-workflow) +- [Examples](#examples) + +## Constraint Violation Diagnostics + +Flutter layout operates on a strict rule: **Constraints go down. Sizes go up. Parent sets position.** Layout errors occur when this negotiation fails, typically due to unbounded constraints or unconstrained children. + +Diagnose layout failures using the following error signatures: + +* **"Vertical viewport was given unbounded height"**: Triggered when a scrollable widget (`ListView`, `GridView`) is placed inside an unconstrained vertical parent (`Column`). The parent provides infinite height, and the child attempts to expand infinitely. +* **"An InputDecorator...cannot have an unbounded width"**: Triggered when a `TextField` or `TextFormField` is placed inside an unconstrained horizontal parent (`Row`). The text field attempts to determine its width based on infinite available space. +* **"RenderFlex overflowed"**: Triggered when a child of a `Row` or `Column` requests a size larger than the parent's allocated constraints. Visually indicated by yellow and black warning stripes. +* **"Incorrect use of ParentData widget"**: Triggered when a `ParentDataWidget` is not a direct descendant of its required ancestor. (e.g., `Expanded` outside a `Flex`, `Positioned` outside a `Stack`). +* **"RenderBox was not laid out"**: A cascading side-effect error. Ignore this and look further up the stack trace for the primary constraint violation (usually an unbounded height/width error). + +## Layout Error Resolution Workflow + +Copy and use this checklist to systematically resolve layout constraint violations. + +### Task Progress +- [ ] Run the application in debug mode to capture the exact layout exception in the console. +- [ ] Identify the primary error message (ignore cascading "RenderBox was not laid out" errors). +- [ ] Apply the conditional fix based on the specific error type: + - **If "Vertical viewport was given unbounded height"**: Wrap the scrollable child (`ListView`, `GridView`) in an `Expanded` widget to consume remaining space, or wrap it in a `SizedBox` to provide an absolute height constraint. + - **If "An InputDecorator...cannot have an unbounded width"**: Wrap the `TextField` or `TextFormField` in an `Expanded` or `Flexible` widget. + - **If "RenderFlex overflowed"**: Constrain the overflowing child by wrapping it in an `Expanded` widget (to force it to fit) or a `Flexible` widget (to allow it to be smaller than the allocated space). + - **If "Incorrect use of ParentData widget"**: Move the `ParentDataWidget` to be a direct child of its required parent. Ensure `Expanded`/`Flexible` are direct children of `Row`/`Column`/`Flex`. Ensure `Positioned` is a direct child of `Stack`. +- [ ] Execute Flutter hot reload. +- [ ] Run validator -> review errors -> fix: Inspect the UI to verify the red/grey error screen or yellow/black overflow stripes are resolved. If new layout errors appear, repeat the workflow. + +## Examples + +### Fixing Unbounded Height (ListView in Column) + +**Input (Error State):** +```dart +// Throws "Vertical viewport was given unbounded height" +Column( + children: [ + const Text('Header'), + ListView( + children: const [ + ListTile(title: Text('Item 1')), + ListTile(title: Text('Item 2')), + ], + ), + ], +) +``` + +**Output (Resolved State):** +```dart +// Wrap ListView in Expanded to constrain its height to the remaining Column space +Column( + children: [ + const Text('Header'), + Expanded( + child: ListView( + children: const [ + ListTile(title: Text('Item 1')), + ListTile(title: Text('Item 2')), + ], + ), + ), + ], +) +``` + +### Fixing Unbounded Width (TextField in Row) + +**Input (Error State):** +```dart +// Throws "An InputDecorator...cannot have an unbounded width" +Row( + children: [ + const Icon(Icons.search), + TextField(), + ], +) +``` + +**Output (Resolved State):** +```dart +// Wrap TextField in Expanded to constrain its width to the remaining Row space +Row( + children: [ + const Icon(Icons.search), + Expanded( + child: TextField(), + ), + ], +) +``` + +### Fixing RenderFlex Overflow + +**Input (Error State):** +```dart +// Throws "A RenderFlex overflowed by X pixels on the right" +Row( + children: [ + const Icon(Icons.info), + const Text('This is a very long text string that will definitely overflow the available screen width and cause a RenderFlex error.'), + ], +) +``` + +**Output (Resolved State):** +```dart +// Wrap the Text widget in Expanded to force it to wrap within the available constraints +Row( + children: [ + const Icon(Icons.info), + Expanded( + child: const Text('This is a very long text string that will definitely overflow the available screen width and cause a RenderFlex error.'), + ), + ], +) +``` diff --git a/.claude/skills/flutter-implement-json-serialization/SKILL.md b/.claude/skills/flutter-implement-json-serialization/SKILL.md new file mode 100644 index 0000000..14009f4 --- /dev/null +++ b/.claude/skills/flutter-implement-json-serialization/SKILL.md @@ -0,0 +1,153 @@ +--- +name: flutter-implement-json-serialization +description: Create model classes with `fromJson` and `toJson` methods using `dart:convert`. Use when manually mapping JSON keys to class properties for simple data structures. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Tue, 21 Apr 2026 21:44:50 GMT +--- +# Serializing JSON Manually in Flutter + +## Contents +- [Core Guidelines](#core-guidelines) +- [Workflow: Implementing a Serializable Model](#workflow-implementing-a-serializable-model) +- [Workflow: Fetching and Parsing JSON](#workflow-fetching-and-parsing-json) +- [Examples](#examples) + +## Core Guidelines + +- **Import `dart:convert`**: Utilize Flutter's built-in `dart:convert` library for manual JSON encoding (`jsonEncode`) and decoding (`jsonDecode`). +- **Enforce Type Safety**: Always cast the `dynamic` result of `jsonDecode()` to the expected type, typically `Map` for objects or `List` for arrays. +- **Encapsulate Serialization Logic**: Define plain model classes containing properties corresponding to the JSON structure. Implement a `fromJson` factory constructor and a `toJson` method within the model. +- **Handle Background Parsing**: If parsing large JSON documents (execution time > 16ms), offload the parsing logic to a separate isolate using Flutter's `compute()` function to prevent UI jank. +- **Throw Exceptions on Failure**: When handling HTTP responses, throw an exception if the status code is not successful (e.g., not 200 OK or 201 Created). Do not return `null`. + +## Workflow: Implementing a Serializable Model + +Use this checklist to implement manual JSON serialization for a data model. + +**Task Progress:** +- [ ] Define the plain model class with `final` properties. +- [ ] Implement the `factory Model.fromJson(Map json)` constructor. +- [ ] Implement the `Map toJson()` method. +- [ ] Write unit tests for both serialization methods. +- [ ] Run validator -> review type mismatch errors -> fix casting logic. + +1. **Define the Model**: Create a class with properties matching the JSON keys. +2. **Implement `fromJson`**: Extract values from the `Map` and cast them to the appropriate Dart types. Use pattern matching or explicit casting. +3. **Implement `toJson`**: Return a `Map` mapping the class properties back to their JSON string keys. +4. **Validate**: Execute unit tests to ensure type safety, autocompletion, and compile-time exception handling function correctly. + +## Workflow: Fetching and Parsing JSON + +Use this conditional workflow when retrieving and parsing JSON from a network request. + +**Task Progress:** +- [ ] Execute the HTTP request. +- [ ] Validate the response status code. +- [ ] Determine parsing strategy (Synchronous vs. Isolate). +- [ ] Decode and map the JSON to the model. + +1. **Execute Request**: Use the `http` package to perform the network call. +2. **Validate Response**: + - If `response.statusCode == 200` (or 201 for POST), proceed to parsing. + - If the status code indicates failure, throw an `Exception`. +3. **Determine Parsing Strategy**: + - If parsing a **small payload** (e.g., a single object), parse synchronously on the main thread. + - If parsing a **large payload** (e.g., an array of thousands of objects), use `compute(parseFunction, response.body)` to parse in a background isolate. +4. **Decode and Map**: Pass the decoded JSON to your model's `fromJson` constructor. + +## Examples + +### High-Fidelity Model Implementation + +```dart +import 'dart:convert'; + +class User { + final int id; + final String name; + final String email; + + const User({ + required this.id, + required this.name, + required this.email, + }); + + // Factory constructor for deserialization + factory User.fromJson(Map json) { + return switch (json) { + { + 'id': int id, + 'name': String name, + 'email': String email, + } => + User( + id: id, + name: name, + email: email, + ), + _ => throw const FormatException('Failed to load User.'), + }; + } + + // Method for serialization + Map toJson() { + return { + 'id': id, + 'name': name, + 'email': email, + }; + } +} +``` + +### Synchronous Parsing (Small Payload) + +```dart +import 'dart:convert'; +import 'package:http/http.dart' as http; + +Future fetchUser(http.Client client, int userId) async { + final response = await client.get( + Uri.parse('https://api.example.com/users/$userId'), + headers: {'Accept': 'application/json'}, + ); + + if (response.statusCode == 200) { + // Decode returns dynamic, cast to Map + final Map jsonMap = jsonDecode(response.body) as Map; + return User.fromJson(jsonMap); + } else { + throw Exception('Failed to load user'); + } +} +``` + +### Background Parsing (Large Payload) + +```dart +import 'dart:convert'; +import 'package:flutter/foundation.dart'; +import 'package:http/http.dart' as http; + +// Top-level function required for compute() +List parseUsers(String responseBody) { + final parsed = (jsonDecode(responseBody) as List).cast>(); + return parsed.map((json) => User.fromJson(json)).toList(); +} + +Future> fetchUsers(http.Client client) async { + final response = await client.get( + Uri.parse('https://api.example.com/users'), + headers: {'Accept': 'application/json'}, + ); + + if (response.statusCode == 200) { + // Offload expensive parsing to a background isolate + return compute(parseUsers, response.body); + } else { + throw Exception('Failed to load users'); + } +} +``` diff --git a/.claude/skills/flutter-setup-declarative-routing/SKILL.md b/.claude/skills/flutter-setup-declarative-routing/SKILL.md new file mode 100644 index 0000000..2720311 --- /dev/null +++ b/.claude/skills/flutter-setup-declarative-routing/SKILL.md @@ -0,0 +1,255 @@ +--- +name: flutter-setup-declarative-routing +description: Configure `MaterialApp.router` using a package like `go_router` for advanced URL-based navigation. Use when developing web applications or mobile apps that require specific deep linking and browser history support. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Tue, 21 Apr 2026 21:08:03 GMT +--- +# Implementing Routing and Deep Linking + +## Contents +- [Core Concepts](#core-concepts) +- [Workflow: Initializing the Application and Router](#workflow-initializing-the-application-and-router) +- [Workflow: Configuring Platform Deep Linking](#workflow-configuring-platform-deep-linking) +- [Workflow: Implementing Nested Navigation](#workflow-implementing-nested-navigation) +- [Examples](#examples) + +## Core Concepts + +Use the `go_router` package for declarative routing in Flutter. It provides a robust API for complex routing scenarios, deep linking, and nested navigation. + +- **GoRouter**: The central configuration object defining the application's route tree. +- **GoRoute**: A standard route mapping a URL path to a Flutter screen. +- **ShellRoute / StatefulShellRoute**: Wraps child routes in a persistent UI shell (e.g., a `BottomNavigationBar`). `StatefulShellRoute` maintains the state of parallel navigation branches. +- **Path URL Strategy**: Removes the default `#` fragment from web URLs, essential for clean deep linking across platforms. + +## Workflow: Initializing the Application and Router + +Follow this workflow to bootstrap a new Flutter application with `go_router` and configure the root routing mechanism. + +### Task Progress +- [ ] Create the Flutter application. +- [ ] Add the `go_router` dependency. +- [ ] Configure the URL strategy for web/deep linking. +- [ ] Implement the `GoRouter` configuration. +- [ ] Bind the router to `MaterialApp.router`. + +### 1. Scaffold the Application +Run the following commands to create the app and add the required routing package: +```bash +flutter create +cd +flutter pub add go_router +``` + +### 2. Configure the Router +Define a top-level `GoRouter` instance. Handle authentication or state-based routing using the `redirect` parameter. + +```dart +import 'package:flutter/material.dart'; +import 'package:go_router/go_router.dart'; +import 'package:flutter_web_plugins/url_strategy.dart'; + +void main() { + // Use path URL strategy to remove the '#' from web URLs + usePathUrlStrategy(); + runApp(const MyApp()); +} + +final GoRouter _router = GoRouter( + initialLocation: '/', + routes: [ + GoRoute( + path: '/', + builder: (context, state) => const HomeScreen(), + routes: [ + GoRoute( + path: 'details/:id', + builder: (context, state) => DetailsScreen(id: state.pathParameters['id']!), + ), + ], + ), + ], + errorBuilder: (context, state) => ErrorScreen(error: state.error), +); + +class MyApp extends StatelessWidget { + const MyApp({super.key}); + + @override + Widget build(BuildContext context) { + return MaterialApp.router( + routerConfig: _router, + title: 'Routing App', + ); + } +} +``` + +## Workflow: Configuring Platform Deep Linking + +Configure the native platforms to intercept specific URLs and route them into the Flutter application. + +### Task Progress +- [ ] Determine target platforms (iOS, Android, or both). +- [ ] Apply conditional configuration for Android (Manifest + Asset Links). +- [ ] Apply conditional configuration for iOS (Plist + Entitlements + AASA). +- [ ] Run validator -> review errors -> fix. + +### If configuring for Android: +1. **Modify `AndroidManifest.xml`**: Add the intent filter inside the `` tag for `.MainActivity`. +```xml + + + + + + + +``` +2. **Host `assetlinks.json`**: Serve the following JSON at `https://yourdomain.com/.well-known/assetlinks.json`. +```json +[{ + "relation": ["delegate_permission/common.handle_all_urls"], + "target": { + "namespace": "android_app", + "package_name": "com.yourcompany.yourapp", + "sha256_cert_fingerprints": ["YOUR_SHA256_FINGERPRINT"] + } +}] +``` + +### If configuring for iOS: +1. **Modify `Info.plist`**: Opt-in to Flutter's default deep link handler. +*Note: If using a third-party deep linking plugin (e.g., `app_links`), set this to `NO` to prevent conflicts.* +```xml +FlutterDeepLinkingEnabled + +``` +2. **Modify `Runner.entitlements`**: Add the associated domain. +```xml +com.apple.developer.associated-domains + + applinks:yourdomain.com + +``` +3. **Host `apple-app-site-association`**: Serve the following JSON (without a `.json` extension) at `https://yourdomain.com/.well-known/apple-app-site-association`. +```json +{ + "applinks": { + "apps": [], + "details": [{ + "appIDs": ["TEAM_ID.com.yourcompany.yourapp"], + "paths": ["*"], + "components": [{"/": "/*"}] + }] + } +} +``` + +### Validation Loop +Run validator -> review errors -> fix. +- **Android**: Test using ADB. + ```bash + adb shell 'am start -a android.intent.action.VIEW -c android.intent.category.BROWSABLE -d "https://yourdomain.com/details/123"' com.yourcompany.yourapp + ``` +- **iOS**: Test using `xcrun` on a booted simulator. + ```bash + xcrun simctl openurl booted https://yourdomain.com/details/123 + ``` + +## Workflow: Implementing Nested Navigation + +Use `StatefulShellRoute` to implement persistent UI shells (like a bottom navigation bar) that maintain the state of their child routes. + +### Task Progress +- [ ] Define `StatefulShellRoute.indexedStack` in the `GoRouter` configuration. +- [ ] Create `StatefulShellBranch` instances for each navigation tab. +- [ ] Implement the shell widget using `StatefulNavigationShell`. + +```dart +final GoRouter _router = GoRouter( + initialLocation: '/home', + routes: [ + StatefulShellRoute.indexedStack( + builder: (context, state, navigationShell) { + return ScaffoldWithNavBar(navigationShell: navigationShell); + }, + branches: [ + StatefulShellBranch( + routes: [ + GoRoute( + path: '/home', + builder: (context, state) => const HomeScreen(), + ), + ], + ), + StatefulShellBranch( + routes: [ + GoRoute( + path: '/settings', + builder: (context, state) => const SettingsScreen(), + ), + ], + ), + ], + ), + ], +); +``` + +## Examples + +### High-Fidelity Shell Widget Implementation +Implement the UI shell that consumes the `StatefulNavigationShell` to handle branch switching. + +```dart +class ScaffoldWithNavBar extends StatelessWidget { + const ScaffoldWithNavBar({ + required this.navigationShell, + super.key, + }); + + final StatefulNavigationShell navigationShell; + + void _goBranch(int index) { + navigationShell.goBranch( + index, + // Support navigating to the initial location when tapping the active tab. + initialLocation: index == navigationShell.currentIndex, + ); + } + + @override + Widget build(BuildContext context) { + return Scaffold( + body: navigationShell, + bottomNavigationBar: NavigationBar( + selectedIndex: navigationShell.currentIndex, + onDestinationSelected: _goBranch, + destinations: const [ + NavigationDestination(icon: Icon(Icons.home), label: 'Home'), + NavigationDestination(icon: Icon(Icons.settings), label: 'Settings'), + ], + ), + ); + } +} +``` + +### Programmatic Navigation +Use the `context.go()` and `context.push()` extension methods provided by `go_router`. + +```dart +// Replaces the current route stack with the target route (Declarative) +context.go('/details/123'); + +// Pushes the target route onto the existing stack (Imperative) +context.push('/details/123'); + +// Navigates using a named route and path parameters +context.goNamed('details', pathParameters: {'id': '123'}); + +// Pops the current route +context.pop(); +``` diff --git a/.claude/skills/flutter-setup-localization/SKILL.md b/.claude/skills/flutter-setup-localization/SKILL.md new file mode 100644 index 0000000..d3dd459 --- /dev/null +++ b/.claude/skills/flutter-setup-localization/SKILL.md @@ -0,0 +1,210 @@ +--- +name: flutter-setup-localization +description: Add `flutter_localizations` and `intl` dependencies, enable "generate true" in `pubspec.yaml`, and create an `l10n.yaml` configuration file. Use when initializing localization support for a new Flutter project. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Tue, 21 Apr 2026 21:27:35 GMT +--- +# Internationalizing Flutter Applications + +## Contents +- [Core Concepts](#core-concepts) +- [Setup Workflow](#setup-workflow) +- [Implementation Workflow](#implementation-workflow) +- [Advanced Formatting](#advanced-formatting) +- [Examples](#examples) + +## Core Concepts +Flutter handles internationalization (i18n) and localization (l10n) via the `flutter_localizations` and `intl` packages. The standard approach uses App Resource Bundle (`.arb`) files to define localized strings, which are then compiled into a generated `AppLocalizations` class for type-safe access within the widget tree. + +## Setup Workflow + +Copy and track this checklist when initializing internationalization in a Flutter project: + +- [ ] **Task Progress** + - [ ] 1. Add dependencies to `pubspec.yaml`. + - [ ] 2. Enable the `generate` flag. + - [ ] 3. Create the `l10n.yaml` configuration file. + - [ ] 4. Configure `MaterialApp` or `CupertinoApp`. + +### 1. Add Dependencies +Add the required localization packages to the project. Execute the following commands in the terminal: +```bash +flutter pub add flutter_localizations --sdk=flutter +flutter pub add intl:any +``` + +Verify your `pubspec.yaml` includes the following under `dependencies`: +```yaml +dependencies: + flutter: + sdk: flutter + flutter_localizations: + sdk: flutter + intl: any +``` + +### 2. Enable Code Generation +Open `pubspec.yaml` and enable the `generate` flag within the `flutter` section to automate localization tasks: +```yaml +flutter: + generate: true +``` + +### 3. Create Configuration File +Create a new file named `l10n.yaml` in the root directory of the Flutter project. Define the input directory, template file, and output file: +```yaml +arb-dir: lib/l10n +template-arb-file: app_en.arb +output-localization-file: app_localizations.dart +synthetic-package: true +``` + +### 4. Configure the App Entry Point +Import the generated localizations and the `flutter_localizations` library in your `main.dart`. Inject the delegates and supported locales into your `MaterialApp` or `CupertinoApp`. + +```dart +import 'package:flutter_localizations/flutter_localizations.dart'; +import 'package:flutter_gen/gen_l10n/app_localizations.dart'; // Adjust path if synthetic-package is false + +// ... inside build method +return MaterialApp( + localizationsDelegates: const [ + AppLocalizations.delegate, + GlobalMaterialLocalizations.delegate, + GlobalWidgetsLocalizations.delegate, + GlobalCupertinoLocalizations.delegate, + ], + supportedLocales: const [ + Locale('en'), // English + Locale('es'), // Spanish + ], + home: const MyHomePage(), +); +``` + +## Implementation Workflow + +Follow this workflow when adding or modifying localized content. + +### 1. Define ARB Files +* **If creating NEW content:** Add the base string to the template file (`lib/l10n/app_en.arb`). Include a description for context. +* **If EDITING existing content:** Locate the key in all supported `.arb` files and update the values. + +```json +{ + "helloWorld": "Hello World!", + "@helloWorld": { + "description": "The conventional newborn programmer greeting" + } +} +``` + +Create corresponding files for other locales (e.g., `app_es.arb`): +```json +{ + "helloWorld": "¡Hola Mundo!" +} +``` + +### 2. Generate Localization Classes +Run the following command to trigger code generation: +```bash +flutter pub get +``` +*Feedback Loop:* Run validator -> review terminal output for ARB syntax errors -> fix missing commas or mismatched placeholders -> re-run `flutter pub get`. + +### 3. Consume Localized Strings +Access the localized strings in your widget tree using `AppLocalizations.of(context)`. Ensure the widget calling this is a descendant of `MaterialApp`. + +```dart +Text(AppLocalizations.of(context)!.helloWorld) +``` + +## Advanced Formatting + +Use placeholders for dynamic data, plurals, and conditional selects. + +### Placeholders +Define parameters within curly braces and specify their type in the metadata object. +```json +"hello": "Hello {userName}", +"@hello": { + "description": "A message with a single parameter", + "placeholders": { + "userName": { + "type": "String", + "example": "Bob" + } + } +} +``` + +### Plurals +Use the `plural` syntax to handle quantity-based string variations. The `other` case is mandatory. +```json +"nWombats": "{count, plural, =0{no wombats} =1{1 wombat} other{{count} wombats}}", +"@nWombats": { + "description": "A plural message", + "placeholders": { + "count": { + "type": "num", + "format": "compact" + } + } +} +``` + +### Selects +Use the `select` syntax for conditional strings, such as gendered text. +```json +"pronoun": "{gender, select, male{he} female{she} other{they}}", +"@pronoun": { + "description": "A gendered message", + "placeholders": { + "gender": { + "type": "String" + } + } +} +``` + +## Examples + +### Complete `l10n.yaml` +```yaml +arb-dir: lib/l10n +template-arb-file: app_en.arb +output-localization-file: app_localizations.dart +synthetic-package: true +use-escaping: true +``` + +### Complete Widget Implementation +```dart +import 'package:flutter/material.dart'; +import 'package:flutter_gen/gen_l10n/app_localizations.dart'; + +class GreetingWidget extends StatelessWidget { + final String userName; + final int notificationCount; + + const GreetingWidget({ + super.key, + required this.userName, + required this.notificationCount, + }); + + @override + Widget build(BuildContext context) { + final l10n = AppLocalizations.of(context)!; + + return Column( + children: [ + Text(l10n.hello(userName)), + Text(l10n.nWombats(notificationCount)), + ], + ); + } +} +``` diff --git a/.claude/skills/flutter-use-http-package/SKILL.md b/.claude/skills/flutter-use-http-package/SKILL.md new file mode 100644 index 0000000..bb60468 --- /dev/null +++ b/.claude/skills/flutter-use-http-package/SKILL.md @@ -0,0 +1,174 @@ +--- +name: flutter-use-http-package +description: Use the `http` package to execute GET, POST, PUT, or DELETE requests. Use when you need to fetch from or send data to a REST API. +metadata: + model: models/gemini-3.1-pro-preview + last_modified: Tue, 21 Apr 2026 21:36:42 GMT +--- +# Implementing Flutter Networking + +## Contents +- [Configuration & Permissions](#configuration--permissions) +- [Request Execution & Response Handling](#request-execution--response-handling) +- [Background Parsing](#background-parsing) +- [Workflow: Executing Network Operations](#workflow-executing-network-operations) +- [Examples](#examples) + +## Configuration & Permissions + +Configure the environment and platform-specific permissions required for network access. + +1. Add the `http` package dependency via the terminal: + ```bash + flutter pub add http + ``` +2. Import the package in your Dart files: + ```dart + import 'package:http/http.dart' as http; + ``` +3. Configure Android permissions by adding the Internet permission to `android/app/src/main/AndroidManifest.xml`: + ```xml + + ``` +4. Configure macOS entitlements by adding the network client key to both `macos/Runner/DebugProfile.entitlements` and `macos/Runner/Release.entitlements`: + ```xml + com.apple.security.network.client + + ``` + +## Request Execution & Response Handling + +Execute HTTP operations and map responses to strongly typed Dart objects. + +* **URIs:** Always parse URL strings using `Uri.parse('your_url')`. +* **Headers:** Inject authorization and content-type headers via the `headers` parameter map. Use `HttpHeaders.authorizationHeader` for auth tokens. +* **Payloads:** For POST and PUT requests, encode the body using `jsonEncode()` from `dart:convert`. +* **Status Validation:** Evaluate `response.statusCode`. Treat `200 OK` (GET/PUT/DELETE) and `201 CREATED` (POST) as success. +* **Error Handling:** Throw explicit exceptions for non-success status codes. Never return `null` on failure, as this prevents `FutureBuilder` from triggering its error state and causes infinite loading indicators. +* **Deserialization:** Parse the raw string using `jsonDecode(response.body)` and map it to a custom Dart object using a factory constructor (e.g., `fromJson`). + +## Background Parsing + +Offload expensive JSON parsing to a separate Isolate to prevent UI jank (frame drops). + +* Import `package:flutter/foundation.dart`. +* Use the `compute()` function to run the parsing logic in a background isolate. +* Ensure the parsing function passed to `compute()` is a top-level function or a static method, as closures or instance methods cannot be passed across isolates. + +## Workflow: Executing Network Operations + +Use the following checklist to implement and validate network operations. + +**Task Progress:** +- [ ] 1. Define the strongly typed Dart model with a `fromJson` factory constructor. +- [ ] 2. Implement the network request method returning a `Future`. +- [ ] 3. Apply conditional logic based on the operation type: + - **If fetching data (GET):** Append query parameters to the URI. + - **If mutating data (POST/PUT):** Set `'Content-Type': 'application/json; charset=UTF-8'` and attach the `jsonEncode` body. + - **If deleting data (DELETE):** Return an empty model instance on success (`200 OK`). +- [ ] 4. Validate the `statusCode` and throw an `Exception` on failure. +- [ ] 5. Integrate the `Future` into the UI using `FutureBuilder`. +- [ ] 6. Handle `snapshot.hasData`, `snapshot.hasError`, and default to a `CircularProgressIndicator`. +- [ ] 7. **Feedback Loop:** Run the app -> trigger the network request -> review console for unhandled exceptions -> fix parsing or permission errors. + +## Examples + +### High-Fidelity Implementation: Fetching and Parsing in the Background + +```dart +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; +import 'package:http/http.dart' as http; + +// 1. Top-level parsing function for Isolate +List parsePhotos(String responseBody) { + final parsed = (jsonDecode(responseBody) as List) + .cast>(); + return parsed.map(Photo.fromJson).toList(); +} + +// 2. Network execution with background parsing +Future> fetchPhotos() async { + final response = await http.get( + Uri.parse('https://jsonplaceholder.typicode.com/photos'), + headers: { + HttpHeaders.authorizationHeader: 'Bearer your_token_here', + HttpHeaders.acceptHeader: 'application/json', + }, + ); + + if (response.statusCode == 200) { + // Offload heavy parsing to a background isolate + return compute(parsePhotos, response.body); + } else { + throw Exception('Failed to load photos. Status: ${response.statusCode}'); + } +} + +// 3. Strongly typed model +class Photo { + final int id; + final String title; + final String thumbnailUrl; + + const Photo({ + required this.id, + required this.title, + required this.thumbnailUrl, + }); + + factory Photo.fromJson(Map json) { + return Photo( + id: json['id'] as int, + title: json['title'] as String, + thumbnailUrl: json['thumbnailUrl'] as String, + ); + } +} + +// 4. UI Integration +class PhotoGallery extends StatefulWidget { + const PhotoGallery({super.key}); + + @override + State createState() => _PhotoGalleryState(); +} + +class _PhotoGalleryState extends State { + late Future> _futurePhotos; + + @override + void initState() { + super.initState(); + // Initialize Future once to prevent re-fetching on rebuilds + _futurePhotos = fetchPhotos(); + } + + @override + Widget build(BuildContext context) { + return FutureBuilder>( + future: _futurePhotos, + builder: (context, snapshot) { + if (snapshot.hasData) { + final photos = snapshot.data!; + return ListView.builder( + itemCount: photos.length, + itemBuilder: (context, index) => ListTile( + leading: Image.network(photos[index].thumbnailUrl), + title: Text(photos[index].title), + ), + ); + } else if (snapshot.hasError) { + return Center(child: Text('Error: ${snapshot.error}')); + } + + // Default loading state + return const Center(child: CircularProgressIndicator()); + }, + ); + } +} +``` diff --git a/.claude/skills/shadcn-ui-flutter/SKILL.md b/.claude/skills/shadcn-ui-flutter/SKILL.md new file mode 100644 index 0000000..3a9256a --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/SKILL.md @@ -0,0 +1,165 @@ +--- +name: shadcn-ui-flutter +description: A comprehensive Flutter UI library inspired by shadcn/ui. Provides high-quality, customizable, and accessible components including Buttons, Cards, Forms, and more. Use this skill when building Flutter UIs, implementing design systems, or needing specific component usage examples. +--- + +# Shadcn UI for Flutter + +This skill provides documentation and examples for using the `shadcn_ui` package in Flutter. + +## Theming and Customization +Shadcn UI for Flutter provides a powerful theming system. You can use built-in color schemes (blue, gray, green, neutral, orange, red, rose, slate, stone, violet, yellow, zinc) or create your own. + +### Applying a Theme +Use `ShadThemeData` within `ShadApp` to define your light and dark themes. + +### Detailed Guides +- [Theming](guides/theming.md) +- [Typography](guides/typography.md) +- [Material & Cupertino Interop](guides/interop.md) +- [Responsive](guides/responsive.md) +- [Decorator](guides/decorator.md) + +## Components +| Name | Description | Reference | +| :--- | :--- | :--- | +| Accordion | A vertically stacked set of interactive headings that each reveal a section of content. | [accordion.md](components/accordion.md) | +| Alert | Displays a callout for user attention. | [alert.md](components/alert.md) | +| Avatar | An image element with a placeholder for representing the user. | [avatar.md](components/avatar.md) | +| Badge | Displays a badge or a component that looks like a badge. | [badge.md](components/badge.md) | +| Breadcrumb | Displays the path to the current resource using a hierarchy of links. | [breadcrumb.md](components/breadcrumb.md) | +| Button | Displays a button or a component that looks like a button. | [button.md](components/button.md) | +| Calendar | A date field component that allows users to enter and edit date. | [calendar.md](components/calendar.md) | +| Card | Displays a card with header, content, and footer. | [card.md](components/card.md) | +| Checkbox | A control that allows the user to toggle between checked and not checked. | [checkbox.md](components/checkbox.md) | +| Context Menu | Displays a menu to the user — such as a set of actions or functions — triggered by a mouse right-click. | [context-menu.md](components/context-menu.md) | +| Date Picker | A date picker component with range and presets. | [date-picker.md](components/date-picker.md) | +| Dialog | A modal dialog that interrupts the user. | [dialog.md](components/dialog.md) | +| Form | Builds a form with validation and easy access to form fields values. | [form.md](components/form.md) | +| IconButton | Displays an icon button or a component that looks like a button with an icon. | [icon-button.md](components/icon-button.md) | +| Input | Displays a form input field or a component that looks like an input field. | [input.md](components/input.md) | +| InputOTP | Accessible one-time password component with copy paste functionality. | [input-otp.md](components/input-otp.md) | +| Menubar | A visually persistent menu common in desktop applications that provides quick access to a consistent set of commands. | [menubar.md](components/menubar.md) | +| Popover | Displays rich content in a portal, triggered by a button. | [popover.md](components/popover.md) | +| Progress | Displays an indicator showing the completion progress of a task, typically displayed as a progress bar. | [progress.md](components/progress.md) | +| RadioGroup | A set of checkable buttons—known as radio buttons—where no more than one of the buttons can be checked at a time. | [radio-group.md](components/radio-group.md) | +| Resizable | Resizable panel groups and layouts. | [resizable.md](components/resizable.md) | +| Select | Displays a list of options for the user to pick from—triggered by a button. | [select.md](components/select.md) | +| Separator | Visually or semantically separates content. | [separator.md](components/separator.md) | +| Sheet | Extends the Dialog component to display content that complements the main content of the screen. | [sheet.md](components/sheet.md) | +| Slider | An input where the user selects a value from within a given range. | [slider.md](components/slider.md) | +| Sonner | An opinionated toast component. | [sonner.md](components/sonner.md) | +| Switch | A control that allows the user to toggle between checked and not checked. | [switch.md](components/switch.md) | +| Table | A responsive table component. | [table.md](components/table.md) | +| Tabs | A set of layered sections of content—known as tab panels—that are displayed one at a time. | [tabs.md](components/tabs.md) | +| Textarea | Displays a form textarea or a component that looks like a textarea. | [textarea.md](components/textarea.md) | +| Time Picker | A time picker component. | [time-picker.md](components/time-picker.md) | +| Toast | A succinct message that is displayed temporarily. | [toast.md](components/toast.md) | +| Tooltip | A popup that displays information related to an element when the element receives keyboard focus or the mouse hovers over it. | [tooltip.md](components/tooltip.md) | + +## Usage Examples +Examples are available at the bottom of each component page. + +### Basic Setup +Here is a complete example of a Counter App using `shadcn_ui`, including light and dark theme support. +```dart +import 'package:shadcn_ui/shadcn_ui.dart'; + +void main() { + runApp(const MyApp()); +} + +class MyApp extends StatelessWidget { + const MyApp({super.key}); + + @override + Widget build(BuildContext context) { + return ShadApp( + debugShowCheckedModeBanner: false, + theme: ShadThemeData( + brightness: Brightness.light, + colorScheme: const ShadZincColorScheme.light(), + ), + darkTheme: ShadThemeData( + brightness: Brightness.dark, + colorScheme: const ShadZincColorScheme.dark(), + ), + themeMode: ThemeMode.system, + home: const CounterPage(), + ); + } +} + +class CounterPage extends StatefulWidget { + const CounterPage({super.key}); + + @override + State createState() => _CounterPageState(); +} + +class _CounterPageState extends State { + int _counter = 0; + + void _incrementCounter() { + setState(() { + _counter++; + }); + } + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return Scaffold( + appBar: AppBar(title: const Text('Shadcn Counter')), + body: Center( + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Text( + 'You have pushed the button this many times:', + style: theme.textTheme.muted, + ), + Text( + '$_counter', + style: theme.textTheme.h1, + ), + ], + ), + ), + floatingActionButton: ShadButton( + onPressed: _incrementCounter, + child: const Icon(LucideIcons.plus), + ), + ); + } +} +``` + +## Packages included in the library + +Flutter Shadcn UI consists of fantastic open-source libraries that are exported and you can use them without importing them into your project. + +### [flutter_animate](https://pub.dev/packages/flutter_animate) + +The flutter animate library is a very cool animations library extensively used in Shadcn UI Components. + +With flutter_animate animations can be easily customized from the user, because components will take a `List`. + +### [lucide_icons_flutter](https://pub.dev/packages/lucide_icons_flutter) + +A nice icon library that is used in Shadcn UI Components. +You can use Lucide icons with the `LucideIcons` class, for example `LucideIcons.activity`. + +You can browse all the icons [here](https://lucide.dev/icons/). + +### [two_dimensional_scrollables](https://pub.dev/packages/two_dimensional_scrollables) + +A nice raw table (very performant) implementation used by the [ShadTable](../components/table) component. + +### [intl](https://pub.dev/packages/intl) + +The intl package provides internationalization and localization facilities, including message translation. + +### [universal_image](https://pub.dev/packages/universal_image) + +Support multiple image formats. Used by the [ShadAvatar](../components/avatar) component. diff --git a/.claude/skills/shadcn-ui-flutter/components/accordion.md b/.claude/skills/shadcn-ui-flutter/components/accordion.md new file mode 100644 index 0000000..63e187c --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/accordion.md @@ -0,0 +1,161 @@ +# Accordion + +A vertically stacked set of interactive headings that each reveal a section of content. + + + +```dart +final details = [ + ( + title: 'Is it acceptable?', + content: 'Yes. It adheres to the WAI-ARIA design pattern.', + ), + ( + title: 'Is it styled?', + content: + "Yes. It comes with default styles that matches the other components' aesthetic.", + ), + ( + title: 'Is it animated?', + content: + "Yes. It's animated by default, but you can disable it if you prefer.", + ), +]; + +@override +Widget build(BuildContext context) { + return ShadAccordion<({String content, String title})>( + children: details.map( + (detail) => ShadAccordionItem( + value: detail, + title: Text(detail.title), + child: Text(detail.content), + ), + ), + ); +} +``` + + + +## Multiple + + + +```dart +final details = [ + ( + title: 'Is it acceptable?', + content: 'Yes. It adheres to the WAI-ARIA design pattern.', + ), + ( + title: 'Is it styled?', + content: + "Yes. It comes with default styles that matches the other components' aesthetic.", + ), + ( + title: 'Is it animated?', + content: + "Yes. It's animated by default, but you can disable it if you prefer.", + ), +]; + +@override +Widget build(BuildContext context) { + return ShadAccordion<({String content, String title})>.multiple( + children: details.map( + (detail) => ShadAccordionItem( + value: detail, + title: Text(detail.title), + child: Text(detail.content), + ), + ), + ); +} +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/widgets.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +final details = [ + ( + title: 'Is it acceptable?', + content: 'Yes. It adheres to the WAI-ARIA design pattern.', + ), + ( + title: 'Is it styled?', + content: + "Yes. It comes with default styles that matches the other components' aesthetic.", + ), + ( + title: 'Is it animated?', + content: + "Yes. It's animated by default, but you can disable it if you prefer.", + ), +]; + +class AccordionPage extends StatefulWidget { + const AccordionPage({super.key}); + + @override + State createState() => _AccordionPageState(); +} + +class _AccordionPageState extends State { + var type = ShadAccordionVariant.single; + var underlineTitle = true; + + @override + Widget build(BuildContext context) { + final children = details.map( + (detail) { + return ShadAccordionItem( + value: detail, + title: Text(detail.title), + underlineTitleOnHover: underlineTitle, + child: Text(detail.content), + ); + }, + ); + return BaseScaffold( + appBarTitle: 'Accordion', + editable: [ + MyEnumProperty( + label: 'Type', + value: type, + values: ShadAccordionVariant.values, + onChanged: (value) { + if (value != null) { + setState(() => type = value); + } + }, + ), + MyBoolProperty( + label: 'Underline title', + value: underlineTitle, + onChanged: (v) => setState(() => underlineTitle = v), + ), + ], + children: [ + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: type == ShadAccordionVariant.single + ? ShadAccordion<({String content, String title})>( + children: children, + ) + : ShadAccordion<({String content, String title})>.multiple( + children: children, + ), + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/alert.md b/.claude/skills/shadcn-ui-flutter/components/alert.md new file mode 100644 index 0000000..e471f91 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/alert.md @@ -0,0 +1,69 @@ +# Alert + +Displays a callout for user attention. + + + +```dart +ShadAlert( + icon: Icon(LucideIcons.terminal), + title: Text('Heads up!'), + description: + Text('You can add components to your app using the cli.'), +), +``` + + + +## Destructive + + + +```dart +ShadAlert.destructive( + icon: Icon(LucideIcons.circleAlert), + title: Text('Error'), + description: + Text('Your session has expired. Please log in again.'), +) +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/widgets.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class AlertPage extends StatelessWidget { + const AlertPage({super.key}); + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Alert', + children: [ + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: const ShadAlert( + icon: Icon(LucideIcons.terminal), + title: Text('Heads up!'), + description: Text( + 'You can add components to your app using the cli.', + ), + ), + ), + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: const ShadAlert.destructive( + icon: Icon(LucideIcons.circleAlert), + title: Text('Error'), + description: Text('Your session has expired. Please log in again.'), + ), + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/avatar.md b/.claude/skills/shadcn-ui-flutter/components/avatar.md new file mode 100644 index 0000000..1766014 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/avatar.md @@ -0,0 +1,38 @@ +# Avatar + +An image element with a placeholder for representing the user. + + + +```dart +ShadAvatar( + 'https://app.requestly.io/delay/2000/avatars.githubusercontent.com/u/124599?v=4', + placeholder: Text('CN'), +) +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/widgets.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class AvatarPage extends StatelessWidget { + const AvatarPage({super.key}); + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Avatar', + children: [ + ShadAvatar( + 'https://app.requestly.io/delay/2000/avatars.githubusercontent.com/u/124599?v=4&t=${DateTime.now().millisecondsSinceEpoch}', + placeholder: const Text('CN'), + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/badge.md b/.claude/skills/shadcn-ui-flutter/components/badge.md new file mode 100644 index 0000000..49b507a --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/badge.md @@ -0,0 +1,74 @@ +# Badge + +Displays a badge or a component that looks like a badge. + +## Primary + + + +```dart +ShadBadge( + child: const Text('Primary'), +) +``` + + + +## Secondary + + + +```dart +ShadBadge.secondary( + child: const Text('Secondary'), +) +``` + + + +## Destructive + + + +```dart +ShadBadge.destructive( + child: const Text('Destructive'), +) +``` + + + +## Outline + + + +```dart +ShadBadge.outline( + child: const Text('Outline'), +) +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class BadgePage extends StatelessWidget { + const BadgePage({super.key}); + + @override + Widget build(BuildContext context) { + return const BaseScaffold( + appBarTitle: 'Badge', + children: [ + ShadBadge(child: Text('Primary')), + ShadBadge.secondary(child: Text('Secondary')), + ShadBadge.destructive(child: Text('Destructive')), + ShadBadge.outline(child: Text('Outline')), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/breadcrumb.md b/.claude/skills/shadcn-ui-flutter/components/breadcrumb.md new file mode 100644 index 0000000..d166efc --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/breadcrumb.md @@ -0,0 +1,287 @@ +# Breadcrumb + +Displays the path to the current resource using a hierarchy of links. + + + + +```dart +class PrimaryBreadcrumb extends StatelessWidget { + const PrimaryBreadcrumb({super.key}); + + @override + Widget build(BuildContext context) { + return ShadBreadcrumb( + children: [ + ShadBreadcrumbLink( + onPressed: () => print('Navigating to Home'), + child: const Text('Home'), + ), + ShadBreadcrumbDropdown( + items: [ + ShadBreadcrumbDropMenuItem( + onPressed: () => print('Navigating to Documentation'), + child: const Text('Documentation'), + ), + ShadBreadcrumbDropMenuItem( + onPressed: () => print('Navigating to Themes'), + child: const Text('Themes'), + ), + ShadBreadcrumbDropMenuItem( + onPressed: () => print('Navigating to Github'), + child: const Text('Github'), + ), + ], + showDropdownArrow: false, + child: ShadBreadcrumbEllipsis(), + ), + Text('Components'), + Text('Breadcrumb'), + ], + ); + } +} +``` + + + +## Custom separator + +Use a custom `separator` to change the default `>` separator. + + + +```dart +class CustomSeparatorBreadcrumb extends StatelessWidget { + const CustomSeparatorBreadcrumb({super.key}); + + @override + Widget build(BuildContext context) { + return ShadBreadcrumb( + separator: const Icon(LucideIcons.slash), + children: [ + ShadBreadcrumbLink( + onPressed: () => print('Navigating to Home'), + child: const Text('Home'), + ), + ShadBreadcrumbLink( + onPressed: () => print('Navigating to Components'), + child: const Text('Components'), + ), + Text('Breadcrumb'), + ], + ); + } +} +``` + + + + +## Dropdown + +You can use `ShadBreadcrumbDropdown` to create a dropdown in the breadcrumb. + + + +```dart +class DropdownBreadcrumb extends StatelessWidget { + const DropdownBreadcrumb({super.key}); + + @override + Widget build(BuildContext context) { + return ShadBreadcrumb( + children: [ + ShadBreadcrumbLink( + onPressed: () => print('Navigating to Home'), + child: const Text('Home'), + ), + ShadBreadcrumbDropdown( + items: [ + ShadBreadcrumbDropMenuItem( + onPressed: () => print('Navigating to Documentation'), + child: const Text('Documentation'), + ), + ShadBreadcrumbDropMenuItem( + onPressed: () => print('Navigating to Themes'), + child: const Text('Themes'), + ), + ShadBreadcrumbDropMenuItem( + onPressed: () => print('Navigating to Github'), + child: const Text('Github'), + ), + ], + child: const Text('Components'), + ), + Text('Breadcrumb'), + ], + ); + } +} +``` + +## Example +```dart +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; +import '../common/base_scaffold.dart'; + +class BreadcrumbPage extends StatelessWidget { + const BreadcrumbPage({super.key}); + + void _navigateToHome() { + print('Navigating to Home'); + } + + void _navigateToComponents() { + print('Navigating to Components'); + } + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Breadcrumb', + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + const Text( + 'Simple Breadcrumb', + style: TextStyle(fontSize: 18, fontWeight: FontWeight.bold), + ), + const ShadBreadcrumb( + children: [ + Text('Home'), + Text('Library'), + Text('Data'), + ], + ), + const SizedBox( + height: 20, + ), + const Text( + 'Breadcrumb with Links', + style: TextStyle(fontSize: 18, fontWeight: FontWeight.bold), + ), + ShadBreadcrumb( + children: [ + ShadBreadcrumbLink( + onPressed: _navigateToHome, + child: const Text('Home'), + ), + ShadBreadcrumbLink( + onPressed: _navigateToComponents, + child: const Text('Components'), + ), + const Text('Breadcrumb'), + ], + ), + const SizedBox( + height: 20, + ), + const Text( + 'Breadcrumb with Ellipsis', + style: TextStyle(fontSize: 18, fontWeight: FontWeight.bold), + ), + ShadBreadcrumb( + children: [ + ShadBreadcrumbLink( + onPressed: _navigateToHome, + child: const Text('Home'), + ), + const ShadBreadcrumbEllipsis(), + ShadBreadcrumbLink( + onPressed: _navigateToComponents, + child: const Text('Components'), + ), + const Text('Breadcrumb'), + ], + ), + const SizedBox(height: 20), + const Text( + 'Custom Separator', + style: TextStyle(fontSize: 18, fontWeight: FontWeight.bold), + ), + ShadBreadcrumb( + separator: const Icon(LucideIcons.slash), + children: [ + ShadBreadcrumbLink( + onPressed: _navigateToHome, + child: const Text('Home'), + ), + ShadBreadcrumbLink( + onPressed: _navigateToComponents, + child: const Text('Components'), + ), + const Text('Breadcrumb'), + ], + ), + const SizedBox( + height: 20, + ), + const Text( + 'Breadcrumb with Dropdown', + style: TextStyle(fontSize: 18, fontWeight: FontWeight.bold), + ), + ShadBreadcrumb( + children: [ + ShadBreadcrumbLink( + onPressed: _navigateToHome, + child: const Text('Home'), + ), + ShadBreadcrumbDropdown( + items: [ + ShadBreadcrumbDropMenuItem( + onPressed: () => print('Navigating to Documentation'), + child: const Text('Documentation'), + ), + ShadBreadcrumbDropMenuItem( + onPressed: () => print('Navigating to Themes'), + child: const Text('Themes'), + ), + ShadBreadcrumbDropMenuItem( + onPressed: () => print('Navigating to Github'), + child: const Text('Github'), + ), + ], + child: const Text('Components'), + ), + Text('Breadcrumb'), + ], + ), + const SizedBox( + height: 20, + ), + const Text( + 'Long Breadcrumb', + style: TextStyle(fontSize: 18, fontWeight: FontWeight.bold), + ), + ShadBreadcrumb( + children: [ + ShadBreadcrumbLink( + onPressed: _navigateToHome, + child: const Text('Home'), + ), + ShadBreadcrumbLink( + onPressed: _navigateToComponents, + child: const Text('Component 1'), + ), + ShadBreadcrumbLink( + onPressed: _navigateToComponents, + child: const Text('Component 2'), + ), + ShadBreadcrumbLink( + onPressed: _navigateToComponents, + child: const Text('Component 3'), + ), + ShadBreadcrumbLink( + onPressed: _navigateToComponents, + child: const Text('Component 4'), + ), + Text('Breadcrumb'), + ], + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/button.md b/.claude/skills/shadcn-ui-flutter/components/button.md new file mode 100644 index 0000000..e54753f --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/button.md @@ -0,0 +1,259 @@ +# Button + +Displays a button or a component that looks like a button. + +## Primary + + + +```dart +ShadButton( + child: const Text('Primary'), + onPressed: () {}, +) +``` + + + +## Secondary + + + +```dart +ShadButton.secondary( + child: const Text('Secondary'), + onPressed: () {}, +) +``` + + + +## Destructive + + + +```dart +ShadButton.destructive( + child: const Text('Destructive'), + onPressed: () {}, +) +``` + + + +## Outline + + + +```dart +ShadButton.outline( + child: const Text('Outline'), + onPressed: () {}, +) +``` + + + +## Ghost + + + +```dart +ShadButton.ghost( + child: const Text('Ghost'), + onPressed: () {}, +) +``` + + + +## Link + + + +```dart +ShadButton.link( + child: const Text('Link'), + onPressed: () {}, +) +``` + + + +## Text and Icon + + + +```dart +ShadButton( + onPressed: () {}, + leading: const Icon(LucideIcons.mail), + child: const Text('Login with Email'), +) +``` + + + +## Loading + + + +```dart +ShadButton( + onPressed: () {}, + leading: SizedBox.square( + dimension: 16, + child: CircularProgressIndicator( + strokeWidth: 2, + color: ShadTheme.of(context).colorScheme.primaryForeground, + ), + ), + child: const Text('Please wait'), +) +``` + + + +## Gradient and Shadow + + + +```dart +ShadButton( + onPressed: () {}, + gradient: const LinearGradient(colors: [ + Colors.cyan, + Colors.indigo, + ]), + shadows: [ + BoxShadow( + color: Colors.blue.withOpacity(.4), + spreadRadius: 4, + blurRadius: 10, + offset: const Offset(0, 2), + ), + ], + child: const Text('Gradient with Shadow'), +) +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class ButtonPage extends StatefulWidget { + const ButtonPage({super.key}); + + @override + State createState() => _ButtonPageState(); +} + +class _ButtonPageState extends State { + var size = ShadButtonSize.regular; + var enabled = true; + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return FocusTraversalGroup( + policy: WidgetOrderTraversalPolicy(), + child: BaseScaffold( + appBarTitle: 'Button', + editable: [ + MyEnumProperty( + label: 'Size', + value: size, + values: ShadButtonSize.values, + onChanged: (value) { + if (value != null) { + setState(() => size = value); + } + }, + ), + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + ], + children: [ + ShadButton( + size: size, + enabled: enabled, + child: const Text('Primary'), + onPressed: () => print('Primary'), + ), + ShadButton.secondary( + size: size, + enabled: enabled, + child: const Text('Secondary'), + onPressed: () => print('Secondary'), + ), + ShadButton.destructive( + size: size, + enabled: enabled, + child: const Text('Destructive'), + ), + ShadButton.outline( + size: size, + enabled: enabled, + child: const Text('Outline'), + ), + ShadButton.ghost( + size: size, + enabled: enabled, + child: const Text('Ghost'), + ), + ShadButton.link( + size: size, + enabled: enabled, + child: const Text('Link'), + ), + ShadButton( + size: size, + enabled: enabled, + leading: const Icon(LucideIcons.mail), + child: const Text('Login with Email'), + ), + ShadButton( + size: size, + enabled: enabled, + leading: SizedBox.square( + dimension: 16, + child: CircularProgressIndicator( + strokeWidth: 2, + color: theme.colorScheme.primaryForeground, + ), + ), + child: const Text('Please wait'), + ), + ShadButton( + size: size, + enabled: enabled, + gradient: const LinearGradient( + colors: [ + Colors.cyan, + Colors.indigo, + ], + ), + shadows: [ + BoxShadow( + color: Colors.blue.withValues(alpha: .4), + spreadRadius: 4, + blurRadius: 10, + offset: const Offset(0, 2), + ), + ], + child: const Text('Gradient with Shadow'), + ), + ], + ), + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/calendar.md b/.claude/skills/shadcn-ui-flutter/components/calendar.md new file mode 100644 index 0000000..eb5e6bc --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/calendar.md @@ -0,0 +1,326 @@ +# Calendar + +A date field component that allows users to enter and edit date. + + + +```dart +class SingleCalendar extends StatefulWidget { + const SingleCalendar({super.key}); + + @override + State createState() => _SingleCalendarState(); +} + +class _SingleCalendarState extends State { + final today = DateTime.now(); + + @override + Widget build(BuildContext context) { + return ShadCalendar( + selected: today, + fromMonth: DateTime(today.year - 1), + toMonth: DateTime(today.year, 12), + ); + } +} +``` + + + +## Multiple + + + +```dart +class MultipleCalendar extends StatefulWidget { + const MultipleCalendar({super.key}); + + @override + State createState() => _MultipleCalendarState(); +} + +class _MultipleCalendarState extends State { + final today = DateTime.now(); + + @override + Widget build(BuildContext context) { + return ShadCalendar.multiple( + numberOfMonths: 2, + fromMonth: DateTime(today.year), + toMonth: DateTime(today.year + 1, 12), + min: 5, + max: 10, + ); + } +} +``` + + + +## Range + + + +```dart +class RangeCalendar extends StatelessWidget { + const RangeCalendar({super.key}); + + @override + Widget build(BuildContext context) { + return const ShadCalendar.range( + min: 2, + max: 5, + ); + } +} +``` + + + +#### DropdownMonths + + + +```dart +ShadCalendar( + captionLayout: ShadCalendarCaptionLayout.dropdownMonths, +); +``` + + + +#### DropdownYears + + + +```dart +ShadCalendar( + captionLayout: ShadCalendarCaptionLayout.dropdownYears, +); +``` + + + +### Hide Navigation + + + +```dart +ShadCalendar( + hideNavigation: true, +); +``` + + + +### Show Week Numbers + + + +```dart +ShadCalendar( + showWeekNumbers: true, +); +``` + + + +### Show Outside Days (false) + + + +```dart +ShadCalendar( + showOutsideDays: false, +); +``` + + + +### Fixed Weeks + + + +```dart +ShadCalendar( + fixedWeeks: true, +); +``` + + + +### Hide Weekday Names + + + +```dart +ShadCalendar( + hideWeekdayNames: true, +); +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class CalendarPage extends StatefulWidget { + const CalendarPage({super.key}); + + @override + State createState() => _CalendarPageState(); +} + +class _CalendarPageState extends State { + DateTime? selected = DateTime.now(); + bool reverseMonths = false; + ShadCalendarCaptionLayout captionLayout = ShadCalendarCaptionLayout.label; + bool hideNavigation = false; + bool showWeekNumbers = false; + bool showOutsideDays = true; + bool fixedWeeks = false; + bool hideWeekdayNames = false; + bool allowDeselection = true; + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return BaseScaffold( + appBarTitle: 'Calendar', + editable: [ + MyBoolProperty( + label: 'Reverse months', + value: reverseMonths, + onChanged: (value) { + setState(() { + reverseMonths = value; + }); + }, + ), + MyEnumProperty( + label: 'Caption layout', + value: captionLayout, + values: ShadCalendarCaptionLayout.values, + onChanged: (value) { + if (value != null) { + setState(() { + captionLayout = value; + }); + } + }, + ), + MyBoolProperty( + label: 'Hide navigation', + value: hideNavigation, + onChanged: (value) { + setState(() { + hideNavigation = value; + }); + }, + ), + MyBoolProperty( + label: 'Show week numbers', + value: showWeekNumbers, + onChanged: (value) { + setState(() { + showWeekNumbers = value; + }); + }, + ), + MyBoolProperty( + label: 'Show outside days', + value: showOutsideDays, + enabled: !fixedWeeks, + onChanged: (value) { + setState(() { + showOutsideDays = value; + }); + }, + ), + MyBoolProperty( + label: 'Fixed weeks', + value: fixedWeeks, + enabled: showOutsideDays, + onChanged: (value) { + setState(() { + fixedWeeks = value; + }); + }, + ), + MyBoolProperty( + label: 'Hide weekday names', + value: hideWeekdayNames, + onChanged: (value) { + setState(() { + hideWeekdayNames = value; + }); + }, + ), + MyBoolProperty( + label: 'Allow deselection', + value: allowDeselection, + onChanged: (value) { + setState(() { + allowDeselection = value; + }); + }, + ), + ], + children: [ + Text('Single', style: theme.textTheme.h4), + ShadCalendar( + selected: selected, + fromMonth: DateTime(2023), + toMonth: DateTime(2024, 12), + hideNavigation: hideNavigation, + captionLayout: captionLayout, + onMonthChanged: (date) { + print('month changed to ${date.month}'); + }, + showWeekNumbers: showWeekNumbers, + showOutsideDays: showOutsideDays, + fixedWeeks: fixedWeeks, + hideWeekdayNames: hideWeekdayNames, + allowDeselection: allowDeselection, + ), + const ShadSeparator.horizontal(), + Text('Multiple', style: theme.textTheme.h4), + ShadCalendar.multiple( + numberOfMonths: 2, + fromMonth: DateTime(2024), + toMonth: DateTime(2024, 12), + onChanged: (dates) {}, + min: 5, + max: 10, + reverseMonths: reverseMonths, + hideNavigation: hideNavigation, + captionLayout: captionLayout, + showWeekNumbers: showWeekNumbers, + showOutsideDays: showOutsideDays, + fixedWeeks: fixedWeeks, + hideWeekdayNames: hideWeekdayNames, + ), + const ShadSeparator.horizontal(), + Text('Range', style: theme.textTheme.h4), + ShadCalendar.range( + onChanged: print, + min: 2, + max: 4, + hideNavigation: hideNavigation, + captionLayout: captionLayout, + showWeekNumbers: showWeekNumbers, + showOutsideDays: showOutsideDays, + fixedWeeks: fixedWeeks, + hideWeekdayNames: hideWeekdayNames, + allowDeselection: allowDeselection, + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/card.md b/.claude/skills/shadcn-ui-flutter/components/card.md new file mode 100644 index 0000000..e64fbcf --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/card.md @@ -0,0 +1,421 @@ +# Card + +Displays a card with header, content, and footer. + + + +```dart +const frameworks = { + 'next': 'Next.js', + 'react': 'React', + 'astro': 'Astro', + 'nuxt': 'Nuxt.js', +}; + +class CardProject extends StatelessWidget { + const CardProject({super.key}); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ShadCard( + width: 350, + title: Text('Create project', style: theme.textTheme.h4), + description: const Text('Deploy your new project in one-click.'), + footer: Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + ShadButton.outline( + child: const Text('Cancel'), + onPressed: () {}, + ), + ShadButton( + child: const Text('Deploy'), + onPressed: () {}, + ), + ], + ), + child: Padding( + padding: const EdgeInsets.symmetric(vertical: 16), + child: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + const Text('Name'), + const SizedBox(height: 6), + const ShadInput(placeholder: Text('Name of your project')), + const SizedBox(height: 16), + const Text('Framework'), + const SizedBox(height: 6), + ShadSelect( + placeholder: const Text('Select'), + options: frameworks.entries + .map((e) => ShadOption(value: e.key, child: Text(e.value))) + .toList(), + selectedOptionBuilder: (context, value) { + return Text(frameworks[value]!); + }, + onChanged: (value) {}, + ), + ], + ), + ), + ); + } +} +``` + + + +## Notifications Example + + + +```dart + + + +const notifications = [ + ( + title: "Your call has been confirmed.", + description: "1 hour ago", + ), + ( + title: "You have a new message!", + description: "1 hour ago", + ), + ( + title: "Your subscription is expiring soon!", + description: "2 hours ago", + ), +]; + +class CardNotifications extends StatefulWidget { + const CardNotifications({super.key}); + + @override + State createState() => _CardNotificationsState(); +} + +class _CardNotificationsState extends State { + final pushNotifications = ValueNotifier(false); + + @override + void dispose() { + pushNotifications.dispose(); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ShadCard( + width: 380, + title: const Text('Notifications'), + description: const Text('You have 3 unread messages.'), + footer: ShadButton( + width: double.infinity, + leading: const Padding( + padding: EdgeInsets.only(right: 8), + child: Icon(LucideIcons.check), + ), + onPressed: () {}, + child: const Text('Mark all as read'), + ), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + const SizedBox(height: 16), + Container( + padding: const EdgeInsets.all(16), + decoration: BoxDecoration( + borderRadius: theme.radius, + border: Border.all(color: theme.colorScheme.border), + ), + child: Row( + children: [ + Icon( + LucideIcons.bellRing, + size: 24, + color: theme.colorScheme.foreground, + ), + Expanded( + child: Padding( + padding: const EdgeInsets.only(left: 16), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + 'Push Notifications', + style: theme.textTheme.small, + ), + const SizedBox(height: 4), + Text( + 'Send notifications to device.', + style: theme.textTheme.muted, + ) + ], + ), + ), + ), + ValueListenableBuilder( + valueListenable: pushNotifications, + builder: (context, value, child) { + return ShadSwitch( + value: value, + onChanged: (v) => pushNotifications.value = v, + ); + }, + ), + ], + ), + ), + const SizedBox(height: 16), + ...notifications + .map( + (n) => Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Container( + width: 8, + height: 8, + margin: const EdgeInsets.only(top: 4), + decoration: const BoxDecoration( + color: Color(0xFF0CA5E9), + shape: BoxShape.circle, + ), + ), + Expanded( + child: Padding( + padding: const EdgeInsets.only(left: 16), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('Your call has been confirmed.', + style: theme.textTheme.small), + const SizedBox(height: 4), + Text(n.description, style: theme.textTheme.muted), + ], + ), + ), + ) + ], + ), + ) + .separatedBy(const SizedBox(height: 16)), + const SizedBox(height: 16), + ], + ), + ); + } +} +``` + +## Example +```dart +import 'package:awesome_flutter_extensions/awesome_flutter_extensions.dart'; +import 'package:example/common/base_scaffold.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +const frameworks = { + 'next': 'Next.js', + 'react': 'React', + 'astro': 'Astro', + 'nuxt': 'Nuxt.js', +}; + +const notifications = [ + ( + title: "Your call has been confirmed.", + description: "1 hour ago", + ), + ( + title: "You have a new message!", + description: "1 hour ago", + ), + ( + title: "Your subscription is expiring soon!", + description: "2 hours ago", + ), +]; + +class CardPage extends StatefulWidget { + const CardPage({super.key}); + + @override + State createState() => _CardPageState(); +} + +class _CardPageState extends State { + final pushNotifications = ValueNotifier(false); + + @override + void dispose() { + pushNotifications.dispose(); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return BaseScaffold( + appBarTitle: 'Card', + children: [ + ShadCard( + width: 350, + title: const Text('Create project'), + description: const Text('Deploy your new project in one-click.'), + footer: Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + ShadButton.outline( + child: const Text('Cancel'), + onPressed: () {}, + ), + ShadButton( + child: const Text('Deploy'), + onPressed: () {}, + ), + ], + ), + child: Padding( + padding: const EdgeInsets.symmetric(vertical: 16), + child: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + const Text('Name'), + const SizedBox(height: 6), + const ShadInput(placeholder: Text('Name of your project')), + const SizedBox(height: 16), + const Text('Framework'), + const SizedBox(height: 6), + ShadSelect( + placeholder: const Text('Select'), + options: frameworks.entries + .map( + (e) => ShadOption(value: e.key, child: Text(e.value)), + ) + .toList(), + selectedOptionBuilder: (context, value) { + return Text(frameworks[value]!); + }, + onChanged: (value) {}, + ), + ], + ), + ), + ), + const SizedBox(height: 40), + ShadCard( + width: 380, + title: const Text('Notifications'), + description: const Text('You have 3 unread messages.'), + footer: ShadButton( + width: double.infinity, + leading: const Padding( + padding: EdgeInsetsDirectional.only(end: 8), + child: Icon(LucideIcons.check), + ), + onPressed: () {}, + child: const Text('Mark all as read'), + ), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + const SizedBox(height: 16), + Container( + padding: const EdgeInsets.all(16), + decoration: BoxDecoration( + borderRadius: theme.radius, + border: Border.all(color: theme.colorScheme.border), + ), + child: Row( + children: [ + Icon( + LucideIcons.bellRing, + size: 24, + color: theme.colorScheme.foreground, + ), + Expanded( + child: Padding( + padding: const EdgeInsetsDirectional.only(start: 16), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + 'Push Notifications', + style: theme.textTheme.small, + ), + const SizedBox(height: 4), + Text( + 'Send notifications to device.', + style: theme.textTheme.muted, + ), + ], + ), + ), + ), + ValueListenableBuilder( + valueListenable: pushNotifications, + builder: (context, value, child) { + return ShadSwitch( + value: value, + onChanged: (v) => pushNotifications.value = v, + ); + }, + ), + ], + ), + ), + const SizedBox(height: 16), + ...notifications + .map( + (n) => Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Container( + width: 8, + height: 8, + margin: const EdgeInsets.only(top: 4), + decoration: const BoxDecoration( + color: Color(0xFF0CA5E9), + shape: BoxShape.circle, + ), + ), + Expanded( + child: Padding( + padding: const EdgeInsetsDirectional.only( + start: 16, + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + 'Your call has been confirmed.', + style: theme.textTheme.small, + ), + const SizedBox(height: 4), + Text( + n.description, + style: theme.textTheme.muted, + ), + ], + ), + ), + ), + ], + ), + ) + .separatedBy(const SizedBox(height: 16)), + const SizedBox(height: 16), + ], + ), + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/checkbox.md b/.claude/skills/shadcn-ui-flutter/components/checkbox.md new file mode 100644 index 0000000..2e68630 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/checkbox.md @@ -0,0 +1,244 @@ +# Checkbox + +A control that allows the user to toggle between checked and not checked. + + + +```dart +class CheckboxSample extends StatefulWidget { + const CheckboxSample({super.key}); + + @override + State createState() => _CheckboxSampleState(); +} + +class _CheckboxSampleState extends State { + bool value = false; + + @override + Widget build(BuildContext context) { + return ShadCheckbox( + value: value, + onChanged: (v) => setState(() => value = v), + label: const Text('Accept terms and conditions'), + sublabel: const Text( + 'You agree to our Terms of Service and Privacy Policy.', + ), + ); + } +} +``` + + + +## Form + + + +```dart +ShadCheckboxFormField( + id: 'terms', + initialValue: false, + inputLabel: + const Text('I accept the terms and conditions'), + onChanged: (v) {}, + inputSublabel: + const Text('You agree to our Terms and Conditions'), + validator: (v) { + if (!v) { + return 'You must accept the terms and conditions'; + } + return null; + }, +) +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/widgets.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class CheckboxPage extends StatefulWidget { + const CheckboxPage({super.key}); + + @override + State createState() => _CheckboxPageState(); +} + +class _CheckboxPageState extends State { + bool value = false; + bool enabled = true; + final focusNode = FocusNode(); + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Checkbox', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyBoolProperty( + label: 'Focused', + value: focusNode.hasFocus, + onChanged: enabled + ? (value) { + setState(() { + if (value) { + focusNode.requestFocus(); + } else { + focusNode.unfocus(); + } + }); + } + : null, + ), + ], + children: [ + ShadCheckbox( + value: value, + focusNode: focusNode, + onChanged: (v) => setState(() => value = v), + enabled: enabled, + label: const Text('Accept terms and conditions'), + sublabel: const Text( + 'You agree to our Terms of Service and Privacy Policy.', + ), + ), + ], + ); + } +} + +``` + +## Form Example +```dart +// ignore_for_file: avoid_print + +import 'dart:convert'; + +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class CheckboxFormFieldPage extends StatefulWidget { + const CheckboxFormFieldPage({super.key}); + + @override + State createState() => _CheckboxFormFieldPageState(); +} + +class _CheckboxFormFieldPageState extends State { + bool enabled = true; + var autovalidateMode = ShadAutovalidateMode.alwaysAfterFirstValidation; + bool initialValue = false; + Map formValue = {}; + final formKey = GlobalKey(); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ShadForm( + key: formKey, + enabled: enabled, + autovalidateMode: autovalidateMode, + initialValue: {'terms': initialValue}, + child: BaseScaffold( + appBarTitle: 'CheckboxFormField', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyEnumProperty( + label: 'autovalidateMode', + value: autovalidateMode, + values: ShadAutovalidateMode.values, + onChanged: (value) { + if (value != null) { + setState(() => autovalidateMode = value); + } + }, + ), + MyBoolProperty( + label: 'Form Initial Value', + value: initialValue, + onChanged: (value) { + formKey.currentState!.setFieldValue('terms', value); + setState(() { + initialValue = value; + }); + }, + ), + ], + children: [ + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 350), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + ShadCheckboxFormField( + id: 'terms', + initialValue: initialValue, + inputLabel: const Text('I accept the terms and conditions'), + onChanged: print, + inputSublabel: const Text( + 'You agree to our Terms and Conditions', + ), + validator: (v) { + if (!v) { + return 'You must accept the terms and conditions'; + } + return null; + }, + ), + const SizedBox(height: 16), + ShadButton( + child: const Text('Submit'), + onPressed: () { + print('submitted'); + if (formKey.currentState!.saveAndValidate()) { + setState(() { + formValue = formKey.currentState!.value; + }); + } else { + print('validation failed'); + } + }, + ), + if (formValue.isNotEmpty) + Padding( + padding: const EdgeInsets.only(top: 24, left: 12), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('FormValue', style: theme.textTheme.p), + const SizedBox(height: 4), + SelectableText( + const JsonEncoder.withIndent( + ' ', + ).convert(formValue), + style: theme.textTheme.small, + ), + ], + ), + ), + ], + ), + ), + ], + ), + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/context-menu.md b/.claude/skills/shadcn-ui-flutter/components/context-menu.md new file mode 100644 index 0000000..9186b54 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/context-menu.md @@ -0,0 +1,195 @@ +# Context Menu + +Displays a menu to the user — such as a set of actions or functions — triggered by a mouse right-click. + + + +```dart + + + +class ContextMenuPage extends StatelessWidget { + const ContextMenuPage({super.key}); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return Scaffold( + body: Padding( + padding: const EdgeInsets.all(16), + child: ShadContextMenuRegion( + constraints: const BoxConstraints(minWidth: 300), + items: [ + const ShadContextMenuItem.inset( + child: Text('Back'), + ), + const ShadContextMenuItem.inset( + enabled: false, + child: Text('Forward'), + ), + const ShadContextMenuItem.inset( + child: Text('Reload'), + ), + const ShadContextMenuItem.inset( + trailing: Icon(LucideIcons.chevronRight), + items: [ + ShadContextMenuItem( + child: Text('Save Page As...'), + ), + ShadContextMenuItem( + child: Text('Create Shortcut...'), + ), + ShadContextMenuItem( + child: Text('Name Window...'), + ), + Divider(height: 8), + ShadContextMenuItem( + child: Text('Developer Tools'), + ), + ], + child: Text('More Tools'), + ), + const Divider(height: 8), + const ShadContextMenuItem( + leading: Icon(LucideIcons.check), + child: Text('Show Bookmarks Bar'), + ), + const ShadContextMenuItem.inset(child: Text('Show Full URLs')), + const Divider(height: 8), + Padding( + padding: const EdgeInsets.fromLTRB(36, 8, 8, 8), + child: Text('People', style: theme.textTheme.small), + ), + const Divider(height: 8), + ShadContextMenuItem( + leading: SizedBox.square( + dimension: 16, + child: Center( + child: Container( + width: 8, + height: 8, + decoration: BoxDecoration( + color: theme.colorScheme.foreground, + shape: BoxShape.circle, + ), + ), + ), + ), + child: const Text('Pedro Duarte'), + ), + const ShadContextMenuItem.inset(child: Text('Colm Tuite')), + ], + child: Container( + width: 300, + height: 200, + alignment: Alignment.center, + decoration: BoxDecoration( + border: Border.all(color: theme.colorScheme.border), + borderRadius: BorderRadius.circular(8), + ), + child: const Text('Right click here'), + ), + ), + ), + ); + } +} +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class ContextMenuPage extends StatelessWidget { + const ContextMenuPage({super.key}); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + const divider = ShadSeparator.horizontal( + margin: EdgeInsets.symmetric(vertical: 4), + ); + return BaseScaffold( + appBarTitle: 'ContextMenu', + children: [ + ShadContextMenuRegion( + constraints: const BoxConstraints(minWidth: 300), + items: [ + const ShadContextMenuItem.inset( + child: Text('Back'), + ), + const ShadContextMenuItem.inset( + enabled: false, + child: Text('Forward'), + ), + const ShadContextMenuItem.inset( + child: Text('Reload'), + ), + const ShadContextMenuItem.inset( + trailing: Icon(LucideIcons.chevronRight), + items: [ + ShadContextMenuItem( + child: Text('Save Page As...'), + ), + ShadContextMenuItem( + child: Text('Create Shortcut...'), + ), + ShadContextMenuItem( + child: Text('Name Window...'), + ), + divider, + ShadContextMenuItem( + child: Text('Developer Tools'), + ), + ], + child: Text('More Tools'), + ), + divider, + const ShadContextMenuItem( + leading: Icon(LucideIcons.check), + child: Text('Show Bookmarks Bar'), + ), + const ShadContextMenuItem.inset(child: Text('Show Full URLs')), + divider, + Padding( + padding: const EdgeInsets.fromLTRB(36, 8, 8, 8), + child: Text('People', style: theme.textTheme.small), + ), + divider, + ShadContextMenuItem( + leading: SizedBox.square( + dimension: 16, + child: Center( + child: Container( + width: 8, + height: 8, + decoration: BoxDecoration( + color: theme.colorScheme.foreground, + shape: BoxShape.circle, + ), + ), + ), + ), + child: const Text('Pedro Duarte'), + ), + const ShadContextMenuItem.inset(child: Text('Colm Tuite')), + ], + child: Container( + width: 300, + height: 200, + alignment: Alignment.center, + decoration: BoxDecoration( + border: Border.all(color: theme.colorScheme.border), + borderRadius: BorderRadius.circular(8), + ), + child: const Text('Right click here'), + ), + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/date-picker.md b/.claude/skills/shadcn-ui-flutter/components/date-picker.md new file mode 100644 index 0000000..83edf59 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/date-picker.md @@ -0,0 +1,383 @@ +# Date Picker + +A date picker component with range and presets. + + + +```dart +class SingleDatePicker extends StatelessWidget { + const SingleDatePicker({super.key}); + + @override + Widget build(BuildContext context) { + return ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: const ShadDatePicker(), + ); + } +} +``` + + + +## Date Range Picker + + + +```dart +class RangeDatePicker extends StatelessWidget { + const RangeDatePicker({super.key}); + + @override + Widget build(BuildContext context) { + return ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: const ShadDatePicker.range(), + ); + } +} +``` + + + +## With Presets + + + +```dart +const presets = { + 0: 'Today', + 1: 'Tomorrow', + 3: 'In 3 days', + 7: 'In a week', +}; + +class PresetsDatePicker extends StatefulWidget { + const PresetsDatePicker({super.key}); + + @override + State createState() => _PresetsDatePickerState(); +} + +class _PresetsDatePickerState extends State { + final groupId = UniqueKey(); + final today = DateTime.now().startOfDay; + DateTime? selected; + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: ShadDatePicker( + // Using the same groupId to keep the date picker popover open when the + // select popover is closed. + groupId: groupId, + header: Padding( + padding: const EdgeInsets.only(bottom: 4), + child: ShadSelect( + groupId: groupId, + minWidth: 276, + placeholder: const Text('Select'), + options: presets.entries + .map((e) => ShadOption(value: e.key, child: Text(e.value))) + .toList(), + selectedOptionBuilder: (context, value) { + return Text(presets[value]!); + }, + onChanged: (value) { + if (value == null) return; + setState(() { + selected = today.add(Duration(days: value)); + }); + }, + ), + ), + selected: selected, + calendarDecoration: theme.calendarTheme.decoration, + popoverPadding: const EdgeInsets.all(4), + ), + ); + } +} +``` + + + +## Form + + + +```dart +ShadDatePickerFormField( + label: const Text('Date of birth'), + onChanged: print, + description: const Text( + 'Your date of birth is used to calculate your age.'), + validator: (v) { + if (v == null) { + return 'A date of birth is required.'; + } + return null; + }, +), +``` + + + +## DateRangePickerFormField + + + +```dart +ShadDateRangePickerFormField( + label: const Text('Range of dates'), + onChanged: print, + description: const Text( + 'Select the range of dates you want to search between.'), + validator: (v) { + if (v == null) return 'A range of dates is required.'; + if (v.start == null) { + return 'The start date is required.'; + } + if (v.end == null) return 'The end date is required.'; + + return null; + }, +), + +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +const presets = { + 0: 'Today', + 1: 'Tomorrow', + 3: 'In 3 days', + 7: 'In a week', +}; + +class DatePickerPage extends StatefulWidget { + const DatePickerPage({super.key}); + + @override + State createState() => _DatePickerPageState(); +} + +class _DatePickerPageState extends State { + bool closeOnSelection = false; + bool allowDeselection = true; + final today = DateTime.now().startOfDay; + final groupId = UniqueKey(); + + DateTime? selected; + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return BaseScaffold( + appBarTitle: 'DatePicker', + editable: [ + MyBoolProperty( + label: 'closeOnSelection', + value: closeOnSelection, + onChanged: (value) => setState(() => closeOnSelection = value), + ), + MyBoolProperty( + label: 'allowDeselection', + value: allowDeselection, + onChanged: (value) => setState(() => allowDeselection = value), + ), + ], + children: [ + Text('Single', style: theme.textTheme.h4), + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: ShadDatePicker( + closeOnSelection: closeOnSelection, + allowDeselection: allowDeselection, + ), + ), + const ShadSeparator.horizontal(), + Text('Range', style: theme.textTheme.h4), + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: ShadDatePicker.range( + closeOnSelection: closeOnSelection, + allowDeselection: allowDeselection, + ), + ), + const ShadSeparator.horizontal(), + Text('With Presets', style: theme.textTheme.h4), + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: ShadDatePicker( + // Using the same groupId to keep the date picker popover open when the + // select popover is closed. + groupId: groupId, + header: Padding( + padding: const EdgeInsets.only(bottom: 4), + child: ShadSelect( + groupId: groupId, + minWidth: 276, + placeholder: const Text('Select'), + options: presets.entries + .map((e) => ShadOption(value: e.key, child: Text(e.value))) + .toList(), + selectedOptionBuilder: (context, value) { + return Text(presets[value]!); + }, + onChanged: (value) { + if (value == null) return; + setState(() { + selected = today.add(Duration(days: value)); + }); + }, + ), + ), + closeOnSelection: closeOnSelection, + allowDeselection: allowDeselection, + selected: selected, + calendarDecoration: theme.calendarTheme.decoration, + popoverPadding: const EdgeInsets.all(4), + ), + ), + ], + ); + } +} + +``` + +## Form Example +```dart +// ignore_for_file: avoid_print + +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class DatePickerFormFieldPage extends StatefulWidget { + const DatePickerFormFieldPage({super.key}); + + @override + State createState() => + _DatePickerFormFieldPageState(); +} + +class _DatePickerFormFieldPageState extends State { + bool enabled = true; + var autovalidateMode = ShadAutovalidateMode.alwaysAfterFirstValidation; + Map formValue = {}; + final formKey = GlobalKey(); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ShadForm( + key: formKey, + enabled: enabled, + autovalidateMode: autovalidateMode, + child: BaseScaffold( + appBarTitle: 'DatePickerFormField', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyEnumProperty( + label: 'autovalidateMode', + value: autovalidateMode, + values: ShadAutovalidateMode.values, + onChanged: (value) { + if (value != null) { + setState(() => autovalidateMode = value); + } + }, + ), + ], + children: [ + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 350), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('Single', style: theme.textTheme.h4), + ShadDatePickerFormField( + id: 'date', + label: const Text('Date of birth'), + onChanged: print, + description: const Text( + 'Your date of birth is used to calculate your age.', + ), + validator: (v) { + if (v == null) { + return 'A date of birth is required.'; + } + return null; + }, + ), + const ShadSeparator.horizontal(), + Text('Range', style: theme.textTheme.h4), + ShadDateRangePickerFormField( + id: 'range-date', + label: const Text('Range of dates'), + onChanged: print, + description: const Text( + 'Select the range of dates you want to search between.', + ), + validator: (v) { + if (v == null) return 'A range of dates is required.'; + if (v.start == null) return 'The start date is required.'; + if (v.end == null) return 'The end date is required.'; + + return null; + }, + ), + const SizedBox(height: 16), + ShadButton( + child: const Text('Submit'), + onPressed: () { + print('submitted'); + if (formKey.currentState!.saveAndValidate()) { + setState(() { + formValue = formKey.currentState!.value; + }); + } else { + print('validation failed'); + } + }, + ), + if (formValue.isNotEmpty) + Padding( + padding: const EdgeInsets.only(top: 24, left: 12), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('FormValue', style: theme.textTheme.p), + const SizedBox(height: 4), + SelectableText( + formValue.toString(), + style: theme.textTheme.small, + ), + ], + ), + ), + ], + ), + ), + ], + ), + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/dialog.md b/.claude/skills/shadcn-ui-flutter/components/dialog.md new file mode 100644 index 0000000..229bb51 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/dialog.md @@ -0,0 +1,240 @@ +# Dialog + +A modal dialog that interrupts the user. + + + +```dart + + +final profile = [ + (title: 'Name', value: 'Alexandru'), + (title: 'Username', value: 'nank1ro'), +]; + +class DialogExample extends StatelessWidget { + const DialogExample({super.key}); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ShadButton.outline( + child: const Text('Edit Profile'), + onPressed: () { + showShadDialog( + context: context, + builder: (context) => ShadDialog( + title: const Text('Edit Profile'), + description: const Text( + "Make changes to your profile here. Click save when you're done"), + actions: const [ShadButton(child: Text('Save changes'))], + child: Container( + width: 375, + padding: const EdgeInsets.symmetric(vertical: 20), + child: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.end, + spacing: 16, + children: profile + .map( + (p) => Row( + children: [ + Expanded( + child: Text( + p.title, + textAlign: TextAlign.end, + style: theme.textTheme.small, + ), + ), + const SizedBox(width: 16), + Expanded( + flex: 3, + child: ShadInput(initialValue: p.value), + ), + ], + ), + ).toList(), + ), + ), + ), + ); + }, + ); + } +} +``` + + + +## Alert + + + +```dart +class DialogExample extends StatelessWidget { + const DialogExample({super.key}); + + @override + Widget build(BuildContext context) { + return ShadButton.outline( + child: const Text('Show Dialog'), + onPressed: () { + showShadDialog( + context: context, + builder: (context) => ShadDialog.alert( + title: const Text('Are you absolutely sure?'), + description: const Padding( + padding: EdgeInsets.only(bottom: 8), + child: Text( + 'This action cannot be undone. This will permanently delete your account and remove your data from our servers.', + ), + ), + actions: [ + ShadButton.outline( + child: const Text('Cancel'), + onPressed: () => Navigator.of(context).pop(false), + ), + ShadButton( + child: const Text('Continue'), + onPressed: () => Navigator.of(context).pop(true), + ), + ], + ), + ); + }, + ); + } +} +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +final profile = [ + (title: 'Name', value: 'Alexandru'), + (title: 'Username', value: 'nank1ro'), +]; + +class DialogPage extends StatefulWidget { + const DialogPage({super.key}); + + @override + State createState() => _DialogPageState(); +} + +class _DialogPageState extends State { + var titlePinned = false; + var descriptionPinned = false; + var actionsPinned = true; + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return BaseScaffold( + appBarTitle: 'Dialog', + editable: [ + MyBoolProperty( + label: 'titlePinned', + value: titlePinned, + onChanged: (v) => setState(() => titlePinned = v), + ), + MyBoolProperty( + label: 'descriptionPinned', + value: descriptionPinned, + onChanged: (v) => setState(() => descriptionPinned = v), + ), + MyBoolProperty( + label: 'actionsPinned', + value: actionsPinned, + onChanged: (v) => setState(() => actionsPinned = v), + ), + ], + children: [ + ShadButton.outline( + child: const Text('Edit Profile'), + onPressed: () { + showShadDialog( + context: context, + builder: (context) => ShadDialog( + title: const Text('Edit Profile'), + description: const Text( + "Make changes to your profile here. Click save when you're done", + ), + actions: const [ShadButton(child: Text('Save changes'))], + titlePinned: titlePinned, + descriptionPinned: descriptionPinned, + actionsPinned: actionsPinned, + crossAxisAlignment: CrossAxisAlignment.stretch, + child: Container( + width: 375, + padding: const EdgeInsets.symmetric(vertical: 20), + child: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.end, + spacing: 16, + children: profile + .map( + (p) => Row( + children: [ + Expanded( + child: Text( + p.title, + textAlign: TextAlign.end, + style: theme.textTheme.small, + ), + ), + const SizedBox(width: 16), + Expanded( + flex: 3, + child: ShadInput(initialValue: p.value), + ), + ], + ), + ) + .toList(), + ), + ), + ), + ); + }, + ), + ShadButton.outline( + child: const Text('Show Dialog'), + onPressed: () { + showShadDialog( + context: context, + builder: (context) => ShadDialog.alert( + title: const Text('Are you absolutely sure?'), + titlePinned: titlePinned, + descriptionPinned: descriptionPinned, + actionsPinned: actionsPinned, + description: const Padding( + padding: EdgeInsets.only(bottom: 8), + child: Text( + 'This action cannot be undone. This will permanently delete your account and remove your data from our servers.', + ), + ), + actions: [ + ShadButton.outline( + child: const Text('Cancel'), + onPressed: () => Navigator.of(context).pop(false), + ), + ShadButton( + child: const Text('Continue'), + onPressed: () => Navigator.of(context).pop(true), + ), + ], + ), + ); + }, + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/form.md b/.claude/skills/shadcn-ui-flutter/components/form.md new file mode 100644 index 0000000..65c1331 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/form.md @@ -0,0 +1,292 @@ +# Form + +Builds a form with validation and easy access to form fields values. + +The benefits of using `ShadForm` over managing form fields individually are: +- Centralized form state management. +- Easy access to all form field values as a single `Map`. +- No need to manage individual controllers for each form field. + + + +```dart +class FormPage extends StatefulWidget { + const FormPage({ + super.key, + }); + + @override + State createState() => _FormPageState(); +} + +class _FormPageState extends State { + final formKey = GlobalKey(); + + @override + Widget build(BuildContext context) { + return Scaffold( + body: Center( + child: ShadForm( + key: formKey, + child: ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 350), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + mainAxisSize: MainAxisSize.min, + children: [ + ShadInputFormField( + id: 'username', + label: const Text('Username'), + placeholder: const Text('Enter your username'), + description: const Text('This is your public display name.'), + validator: (v) { + if (v.length < 2) { + return 'Username must be at least 2 characters.'; + } + return null; + }, + ), + const SizedBox(height: 16), + ShadButton( + child: const Text('Submit'), + onPressed: () { + if (formKey.currentState!.saveAndValidate()) { + print( + 'validation succeeded with ${formKey.currentState!.value}'); + } else { + print('validation failed'); + } + }, + ), + ], + ), + ), + ), + ), + ); + } +} +``` + + + +## Initial form value + +You can set the initial form value by passing a `Map` to the `initialValue` property of the `ShadForm` widget. + +```dart {2-5} +ShadForm( + initialValue: { + 'username': 'john_doe', + 'email': 'john_doe@example.com' + }, + child: // Your form fields here +) +``` + +All form fields with matching `id`s will be initialized with the corresponding values from the `initialValue` map. +Unless they have their own `initialValue` set. + +## Get the form value + +You can get the current form value by accessing the `value` property of `ShadFormState` using a `GlobalKey`. + +```dart {8-10} +final formKey = GlobalKey(); + +// Your Form widget +ShadForm( + key: formKey, +), + +// To get the form value +final formValue = formKey.currentState!.value; // Returns a Map with the form field values +``` + +You typically need this after getting a successful value from the `saveAndValidate` method, for example: +```dart {4-8} +ShadButton( + child: const Text('Submit'), + onPressed: () { + final formState = formKey.currentState!; + // The form is not valid, return early + if (!formState.saveAndValidate()) return; + // The form is valid, print the form value + print('Form value: ${formState.value}'); + }, +), +``` + +## Manipulate single form field value + +You can set or update the value of specific form fields using the `setFieldValue` method of `ShadFormState`. + +```dart {8-10} +final formKey = GlobalKey(); + +// Your Form widget +ShadForm( + key: formKey, +), + +// To set or update a specific field value +formKey.currentState!.setFieldValue('username', 'new_username'); +``` + +If you don't want to notify the field about the value change, you can pass `notifyField: false` as argument. +This would only update the form value without updating the field UI. + +## Manipulate entire form value + +You can set or update the entire form value using the `setValue` method of `ShadFormState`. + +```dart {8-12} +final formKey = GlobalKey(); + +// Your Form widget +ShadForm( + key: formKey, +), + +// To set or update the entire form value +formKey.currentState!.setValue({ + 'username': 'new_username', + 'email': 'example@email.com' +}); +``` + +If you don't want to notify the fields about the value change, you can pass `notifyFields: false` as argument. +This would only update the form value without updating the fields UI. + +## Value transformers + +You can use value transformers to convert the initial value from the form to the field value and vice versa. + +### fromValueTransformer + +If your `ShadForm` has an initial value like this one `{'date': '2024-02-01'}` and you need to convert the string value to a `DateTime` object for a `ShadDatePickerFormField`: +```dart {3} +ShadDatePickerFormField( + id: 'date', + fromValueTransformer: (value) => DateTime.tryParse(value ?? ''), +), +``` + +Vice versa, you can use the `toValueTransformer` parameter to convert the field value back to the form value. +```dart {3-5} +ShadDatePickerFormField( + id: 'date', + toValueTransformer: (date) => date == null + ? null + : DateFormat('yyyy-MM-dd').format(date), +), +``` + +In this way, the form field can work with `DateTime` objects while the form value remains a `String` both as initial value (input) and when getting the form value (output). + +## Dot notation for nested values + +By default, `ShadForm` supports dot notation in field IDs to automatically create nested map structures. This makes it easier to work with complex, hierarchical form data. + +### How it works + +When you use field IDs with dots (like `user.email` or `profile.settings.theme`), the form automatically converts them into nested maps: + +```dart +ShadForm( + child: Column( + children: [ + ShadInputFormField( + id: 'user.name', + label: const Text('Name'), + ), + ShadInputFormField( + id: 'user.email', + label: const Text('Email'), + ), + ShadInputFormField( + id: 'user.age', + label: const Text('Age'), + ), + ], + ), +) +``` + +When you retrieve the form value, it will be structured as: +```dart +{ + 'user': { + 'name': 'John Doe', + 'email': 'john@example.com', + 'age': '30' + } +} +``` + +### Initial values with nested structure + +The `initialValue` should be provided as a nested map structure (not using dot notation): + +```dart +ShadForm( + initialValue: { + 'user': { + 'name': 'John Doe', + 'email': 'john@example.com', + }, + }, + child: // Your form fields with dot notation IDs +) +``` + +The form will automatically extract values from the nested structure based on the field IDs. For example, a field with `id: 'user.name'` will get the value from `initialValue['user']['name']`. + +### Customizing the separator + +If you prefer a different separator, you can customize it using the `fieldIdSeparator` parameter: + +```dart {2} +ShadForm( + fieldIdSeparator: '/', + child: Column( + children: [ + ShadInputFormField( + id: 'user/name', // Using '/' instead of '.' + label: const Text('Name'), + ), + ], + ), +) +``` + +### Disabling dot notation + +If you want to use dots in your field IDs without creating nested structures, you can disable the feature by setting `fieldIdSeparator` to `null`: + +```dart {2} +ShadForm( + fieldIdSeparator: null, + child: Column( + children: [ + ShadInputFormField( + id: 'user.email', // This will remain as a flat key + label: const Text('Email'), + ), + ], + ), +) +``` + +## Examples + +See the following links for more examples on how to use the `ShadForm` component with other components: + +- [Checkbox](../checkbox#form) +- [Switch](../switch#form) +- [Input](../input#form) +- [Select](../select#form) +- [RadioGroup](../radio-group#form) +- [DatePicker](../date-picker#form) +- [TimePicker](../time-picker#form) + diff --git a/.claude/skills/shadcn-ui-flutter/components/icon-button.md b/.claude/skills/shadcn-ui-flutter/components/icon-button.md new file mode 100644 index 0000000..37ec0aa --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/icon-button.md @@ -0,0 +1,201 @@ +# IconButton + +Displays an icon button or a component that looks like a button with an icon. + +## Primary + + + +```dart +ShadIconButton( + onPressed: () => print('Primary'), + icon: const Icon(LucideIcons.rocket), +) +``` + + + +## Secondary + + + +```dart +ShadIconButton.secondary( + icon: const Icon(LucideIcons.rocket), + onPressed: () => print('Secondary'), +) +``` + + + +## Destructive + + + +```dart +ShadIconButton.destructive( + icon: const Icon(LucideIcons.rocket), + onPressed: () => print('Destructive'), +) +``` + + + +## Outline + + + +```dart +ShadIconButton.outline( + icon: const Icon(LucideIcons.rocket), + onPressed: () => print('Outline'), +) +``` + + + +## Ghost + + + +```dart +ShadIconButton.ghost( + icon: const Icon(LucideIcons.rocket), + onPressed: () => print('Ghost'), +) +``` + + + +## Loading + + + +```dart +ShadIconButton( + icon: SizedBox.square( + dimension: 16, + child: CircularProgressIndicator( + strokeWidth: 2, + color: ShadTheme.of(context).colorScheme.primaryForeground, + ), + ), +) +``` + + + +## Gradient and Shadow + + + +```dart +ShadIconButton( + gradient: const LinearGradient(colors: [ + Colors.cyan, + Colors.indigo, + ]), + shadows: [ + BoxShadow( + color: Colors.blue.withValues(alpha: .4), + spreadRadius: 4, + blurRadius: 10, + offset: const Offset(0, 2), + ), + ], + icon: const Icon(LucideIcons.rocket), +) +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class IconButtonPage extends StatefulWidget { + const IconButtonPage({super.key}); + + @override + State createState() => _IconButtonPageState(); +} + +class _IconButtonPageState extends State { + var enabled = true; + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return FocusTraversalGroup( + policy: WidgetOrderTraversalPolicy(), + child: BaseScaffold( + appBarTitle: 'IconButton', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + ], + children: [ + ShadIconButton( + enabled: enabled, + onPressed: () => print('Primary'), + icon: const Icon(LucideIcons.rocket), + ), + ShadIconButton.secondary( + enabled: enabled, + icon: const Icon(LucideIcons.rocket), + onPressed: () => print('Secondary'), + ), + ShadIconButton.destructive( + enabled: enabled, + icon: const Icon(LucideIcons.rocket), + onPressed: () => print('Destructive'), + ), + ShadIconButton.outline( + enabled: enabled, + icon: const Icon(LucideIcons.rocket), + onPressed: () => print('Outline'), + ), + ShadIconButton.ghost( + enabled: enabled, + icon: const Icon(LucideIcons.rocket), + onPressed: () => print('Ghost'), + ), + ShadIconButton( + enabled: enabled, + gradient: const LinearGradient( + colors: [ + Colors.cyan, + Colors.indigo, + ], + ), + shadows: [ + BoxShadow( + color: Colors.blue.withValues(alpha: .4), + spreadRadius: 4, + blurRadius: 10, + offset: const Offset(0, 2), + ), + ], + icon: const Icon(LucideIcons.rocket), + ), + ShadIconButton( + enabled: enabled, + icon: SizedBox.square( + dimension: 16, + child: CircularProgressIndicator( + strokeWidth: 2, + color: theme.colorScheme.primaryForeground, + ), + ), + ), + ], + ), + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/input-otp.md b/.claude/skills/shadcn-ui-flutter/components/input-otp.md new file mode 100644 index 0000000..a914baf --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/input-otp.md @@ -0,0 +1,334 @@ +# InputOTP + +Accessible one-time password component with copy paste functionality. + + + +```dart +ShadInputOTP( + onChanged: (v) => print('OTP: $v'), + maxLength: 6, + children: const [ + ShadInputOTPGroup( + children: [ + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ], + ), + Icon(size: 24, LucideIcons.dot), + ShadInputOTPGroup( + children: [ + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ], + ), + ], +) +``` + + + +## InputFormatters + +Using InputFormatters you can restrict the input characters. +The example below shows how to restrict the input to only numbers. + + + +```dart +ShadInputOTP( + onChanged: (v) => print('OTP: $v'), + maxLength: 4, + keyboardType: TextInputType.number, + inputFormatters: [ + FilteringTextInputFormatter.digitsOnly, + ], + children: const [ + ShadInputOTPGroup( + children: [ + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ], + ), + ], +) +``` + + + +See also `UpperCaseTextInputFormatter` and `LowerCaseTextInputFormatter` which are provided by the package. + +## Form + + + +```dart +ShadInputOTPFormField( + id: 'otp', + maxLength: 6, + label: const Text('OTP'), + description: const Text('Enter your OTP.'), + validator: (v) { + if (v.contains(' ')) { + return 'Fill the whole OTP code'; + } + return null; + }, + children: const [ + ShadInputOTPGroup( + children: [ + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ], + ), + Icon(size: 24, LucideIcons.dot), + ShadInputOTPGroup( + children: [ + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ], + ), + ], +) +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter/widgets.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class InputOTPPage extends StatefulWidget { + const InputOTPPage({super.key}); + + @override + State createState() => _InputOTPPageState(); +} + +class _InputOTPPageState extends State { + var enabled = true; + var uppercase = true; + var digitsOnly = false; + var jumpToNextWhenFilled = true; + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Input OTP', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyBoolProperty( + label: 'Uppercase', + value: uppercase, + enabled: !digitsOnly, + onChanged: (value) => setState(() => uppercase = value), + ), + MyBoolProperty( + label: 'Digits only', + value: digitsOnly, + enabled: !uppercase, + onChanged: (value) => setState(() => digitsOnly = value), + ), + MyBoolProperty( + label: 'Jump to next when filled', + value: jumpToNextWhenFilled, + onChanged: (value) => setState(() => jumpToNextWhenFilled = value), + ), + ], + children: [ + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: ShadInputOTP( + onChanged: (v) => print('OTP: $v'), + maxLength: 6, + enabled: enabled, + jumpToNextWhenFilled: jumpToNextWhenFilled, + keyboardType: digitsOnly ? TextInputType.number : null, + inputFormatters: [ + if (digitsOnly) FilteringTextInputFormatter.digitsOnly, + if (uppercase) const UpperCaseTextInputFormatter(), + ], + children: const [ + ShadInputOTPGroup( + children: [ + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ], + ), + Icon(LucideIcons.dot), + ShadInputOTPGroup( + children: [ + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ], + ), + ], + ), + ), + ], + ); + } +} + +``` + +## Form Example +```dart +// ignore_for_file: avoid_print + +import 'dart:convert'; + +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:example/common/properties/string_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class InputOTPFormFieldPage extends StatefulWidget { + const InputOTPFormFieldPage({super.key}); + + @override + State createState() => _InputOTPFormFieldPageState(); +} + +class _InputOTPFormFieldPageState extends State { + bool enabled = true; + var autovalidateMode = ShadAutovalidateMode.alwaysAfterFirstValidation; + String? initialValue; + Map formValue = {}; + final formKey = GlobalKey(); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ShadForm( + key: formKey, + enabled: enabled, + autovalidateMode: autovalidateMode, + initialValue: {if (initialValue != null) 'otp': initialValue}, + child: BaseScaffold( + appBarTitle: 'InputFormField', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyEnumProperty( + label: 'autovalidateMode', + value: autovalidateMode, + values: ShadAutovalidateMode.values, + onChanged: (value) { + if (value != null) { + setState(() => autovalidateMode = value); + } + }, + ), + MyStringProperty( + label: 'Form Initial Value', + initialValue: initialValue, + placeholder: const Text('OTP initial value'), + onChanged: (value) { + setState(() { + value.isEmpty ? initialValue = null : initialValue = value; + }); + // Reset the form + WidgetsBinding.instance.addPostFrameCallback((timeStamp) { + formKey.currentState!.reset(); + }); + }, + ), + ], + children: [ + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 350), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + ShadInputOTPFormField( + id: 'otp', + maxLength: 6, + enabled: enabled, + label: const Text('OTP'), + description: const Text('Enter your OTP.'), + validator: (v) { + if (v.contains(' ')) { + return 'Fill the whole OTP code'; + } + return null; + }, + children: const [ + ShadInputOTPGroup( + children: [ + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ], + ), + Icon(LucideIcons.dot), + ShadInputOTPGroup( + children: [ + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ShadInputOTPSlot(), + ], + ), + ], + ), + const SizedBox(height: 16), + ShadButton( + child: const Text('Submit'), + onPressed: () { + print('submitted'); + if (formKey.currentState!.saveAndValidate()) { + setState(() { + formValue = formKey.currentState!.value; + }); + } else { + print('validation failed'); + } + }, + ), + if (formValue.isNotEmpty) + Padding( + padding: const EdgeInsets.only(top: 24, left: 12), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('FormValue', style: theme.textTheme.p), + const SizedBox(height: 4), + SelectableText( + const JsonEncoder.withIndent( + ' ', + ).convert(formValue), + style: theme.textTheme.small, + ), + ], + ), + ), + ], + ), + ), + ], + ), + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/input.md b/.claude/skills/shadcn-ui-flutter/components/input.md new file mode 100644 index 0000000..a110770 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/input.md @@ -0,0 +1,283 @@ +# Input + +Displays a form input field or a component that looks like an input field. + + + +```dart +ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 320), + child: const ShadInput( + placeholder: Text('Email'), + keyboardType: TextInputType.emailAddress, + ), +), +``` + + + +## With leading and trailing + + + +```dart +class PasswordInput extends StatefulWidget { + const PasswordInput({super.key}); + + @override + State createState() => _PasswordInputState(); +} + +class _PasswordInputState extends State { + bool obscure = true; + + @override + Widget build(BuildContext context) { + return ShadInput( + placeholder: const Text('Password'), + obscureText: obscure, + leading: Icon(LucideIcons.lock), + trailing: SizedBox.square( + dimension: 24, + child: OverflowBox( + maxWidth: 28, + maxHeight: 28, + child: ShadIconButton( + iconSize: 20, + padding: EdgeInsets.all(2), + icon: Icon(obscure ? LucideIcons.eyeOff : LucideIcons.eye), + onPressed: () { + setState(() => obscure = !obscure); + }, + ), + ), + ), + ); + } +} +``` + + + +## Form + + + +```dart +ShadInputFormField( + id: 'username', + label: const Text('Username'), + placeholder: const Text('Enter your username'), + description: + const Text('This is your public display name.'), + validator: (v) { + if (v.length < 2) { + return 'Username must be at least 2 characters.'; + } + return null; + }, +), +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class InputPage extends StatefulWidget { + const InputPage({super.key}); + + @override + State createState() => _InputPageState(); +} + +class _InputPageState extends State { + bool enabled = true; + bool obscure = true; + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Input', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyBoolProperty( + label: 'Obscure', + value: obscure, + onChanged: (value) => setState(() => obscure = value), + ), + ], + children: [ + ShadInput( + placeholder: const Text('Email'), + enabled: enabled, + keyboardType: TextInputType.emailAddress, + ), + ShadInput( + placeholder: const Text('Password'), + enabled: enabled, + obscureText: obscure, + leading: Icon(LucideIcons.lock), + trailing: SizedBox.square( + dimension: 24, + child: OverflowBox( + maxWidth: 28, + maxHeight: 28, + child: ShadIconButton( + iconSize: 20, + padding: EdgeInsets.all(2), + icon: Icon(obscure ? LucideIcons.eyeOff : LucideIcons.eye), + onPressed: () { + setState(() => obscure = !obscure); + }, + ), + ), + ), + ), + ], + ); + } +} + +``` + +## Form Example +```dart +import 'dart:convert'; +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:example/common/properties/string_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class InputFormFieldPage extends StatefulWidget { + const InputFormFieldPage({super.key}); + + @override + State createState() => _InputFormFieldPageState(); +} + +class _InputFormFieldPageState extends State { + bool enabled = true; + var autovalidateMode = ShadAutovalidateMode.alwaysAfterFirstValidation; + String? initialValue; + Map formValue = {}; + final formKey = GlobalKey(); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ShadForm( + key: formKey, + enabled: enabled, + autovalidateMode: autovalidateMode, + initialValue: { + if (initialValue != null) 'username': initialValue, + 'profile': {'age': 18}, + }, + child: BaseScaffold( + appBarTitle: 'InputFormField', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyEnumProperty( + label: 'autovalidateMode', + value: autovalidateMode, + values: ShadAutovalidateMode.values, + onChanged: (value) { + if (value != null) { + setState(() => autovalidateMode = value); + } + }, + ), + MyStringProperty( + label: 'Form Initial Value', + initialValue: initialValue, + placeholder: const Text('Name'), + onChanged: (value) { + formKey.currentState!.setFieldValue('username', value); + }, + ), + ], + children: [ + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 350), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + spacing: 16, + children: [ + ShadInputFormField( + id: 'username', + leading: const Icon(LucideIcons.user), + label: const Text('Username'), + placeholder: const Text('Enter your username'), + description: const Text('This is your public display name.'), + validator: (v) { + if (v.length < 2) { + return 'Username must be at least 2 characters.'; + } + return null; + }, + ), + ShadInputFormField( + id: 'profile.age', + fromValueTransformer: (v) => v?.toString(), + toValueTransformer: (String? v) => int.tryParse(v ?? ''), + keyboardType: TextInputType.number, + label: const Text('Age (dot notation)'), + placeholder: const Text('Enter your age'), + description: const Text( + 'This field uses dot notation: profile.age', + ), + ), + ShadButton( + child: const Text('Submit'), + onPressed: () { + print('submitted'); + if (formKey.currentState!.saveAndValidate()) { + setState(() { + formValue = formKey.currentState!.value; + }); + } else { + print('validation failed'); + } + }, + ), + + if (formValue.isNotEmpty) + Padding( + padding: const EdgeInsets.only(top: 24, left: 12), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('FormValue', style: theme.textTheme.p), + const SizedBox(height: 4), + SelectableText( + const JsonEncoder.withIndent( + ' ', + ).convert(formValue), + style: theme.textTheme.small, + ), + ], + ), + ), + ], + ), + ), + ], + ), + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/menubar.md b/.claude/skills/shadcn-ui-flutter/components/menubar.md new file mode 100644 index 0000000..dc672cf --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/menubar.md @@ -0,0 +1,262 @@ +# Menubar + +A visually persistent menu common in desktop applications that provides quick access to a consistent set of commands. + + + +```dart +class MenubarExample extends StatelessWidget { + const MenubarExample({super.key}); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + final square = SizedBox.square( + dimension: 16, + child: Center( + child: SizedBox.square( + dimension: 8, + child: DecoratedBox( + decoration: BoxDecoration( + color: theme.colorScheme.foreground, + shape: BoxShape.circle, + ), + ), + ), + ), + ); + final divider = ShadSeparator.horizontal( + margin: const EdgeInsets.symmetric(vertical: 4), + color: theme.colorScheme.muted, + ); + return ShadMenubar( + items: [ + ShadMenubarItem( + items: [ + const ShadContextMenuItem(child: Text('New Tab')), + const ShadContextMenuItem(child: Text('New Window')), + const ShadContextMenuItem( + enabled: false, + child: Text('New Incognito Window'), + ), + divider, + const ShadContextMenuItem( + trailing: Icon(LucideIcons.chevronRight), + items: [ + ShadContextMenuItem(child: Text('Email Link')), + ShadContextMenuItem(child: Text('Messages')), + ShadContextMenuItem(child: Text('Notes')), + ], + child: Text('Share'), + ), + divider, + const ShadContextMenuItem(child: Text('Print...')), + ], + child: const Text('File'), + ), + ShadMenubarItem( + items: [ + const ShadContextMenuItem(child: Text('Undo')), + const ShadContextMenuItem(child: Text('Redo')), + divider, + ShadContextMenuItem( + trailing: const Icon(LucideIcons.chevronRight), + items: [ + const ShadContextMenuItem(child: Text('Search the web')), + divider, + const ShadContextMenuItem(child: Text('Find...')), + const ShadContextMenuItem(child: Text('Find Next')), + const ShadContextMenuItem(child: Text('Find Previous')), + ], + child: const Text('Find'), + ), + divider, + const ShadContextMenuItem(child: Text('Cut')), + const ShadContextMenuItem(child: Text('Copy')), + const ShadContextMenuItem(child: Text('Paste')), + ], + child: const Text('Edit'), + ), + ShadMenubarItem( + items: [ + const ShadContextMenuItem.inset( + child: Text('Always Show Bookmarks Bar'), + ), + const ShadContextMenuItem( + leading: Icon(LucideIcons.check), + child: Text('Always Show Full URLs'), + ), + divider, + const ShadContextMenuItem.inset(child: Text('Reload')), + const ShadContextMenuItem.inset( + enabled: false, child: Text('Force Reload')), + divider, + const ShadContextMenuItem.inset( + child: Text('Toggle Full Screen'), + ), + divider, + const ShadContextMenuItem.inset(child: Text('Hide Sidebar')), + ], + child: const Text('View'), + ), + ShadMenubarItem(items: [ + const ShadContextMenuItem.inset(child: Text('Andy')), + ShadContextMenuItem(leading: square, child: const Text('Benoit')), + const ShadContextMenuItem.inset(child: Text('Luis')), + divider, + const ShadContextMenuItem.inset(child: Text('Edit...')), + divider, + const ShadContextMenuItem.inset(child: Text('Add Profile...')), + ], child: const Text('Profiles')), + ], + ); + } +} +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/widgets.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class MenubarPage extends StatefulWidget { + const MenubarPage({super.key}); + + @override + State createState() => _MenubarPageState(); +} + +class _MenubarPageState extends State { + var selectOnHover = true; + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + final square = SizedBox.square( + dimension: 16, + child: Center( + child: SizedBox.square( + dimension: 8, + child: DecoratedBox( + decoration: BoxDecoration( + color: theme.colorScheme.foreground, + shape: BoxShape.circle, + ), + ), + ), + ), + ); + final divider = ShadSeparator.horizontal( + margin: const EdgeInsets.symmetric(vertical: 4), + color: theme.colorScheme.muted, + ); + + return BaseScaffold( + appBarTitle: 'Menubar', + editable: [ + MyBoolProperty( + label: 'Select on hover', + value: selectOnHover, + onChanged: (value) => setState(() => selectOnHover = value), + ), + ], + children: [ + ShadMenubar( + selectOnHover: selectOnHover, + items: [ + ShadMenubarItem( + items: [ + const ShadContextMenuItem(child: Text('New Tab')), + const ShadContextMenuItem(child: Text('New Window')), + const ShadContextMenuItem( + enabled: false, + child: Text('New Incognito Window'), + ), + divider, + const ShadContextMenuItem( + trailing: Icon(LucideIcons.chevronRight), + items: [ + ShadContextMenuItem(child: Text('Email Link')), + ShadContextMenuItem(child: Text('Messages')), + ShadContextMenuItem(child: Text('Notes')), + ], + child: Text('Share'), + ), + divider, + const ShadContextMenuItem(child: Text('Print...')), + ], + child: const Text('File'), + ), + ShadMenubarItem( + items: [ + const ShadContextMenuItem(child: Text('Undo')), + const ShadContextMenuItem(child: Text('Redo')), + divider, + ShadContextMenuItem( + trailing: const Icon(LucideIcons.chevronRight), + items: [ + const ShadContextMenuItem(child: Text('Search the web')), + divider, + const ShadContextMenuItem(child: Text('Find...')), + const ShadContextMenuItem(child: Text('Find Next')), + const ShadContextMenuItem(child: Text('Find Previous')), + ], + child: const Text('Find'), + ), + divider, + const ShadContextMenuItem(child: Text('Cut')), + const ShadContextMenuItem(child: Text('Copy')), + const ShadContextMenuItem(child: Text('Paste')), + ], + child: const Text('Edit'), + ), + ShadMenubarItem( + items: [ + const ShadContextMenuItem.inset( + child: Text('Always Show Bookmarks Bar'), + ), + const ShadContextMenuItem( + leading: Icon(LucideIcons.check), + child: Text('Always Show Full URLs'), + ), + divider, + const ShadContextMenuItem.inset(child: Text('Reload')), + const ShadContextMenuItem.inset( + enabled: false, + child: Text('Force Reload'), + ), + divider, + const ShadContextMenuItem.inset( + child: Text('Toggle Full Screen'), + ), + divider, + const ShadContextMenuItem.inset(child: Text('Hide Sidebar')), + ], + child: const Text('View'), + ), + ShadMenubarItem( + items: [ + const ShadContextMenuItem.inset(child: Text('Andy')), + ShadContextMenuItem( + leading: square, + child: const Text('Benoit'), + ), + const ShadContextMenuItem.inset(child: Text('Luis')), + divider, + const ShadContextMenuItem.inset(child: Text('Edit...')), + divider, + const ShadContextMenuItem.inset(child: Text('Add Profile...')), + ], + child: const Text('Profiles'), + ), + ], + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/popover.md b/.claude/skills/shadcn-ui-flutter/components/popover.md new file mode 100644 index 0000000..acfe164 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/popover.md @@ -0,0 +1,178 @@ +# Popover + +Displays rich content in a portal, triggered by a button. + + + +```dart + + + + +class PopoverPage extends StatefulWidget { + const PopoverPage({super.key}); + + @override + State createState() => _PopoverPageState(); +} + +class _PopoverPageState extends State { + final popoverController = ShadPopoverController(); + + final List<({String name, String initialValue})> layer = [ + (name: 'Width', initialValue: '100%'), + (name: 'Max. width', initialValue: '300px'), + (name: 'Height', initialValue: '25px'), + (name: 'Max. height', initialValue: 'none'), + ]; + + @override + void dispose() { + popoverController.dispose(); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + final textTheme = ShadTheme.of(context).textTheme; + return Scaffold( + body: Center( + child: ShadPopover( + controller: popoverController, + popover: (context) => SizedBox( + width: 288, + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + mainAxisSize: MainAxisSize.min, + children: [ + Text( + 'Dimensions', + style: textTheme.h4, + ), + Text( + 'Set the dimensions for the layer.', + style: textTheme.p, + ), + const SizedBox(height: 4), + ...layer + .map( + (e) => Row( + mainAxisAlignment: MainAxisAlignment.start, + children: [ + Expanded( + child: Text( + e.name, + textAlign: TextAlign.start, + )), + Expanded( + flex: 2, + child: ShadInput( + initialValue: e.initialValue, + ), + ) + ], + ), + ) + .separatedBy(const SizedBox(height: 8)), + ], + ), + ), + child: ShadButton.outline( + onPressed: popoverController.toggle, + child: const Text('Open popover'), + ), + ), + ), + ); + } +} +``` + +## Example +```dart +import 'package:awesome_flutter_extensions/awesome_flutter_extensions.dart'; +import 'package:example/common/base_scaffold.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class PopoverPage extends StatefulWidget { + const PopoverPage({super.key}); + + @override + State createState() => _PopoverPageState(); +} + +class _PopoverPageState extends State { + final popoverController = ShadPopoverController(); + + final List<({String name, String initialValue})> layer = [ + (name: 'Width', initialValue: '100%'), + (name: 'Max. width', initialValue: '300px'), + (name: 'Height', initialValue: '25px'), + (name: 'Max. height', initialValue: 'none'), + ]; + + @override + void dispose() { + popoverController.dispose(); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + final textTheme = ShadTheme.of(context).textTheme; + return BaseScaffold( + appBarTitle: 'Popover', + children: [ + ShadPopover( + controller: popoverController, + popover: (_) => SizedBox( + width: 288, + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + mainAxisSize: MainAxisSize.min, + children: [ + Text( + 'Dimensions', + style: textTheme.h4, + ), + Text( + 'Set the dimensions for the layer.', + style: textTheme.p, + ), + const SizedBox(height: 4), + ...layer + .map( + (e) => Row( + mainAxisAlignment: MainAxisAlignment.start, + children: [ + Expanded( + child: Text( + e.name, + textAlign: TextAlign.start, + ), + ), + Expanded( + flex: 2, + child: ShadInput( + initialValue: e.initialValue, + ), + ), + ], + ), + ) + .separatedBy(const SizedBox(height: 8)), + ], + ), + ), + child: ShadButton.outline( + onPressed: popoverController.toggle, + child: const Text('Open popover'), + ), + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/progress.md b/.claude/skills/shadcn-ui-flutter/components/progress.md new file mode 100644 index 0000000..aa6ae13 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/progress.md @@ -0,0 +1,77 @@ +# Progress + +Displays an indicator showing the completion progress of a task, typically displayed as a progress bar. + +## Determinate + + + ```dart +ConstrainedBox( + constraints: BoxConstraints( + maxWidth: MediaQuery.sizeOf(context).width * 0.6, + ), + child: const ShadProgress(value: 0.5), +), +``` + + + +## Indeterminate + + ```dart +ConstrainedBox( + constraints: BoxConstraints( + maxWidth: MediaQuery.sizeOf(context).width * 0.6, + ), + child: const ShadProgress(), +), +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class ProgressPage extends StatefulWidget { + const ProgressPage({super.key}); + + @override + State createState() => _ProgressPageState(); +} + +class _ProgressPageState extends State { + var value = 50; + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Progress', + children: [ + Text('Determinate Progress'), + ShadProgress( + value: value / 100, + ), + Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + ShadIconButton.ghost( + enabled: value != 0, + onPressed: () => setState(() => value -= 10), + icon: const Icon(Icons.remove), + ), + ShadIconButton.ghost( + enabled: value != 100, + onPressed: () => setState(() => value += 10), + icon: const Icon(Icons.add), + ), + ], + ), + Text('Indeterminate Progress'), + ShadProgress(), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/radio-group.md b/.claude/skills/shadcn-ui-flutter/components/radio-group.md new file mode 100644 index 0000000..3e5ab5b --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/radio-group.md @@ -0,0 +1,284 @@ +# RadioGroup + +A set of checkable buttons—known as radio buttons—where no more than one of the buttons can be checked at a time. + + + +```dart +ShadRadioGroup( + items: [ + ShadRadio( + label: Text('Default'), + value: 'default', + ), + ShadRadio( + label: Text('Comfortable'), + value: 'comfortable', + ), + ShadRadio( + label: Text('Nothing'), + value: 'nothing', + ), + ], +), +``` + + + +## Form + + + +```dart +enum NotifyAbout { + all, + mentions, + nothing; + + String get message { + return switch (this) { + all => 'All new messages', + mentions => 'Direct messages and mentions', + nothing => 'Nothing', + }; + } +} + +ShadRadioGroupFormField( + label: const Text('Notify me about'), + items: NotifyAbout.values.map( + (e) => ShadRadio( + value: e, + label: Text(e.message), + ), + ), + validator: (v) { + if (v == null) { + return 'You need to select a notification type.'; + } + return null; + }, +), +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/widgets.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +enum NotifyAbout { + all, + mentions, + nothing; + + String get message { + return switch (this) { + all => 'All new messages', + mentions => 'Direct messages and mentions', + nothing => 'Nothing', + }; + } +} + +class RadioPage extends StatefulWidget { + const RadioPage({super.key}); + + @override + State createState() => _RadioPageState(); +} + +class _RadioPageState extends State { + NotifyAbout? value; + bool enabled = true; + Axis axis = Axis.vertical; + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'RadioGroup', + crossAxisAlignment: CrossAxisAlignment.start, + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyEnumProperty( + label: 'Axis', + value: axis, + onChanged: (value) { + if (value != null) { + setState(() => axis = value); + } + }, + values: Axis.values, + ), + ], + children: [ + ShadRadioGroup( + enabled: enabled, + initialValue: value, + onChanged: (v) { + print('onChange $v'); + }, + axis: axis, + items: NotifyAbout.values.map( + (e) => ShadRadio( + value: e, + label: Text(e.message), + ), + ), + ), + ], + ); + } +} + +``` + +## Form Example +```dart +// ignore_for_file: avoid_print + +import 'dart:convert'; + +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +enum NotifyAbout { + all, + mentions, + nothing; + + String get message { + return switch (this) { + all => 'All new messages', + mentions => 'Direct messages and mentions', + nothing => 'Nothing', + }; + } +} + +class RadioGroupFormFieldPage extends StatefulWidget { + const RadioGroupFormFieldPage({super.key}); + + @override + State createState() => + _RadioGroupFormFieldPageState(); +} + +class _RadioGroupFormFieldPageState extends State { + bool enabled = true; + var autovalidateMode = ShadAutovalidateMode.alwaysAfterFirstValidation; + NotifyAbout? initialValue; + Map formValue = {}; + final formKey = GlobalKey(); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ShadForm( + key: formKey, + enabled: enabled, + autovalidateMode: autovalidateMode, + initialValue: {'notify': initialValue}, + child: BaseScaffold( + appBarTitle: 'RadioGroupFormField', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyEnumProperty( + label: 'autovalidateMode', + value: autovalidateMode, + values: ShadAutovalidateMode.values, + onChanged: (value) { + if (value != null) { + setState(() => autovalidateMode = value); + } + }, + ), + MyEnumProperty( + label: 'Form Initial Value', + value: NotifyAbout.nothing, + values: NotifyAbout.values, + onChanged: (value) { + formKey.currentState!.setFieldValue('notify', value); + }, + ), + ], + children: [ + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 350), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + ShadRadioGroupFormField( + id: 'notify', + enabled: enabled, + initialValue: initialValue, + toValueTransformer: (value) => value?.name, + items: NotifyAbout.values.map( + (e) => ShadRadio( + value: e, + label: Text(e.message), + ), + ), + label: const Text('Notify me about'), + validator: (v) { + if (v == null) { + return 'You need to select a notification type.'; + } + return null; + }, + ), + const SizedBox(height: 16), + ShadButton( + child: const Text('Submit'), + onPressed: () { + print('submitted'); + if (formKey.currentState!.saveAndValidate()) { + setState(() { + formValue = formKey.currentState!.value; + }); + } else { + print('validation failed'); + } + }, + ), + if (formValue.isNotEmpty) + Padding( + padding: const EdgeInsets.only(top: 24, left: 12), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('FormValue', style: theme.textTheme.p), + const SizedBox(height: 4), + SelectableText( + const JsonEncoder.withIndent( + ' ', + ).convert(formValue), + style: theme.textTheme.small, + ), + ], + ), + ), + ], + ), + ), + ], + ), + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/resizable.md b/.claude/skills/shadcn-ui-flutter/components/resizable.md new file mode 100644 index 0000000..b78c67a --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/resizable.md @@ -0,0 +1,284 @@ +# Resizable + +Resizable panel groups and layouts. + + + +```dart +class BasicResizable extends StatelessWidget { + const BasicResizable({super.key}); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ConstrainedBox( + constraints: const BoxConstraints(maxHeight: 200), + child: DecoratedBox( + decoration: BoxDecoration( + borderRadius: theme.radius, + border: Border.all( + color: theme.colorScheme.border, + ), + ), + child: ClipRRect( + borderRadius: theme.radius, + child: ShadResizablePanelGroup( + children: [ + ShadResizablePanel( + id: 0, + defaultSize: .5, + minSize: .2, + maxSize: .8, + child: Center( + child: Text('One', style: theme.textTheme.large), + ), + ), + ShadResizablePanel( + id: 1, + defaultSize: .5, + child: ShadResizablePanelGroup( + axis: Axis.vertical, + children: [ + ShadResizablePanel( + id: 0, + defaultSize: .3, + child: Center( + child: Text('Two', style: theme.textTheme.large)), + ), + ShadResizablePanel( + id: 1, + defaultSize: .7, + child: Align( + child: Text('Three', style: theme.textTheme.large)), + ), + ], + ), + ), + ], + ), + ), + ), + ); + } +} +``` + + + +Try resizing a panel, then double-click on the handle to reset to the default size. + + +## Vertical + +Use the `axis` property to change the direction of the resizable panels. + + + +```dart +class VerticalResizable extends StatelessWidget { + const VerticalResizable({super.key}); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ConstrainedBox( + constraints: const BoxConstraints(maxHeight: 200), + child: DecoratedBox( + decoration: BoxDecoration( + borderRadius: theme.radius, + border: Border.all( + color: theme.colorScheme.border, + ), + ), + child: ClipRRect( + borderRadius: theme.radius, + child: ShadResizablePanelGroup( + axis: Axis.vertical, + children: [ + ShadResizablePanel( + id: 0, + defaultSize: 0.3, + minSize: 0.1, + child: Center( + child: Text('Header', style: theme.textTheme.large), + ), + ), + ShadResizablePanel( + id: 1, + defaultSize: 0.7, + minSize: 0.1, + child: Center( + child: Text('Footer', style: theme.textTheme.large), + ), + ), + ], + ), + ), + ), + ); + } +} +``` + + + +## Handle + +You can show the handle by using the `showHandle` property. + +You can customize it using the `handleIcon` or `handleIconSrc` properties. + + + +```dart +class HandleResizable extends StatelessWidget { + const HandleResizable({super.key}); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ConstrainedBox( + constraints: const BoxConstraints(maxHeight: 200), + child: DecoratedBox( + decoration: BoxDecoration( + borderRadius: theme.radius, + border: Border.all( + color: theme.colorScheme.border, + ), + ), + child: ClipRRect( + borderRadius: theme.radius, + child: ShadResizablePanelGroup( + showHandle: true, + children: [ + ShadResizablePanel( + id: 0, + defaultSize: .5, + minSize: .2, + child: Center( + child: Text('Sidebar', style: theme.textTheme.large), + ), + ), + ShadResizablePanel( + id: 1, + defaultSize: .5, + minSize: .2, + child: Center( + child: Text('Content', style: theme.textTheme.large), + ), + ), + ], + ), + ), + ), + ); + } +} +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class ResizablePage extends StatefulWidget { + const ResizablePage({super.key}); + + @override + State createState() => _ResizablePageState(); +} + +class _ResizablePageState extends State { + var visible = true; + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return BaseScaffold( + appBarTitle: 'Resizable', + editable: [ + MyBoolProperty( + value: visible, + onChanged: (v) => setState(() => visible = v), + label: 'One Visible', + ), + ], + children: [ + SizedBox( + width: 300, + height: 200, + child: DecoratedBox( + decoration: BoxDecoration( + borderRadius: theme.radius, + border: Border.all( + color: theme.colorScheme.border, + ), + ), + child: ClipRRect( + borderRadius: theme.radius, + child: ShadResizablePanelGroup( + mainAxisSize: MainAxisSize.min, + showHandle: true, + children: [ + if (visible) + ShadResizablePanel( + id: 0, + defaultSize: .5, + minSize: 0.1, + maxSize: 0.8, + child: Container( + color: Colors.red, + alignment: Alignment.center, + child: Text( + 'One', + style: theme.textTheme.large, + ), + ), + ), + ShadResizablePanel( + defaultSize: 0.5, + id: 1, + child: ShadResizablePanelGroup( + axis: Axis.vertical, + showHandle: true, + children: [ + ShadResizablePanel( + id: 0, + defaultSize: 0.4, + child: Container( + color: Colors.blue, + alignment: Alignment.center, + child: Text( + 'Two', + style: theme.textTheme.large, + ), + ), + ), + ShadResizablePanel( + id: 1, + defaultSize: 0.6, + child: Container( + color: Colors.green, + alignment: Alignment.center, + child: Text( + 'Three', + style: theme.textTheme.large, + ), + ), + ), + ], + ), + ), + ], + ), + ), + ), + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/select.md b/.claude/skills/shadcn-ui-flutter/components/select.md new file mode 100644 index 0000000..9c351ac --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/select.md @@ -0,0 +1,735 @@ +# Select + +Displays a list of options for the user to pick from—triggered by a button. + + + +```dart +final fruits = { + 'apple': 'Apple', + 'banana': 'Banana', + 'blueberry': 'Blueberry', + 'grapes': 'Grapes', + 'pineapple': 'Pineapple', +}; + +class SelectExample extends StatelessWidget { + const SelectExample({super.key}); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ConstrainedBox( + constraints: const BoxConstraints(minWidth: 180), + child: ShadSelect( + placeholder: const Text('Select a fruit'), + options: [ + Padding( + padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 6), + child: Text( + 'Fruits', + style: theme.textTheme.muted.copyWith( + fontWeight: FontWeight.w600, + color: theme.colorScheme.popoverForeground, + ), + textAlign: TextAlign.start, + ), + ), + ...fruits.entries + .map((e) => ShadOption(value: e.key, child: Text(e.value))), + ], + selectedOptionBuilder: (context, value) => Text(fruits[value]!), + onChanged: print, + ), + ); + } +} +``` + + + +## Scrollable + + + +```dart +final timezones = { + 'North America': { + 'est': 'Eastern Standard Time (EST)', + 'cst': 'Central Standard Time (CST)', + 'mst': 'Mountain Standard Time (MST)', + 'pst': 'Pacific Standard Time (PST)', + 'akst': 'Alaska Standard Time (AKST)', + 'hst': 'Hawaii Standard Time (HST)', + }, + 'Europe & Africa': { + 'gmt': 'Greenwich Mean Time (GMT)', + 'cet': 'Central European Time (CET)', + 'eet': 'Eastern European Time (EET)', + 'west': 'Western European Summer Time (WEST)', + 'cat': 'Central Africa Time (CAT)', + 'eat': 'Eastern Africa Time (EAT)', + }, + 'Asia': { + 'msk': 'Moscow Time (MSK)', + 'ist': 'India Standard Time (IST)', + 'cst_china': 'China Standard Time (CST)', + 'jst': 'Japan Standard Time (JST)', + 'kst': 'Korea Standard Time (KST)', + 'ist_indonasia': 'Indonesia Standard Time (IST)', + }, + 'Australia & Pacific': { + 'awst': 'Australian Western Standard Time (AWST)', + 'acst': 'Australian Central Standard Time (ACST)', + 'aest': 'Australian Eastern Standard Time (AEST)', + 'nzst': 'New Zealand Standard Time (NZST)', + 'fjt': 'Fiji Time (FJT)', + }, + 'South America': { + 'art': 'Argentina Time (ART)', + 'bot': 'Bolivia Time (BOT)', + 'brt': 'Brasilia Time (BRT)', + 'clt': 'Chile Standard Time (CLT)', + }, +}; + +List getTimezonesWidgets(ShadThemeData theme) { + final widgets = []; + for (final zone in timezones.entries) { + widgets.add( + Padding( + padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 6), + child: Text( + zone.key, + style: theme.textTheme.muted.copyWith( + fontWeight: FontWeight.w600, + color: theme.colorScheme.popoverForeground, + ), + textAlign: TextAlign.start, + ), + ), + ); + widgets.addAll(zone.value.entries + .map((e) => ShadOption(value: e.key, child: Text(e.value)))); + } + return widgets; +} + +class SelectExample extends StatelessWidget { + const SelectExample({super.key}); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ConstrainedBox( + constraints: const BoxConstraints(minWidth: 280), + child: ShadSelect( + placeholder: const Text('Select a timezone'), + options: getTimezonesWidgets(theme), + selectedOptionBuilder: (context, value) { + final timezone = timezones.entries + .firstWhere((element) => element.value.containsKey(value)) + .value[value]; + return Text(timezone!); + }, + ), + ); + } +} +``` + + + +## Form + + + +```dart +final verifiedEmails = [ + 'm@example.com', + 'm@google.com', + 'm@support.com', +]; + +class SelectFormField extends StatelessWidget { + const SelectFormField({super.key}); + + @override + Widget build(BuildContext context) { + return ShadSelectFormField( + id: 'email', + minWidth: 350, + initialValue: null, + options: verifiedEmails + .map((email) => ShadOption(value: email, child: Text(email))) + .toList(), + selectedOptionBuilder: (context, value) => value == 'none' + ? const Text('Select a verified email to display') + : Text(value), + placeholder: const Text('Select a verified email to display'), + validator: (v) { + if (v == null) { + return 'Please select an email to display'; + } + return null; + }, + ); + } +} +``` + + + +## With Search + + + +```dart +const frameworks = { +'nextjs': 'Next.js', +'svelte': 'SvelteKit', +'nuxtjs': 'Nuxt.js', +'remix': 'Remix', +'astro': 'Astro', +}; + +class SelectWithSearch extends StatefulWidget { + const SelectWithSearch({super.key}); + + @override + State createState() => _SelectWithSearchState(); +} + +class _SelectWithSearchState extends State { + var searchValue = ''; + + Map get filteredFrameworks => { + for (final framework in frameworks.entries) + if (framework.value.toLowerCase().contains(searchValue.toLowerCase())) + framework.key: framework.value + }; + + @override + Widget build(BuildContext context) { + return ShadSelect.withSearch( + minWidth: 180, + maxWidth: 300, + placeholder: const Text('Select framework...'), + onSearchChanged: (value) => setState(() => searchValue = value), + searchPlaceholder: const Text('Search framework'), + options: [ + if (filteredFrameworks.isEmpty) + const Padding( + padding: EdgeInsets.symmetric(vertical: 24), + child: Text('No framework found'), + ), + ...frameworks.entries.map( + (framework) { + // this offstage is used to avoid the focus loss when the search results appear again + // because it keeps the widget in the tree. + return Offstage( + offstage: !filteredFrameworks.containsKey(framework.key), + child: ShadOption( + value: framework.key, + child: Text(framework.value), + ), + ); + }, + ) + ], + selectedOptionBuilder: (context, value) => Text(frameworks[value]!), + ); + } +} +``` + + + +If you want to be able to deselect an option, you can use the `allowDeselection` property. + + +## Multiple + +This example shows how to select multiple options. + +In addition, the `allowDeselection` property is set to `true` to allow the user to deselect an option and the `closeOnSelect` property is set to `false` to keep the popover open after selecting an option. +If you tap outside the popover, it will close. + + + +```dart +final fruits = { + 'apple': 'Apple', + 'banana': 'Banana', + 'blueberry': 'Blueberry', + 'grapes': 'Grapes', + 'pineapple': 'Pineapple', +}; + +class SelectMultiple extends StatelessWidget { + const SelectMultiple({super.key}); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ShadSelect.multiple( + minWidth: 340, + onChanged: print, + allowDeselection: true, + closeOnSelect: false, + placeholder: const Text('Select multiple fruits'), + options: [ + Padding( + padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 6), + child: Text( + 'Fruits', + style: theme.textTheme.large, + textAlign: TextAlign.start, + ), + ), + ...fruits.entries.map( + (e) => ShadOption( + value: e.key, + child: Text(e.value), + ), + ), + ], + selectedOptionsBuilder: (context, values) => + Text(values.map((v) => v.capitalize()).join(', ')), + ); + } +} +``` + +## Example +```dart +// ignore_for_file: avoid_print + +import 'package:awesome_flutter_extensions/awesome_flutter_extensions.dart'; +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +const fruits = { + 'apple': 'Apple', + 'banana': 'Banana', + 'blueberry': 'Blueberry', + 'grapes': 'Grapes', + 'pineapple': 'Pineapple', +}; + +const timezones = { + 'North America': { + 'est': 'Eastern Standard Time (EST)', + 'cst': 'Central Standard Time (CST)', + 'mst': 'Mountain Standard Time (MST)', + 'pst': 'Pacific Standard Time (PST)', + 'akst': 'Alaska Standard Time (AKST)', + 'hst': 'Hawaii Standard Time (HST)', + }, + 'Europe & Africa': { + 'gmt': 'Greenwich Mean Time (GMT)', + 'cet': 'Central European Time (CET)', + 'eet': 'Eastern European Time (EET)', + 'west': 'Western European Summer Time (WEST)', + 'cat': 'Central Africa Time (CAT)', + 'eat': 'Eastern Africa Time (EAT)', + }, + 'Asia': { + 'msk': 'Moscow Time (MSK)', + 'ist': 'India Standard Time (IST)', + 'cst_china': 'China Standard Time (CST)', + 'jst': 'Japan Standard Time (JST)', + 'kst': 'Korea Standard Time (KST)', + 'ist_indonasia': 'Indonesia Standard Time (IST)', + }, + 'Australia & Pacific': { + 'awst': 'Australian Western Standard Time (AWST)', + 'acst': 'Australian Central Standard Time (ACST)', + 'aest': 'Australian Eastern Standard Time (AEST)', + 'nzst': 'New Zealand Standard Time (NZST)', + 'fjt': 'Fiji Time (FJT)', + }, + 'South America': { + 'art': 'Argentina Time (ART)', + 'bot': 'Bolivia Time (BOT)', + 'brt': 'Brasilia Time (BRT)', + 'clt': 'Chile Standard Time (CLT)', + }, +}; + +const frameworks = { + 'nextjs': 'Next.js', + 'svelte': 'SvelteKit', + 'nuxtjs': 'Nuxt.js', + 'remix': 'Remix', + 'astro': 'Astro', +}; + +class SelectPage extends StatefulWidget { + const SelectPage({super.key}); + + @override + State createState() => _SelectPageState(); +} + +class _SelectPageState extends State { + bool enabled = true; + final focusNodes = [FocusNode(), FocusNode(), FocusNode(), FocusNode()]; + var searchValue = ''; + bool allowDeselection = false; + bool closeOnSelect = true; + bool ensureSelectedVisible = true; + + Map get filteredFrameworks => { + for (final framework in frameworks.entries) + if (framework.value.toLowerCase().contains(searchValue.toLowerCase())) + framework.key: framework.value, + }; + + @override + void dispose() { + for (final node in focusNodes) { + node.dispose(); + } + super.dispose(); + } + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return BaseScaffold( + appBarTitle: 'Select', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyBoolProperty( + label: 'Fruits Focused', + value: focusNodes[0].hasFocus, + onChanged: (value) => setState( + () => + value ? focusNodes[0].requestFocus() : focusNodes[0].unfocus(), + ), + ), + MyBoolProperty( + label: 'Timezone Focused', + value: focusNodes[1].hasFocus, + onChanged: (value) => setState( + () => + value ? focusNodes[1].requestFocus() : focusNodes[1].unfocus(), + ), + ), + MyBoolProperty( + label: 'Framework Focused', + value: focusNodes[2].hasFocus, + onChanged: (value) => setState( + () => + value ? focusNodes[2].requestFocus() : focusNodes[2].unfocus(), + ), + ), + MyBoolProperty( + label: 'Allow deselection', + value: allowDeselection, + onChanged: (value) => setState(() => allowDeselection = value), + ), + MyBoolProperty( + label: 'Close on select', + value: closeOnSelect, + onChanged: (value) => setState(() => closeOnSelect = value), + ), + MyBoolProperty( + label: 'Ensure selected visible', + value: ensureSelectedVisible, + onChanged: (value) => setState(() => ensureSelectedVisible = value), + ), + ], + children: [ + ShadSelect( + minWidth: 180, + onChanged: print, + closeOnSelect: closeOnSelect, + enabled: enabled, + focusNode: focusNodes[0], + placeholder: const Text('Select a fruit'), + allowDeselection: allowDeselection, + ensureSelectedVisible: ensureSelectedVisible, + options: [ + Padding( + padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 6), + child: Text( + 'Fruits', + style: theme.textTheme.large, + textAlign: TextAlign.start, + ), + ), + ...fruits.entries.map( + (e) => ShadOption( + value: e.key, + child: Text(e.value), + ), + ), + ], + selectedOptionBuilder: (context, value) => Text(value.capitalize()), + ), + ShadSelect( + minWidth: 280, + focusNode: focusNodes[1], + onChanged: print, + enabled: enabled, + closeOnSelect: closeOnSelect, + placeholder: const Text('Select a timezone'), + ensureSelectedVisible: ensureSelectedVisible, + options: timezones.entries.map( + (zone) => Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + Padding( + padding: const EdgeInsets.symmetric( + horizontal: 8, + vertical: 6, + ), + child: Text( + zone.key, + style: theme.textTheme.muted.copyWith( + fontWeight: FontWeight.w600, + color: theme.colorScheme.popoverForeground, + ), + textAlign: TextAlign.start, + ), + ), + ...zone.value.entries.map( + (e) => ShadOption( + value: e.key, + child: Text(e.value), + ), + ), + ], + ), + ), + allowDeselection: allowDeselection, + selectedOptionBuilder: (context, value) { + final timezone = timezones.entries + .firstWhere((element) => element.value.containsKey(value)) + .value[value]; + return Text(timezone!); + }, + ), + ShadSelect.withSearch( + enabled: enabled, + focusNode: focusNodes[2], + minWidth: 180, + maxWidth: 300, + placeholder: const Text('Select framework...'), + onSearchChanged: (value) => setState(() => searchValue = value), + closeOnSelect: closeOnSelect, + searchPlaceholder: const Text('Search framework'), + ensureSelectedVisible: ensureSelectedVisible, + options: [ + if (filteredFrameworks.isEmpty) + const Padding( + padding: EdgeInsets.symmetric(vertical: 24), + child: Text('No framework found'), + ), + ...frameworks.entries.map( + (framework) { + // this offstage is used to avoid the focus loss when the search results appear again + // because it keeps the widget in the tree. + return Offstage( + offstage: !filteredFrameworks.containsKey(framework.key), + child: ShadOption( + value: framework.key, + child: Text(framework.value), + ), + ); + }, + ), + ], + selectedOptionBuilder: (context, value) => Text(frameworks[value]!), + onChanged: print, + allowDeselection: allowDeselection, + ), + ShadSelect.multiple( + minWidth: 340, + onChanged: print, + enabled: enabled, + focusNode: focusNodes[3], + allowDeselection: allowDeselection, + placeholder: const Text('Select multiple fruits'), + closeOnSelect: closeOnSelect, + ensureSelectedVisible: ensureSelectedVisible, + options: [ + Padding( + padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 6), + child: Text( + 'Fruits', + style: theme.textTheme.large, + textAlign: TextAlign.start, + ), + ), + ...fruits.entries.map( + (e) => ShadOption( + value: e.key, + child: Text(e.value), + ), + ), + ], + selectedOptionsBuilder: (context, values) => + Text(values.map((v) => v.capitalize()).join(', ')), + ), + ], + ); + } +} + +``` + +## Form Example +```dart +// ignore_for_file: avoid_print + +import 'dart:convert'; + +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class SelectFormFieldPage extends StatefulWidget { + const SelectFormFieldPage({super.key}); + + @override + State createState() => _SelectFormFieldPageState(); +} + +class _SelectFormFieldPageState extends State { + bool enabled = true; + var autovalidateMode = ShadAutovalidateMode.alwaysAfterFirstValidation; + final verifiedEmails = [ + 'm@example.com', + 'm@google.com', + 'm@support.com', + ]; + String? initialValue; + Map formValue = {}; + final formKey = GlobalKey(); + bool allowDeselection = false; + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ShadForm( + key: formKey, + enabled: enabled, + autovalidateMode: autovalidateMode, + initialValue: {'email': initialValue}, + child: BaseScaffold( + appBarTitle: 'SelectFormField', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyEnumProperty( + label: 'autovalidateMode', + value: autovalidateMode, + values: ShadAutovalidateMode.values, + onChanged: (value) { + if (value != null) { + setState(() => autovalidateMode = value); + } + }, + ), + ShadSelect( + options: ['none', ...verifiedEmails].map( + (e) => ShadOption(value: e, child: Text(e.toString())), + ), + initialValue: initialValue, + placeholder: const Text('Form Initial Value'), + onChanged: (v) { + formKey.currentState!.setFieldValue('email', v); + }, + selectedOptionBuilder: (context, value) => Text( + value.toString(), + ), + ), + MyBoolProperty( + label: 'Allow deselection', + value: allowDeselection, + onChanged: (value) => setState(() => allowDeselection = value), + ), + ], + children: [ + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 350), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + ShadSelectFormField( + id: 'email', + allowDeselection: allowDeselection, + minWidth: 350, + initialValue: initialValue, + onChanged: print, + options: verifiedEmails + .map( + (email) => ShadOption(value: email, child: Text(email)), + ) + .toList(), + selectedOptionBuilder: (context, value) => value == 'none' + ? const Text('Select a verified email to display') + : Text(value), + placeholder: const Text('Select a verified email to display'), + validator: (v) { + if (v == null) { + return 'Please select an email to display'; + } + return null; + }, + ), + const SizedBox(height: 16), + ShadButton( + child: const Text('Submit'), + onPressed: () { + print('submitted'); + if (formKey.currentState!.saveAndValidate()) { + setState(() { + formValue = formKey.currentState!.value; + }); + } else { + print('validation failed'); + } + }, + ), + if (formValue.isNotEmpty) + Padding( + padding: const EdgeInsets.only(top: 24, left: 12), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('FormValue', style: theme.textTheme.p), + const SizedBox(height: 4), + SelectableText( + const JsonEncoder.withIndent( + ' ', + ).convert(formValue), + style: theme.textTheme.small, + ), + ], + ), + ), + ], + ), + ), + ], + ), + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/separator.md b/.claude/skills/shadcn-ui-flutter/components/separator.md new file mode 100644 index 0000000..c982c2b --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/separator.md @@ -0,0 +1,110 @@ +# Separator + +Visually or semantically separates content. + + + +```dart +const ShadSeparator.horizontal( + thickness: 4, + margin: EdgeInsets.symmetric(horizontal: 20), + radius: BorderRadius.all(Radius.circular(4)), +) +``` + + + +## Destructive + + + +```dart +const ShadSeparator.vertical( + thickness: 4, + margin: EdgeInsets.symmetric(vertical: 20), + radius: BorderRadius.all(Radius.circular(4)), +) +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/string_property.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class SeparatorPage extends StatefulWidget { + const SeparatorPage({super.key}); + + @override + State createState() => _SeparatorPageState(); +} + +class _SeparatorPageState extends State { + int margin = 4; + int thickness = 1; + int radius = 0; + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return BaseScaffold( + appBarTitle: 'Separator', + editable: [ + MyStringProperty( + label: 'margin', + initialValue: '$margin', + inputFormatters: [FilteringTextInputFormatter.digitsOnly], + onChanged: (value) { + var maybe = int.tryParse(value); + if (maybe != null) setState(() => margin = maybe); + }, + ), + MyStringProperty( + label: 'thickness', + initialValue: '$thickness', + inputFormatters: [FilteringTextInputFormatter.digitsOnly], + onChanged: (value) { + var maybe = int.tryParse(value); + if (maybe != null) setState(() => thickness = maybe); + }, + ), + MyStringProperty( + label: 'radius', + initialValue: '$radius', + inputFormatters: [FilteringTextInputFormatter.digitsOnly], + onChanged: (value) { + var maybe = int.tryParse(value); + if (maybe != null) setState(() => radius = maybe); + }, + ), + ], + children: [ + Text('Horizontal', style: theme.textTheme.h4), + ShadSeparator.horizontal( + thickness: thickness.toDouble(), + margin: EdgeInsets.all(margin.toDouble()), + radius: BorderRadius.all(Radius.circular(radius.toDouble())), + ), + IntrinsicHeight( + child: Row( + mainAxisSize: MainAxisSize.max, + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Text('Vertical', style: theme.textTheme.h4), + ShadSeparator.vertical( + thickness: thickness.toDouble(), + margin: EdgeInsets.all(margin.toDouble()), + radius: BorderRadius.all(Radius.circular(radius.toDouble())), + ), + Text('divider', style: theme.textTheme.h4), + ], + ), + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/sheet.md b/.claude/skills/shadcn-ui-flutter/components/sheet.md new file mode 100644 index 0000000..c4ba88d --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/sheet.md @@ -0,0 +1,273 @@ +# Sheet + +Extends the Dialog component to display content that complements the main content of the screen. + + + +```dart +ShadButton.outline( + child: const Text('Open'), + onPressed: () => showShadSheet( + side: ShadSheetSide.right, + context: context, + builder: (context) => const EditProfileSheet(), + ), +), + +final profile = [ + (title: 'Name', value: 'Alexandru'), + (title: 'Username', value: 'nank1ro'), +]; + +class EditProfileSheet extends StatelessWidget { + const EditProfileSheet({super.key, required this.side}); + + final ShadSheetSide side; + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ShadSheet( + constraints: side == ShadSheetSide.left || side == ShadSheetSide.right + ? const BoxConstraints(maxWidth: 512) + : null, + title: const Text('Edit Profile'), + description: const Text( + "Make changes to your profile here. Click save when you're done"), + child: Padding( + padding: const EdgeInsets.symmetric(vertical: 20), + child: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.stretch, + spacing: 16, + children: profile + .map( + (p) => Row( + children: [ + Expanded( + child: Text( + p.title, + textAlign: TextAlign.end, + style: theme.textTheme.small, + ), + ), + const SizedBox(width: 16), + Expanded( + flex: 5, + child: ShadInput(initialValue: p.value), + ), + ], + ), + ) + .toList(), + ), + ), + actions: const [ + ShadButton(child: Text('Save changes')), + ], + ); + } +} +``` + + + +## Side + +Use the `side` property to `showShadSheet` to indicate the edge of the screen where the component will appear. The values can be `top`, `right`, `bottom` or `left`. + + + +```dart +Row( + mainAxisSize: MainAxisSize.min, + spacing: 16, + children: [ + Column( + spacing: 16, + mainAxisSize: MainAxisSize.min, + children: [ + ShadButton.outline( + width: 100, + child: const Text('Top'), + onPressed: () => showShadSheet( + side: ShadSheetSide.top, + context: context, + builder: (context) => + const EditProfileSheet(side: ShadSheetSide.top), + ), + ), + ShadButton.outline( + width: 100, + child: const Text('Bottom'), + onPressed: () => showShadSheet( + side: ShadSheetSide.bottom, + context: context, + builder: (context) => const EditProfileSheet( + side: ShadSheetSide.bottom), + ), + ), + ], + ), + Column( + spacing: 16, + mainAxisSize: MainAxisSize.min, + children: [ + ShadButton.outline( + width: 100, + child: const Text('Right'), + onPressed: () => showShadSheet( + side: ShadSheetSide.right, + context: context, + builder: (context) => const EditProfileSheet( + side: ShadSheetSide.right), + ), + ), + ShadButton.outline( + width: 100, + child: const Text('Left'), + onPressed: () => showShadSheet( + side: ShadSheetSide.left, + context: context, + builder: (context) => const EditProfileSheet( + side: ShadSheetSide.left), + ), + ), + ], + ), + ], +), + +// See EditProfileSheet code in the previous code example +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/extensions.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +final profile = [ + (title: 'Name', value: 'Alexandru'), + (title: 'Username', value: 'nank1ro'), +]; + +class SheetPage extends StatefulWidget { + const SheetPage({super.key}); + + @override + State createState() => _SheetPageState(); +} + +class _SheetPageState extends State { + var side = ShadSheetSide.bottom; + var draggable = false; + var titlePinned = false; + var descriptionPinned = false; + var actionsPinned = true; + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return BaseScaffold( + appBarTitle: 'Sheet', + editable: [ + MyEnumProperty( + label: 'Side', + value: side, + values: ShadSheetSide.values, + onChanged: (value) { + if (value != null) { + setState(() { + side = value; + }); + } + }, + ), + MyBoolProperty( + label: 'Draggable', + value: draggable, + onChanged: (value) => setState(() => draggable = value), + ), + MyBoolProperty( + label: 'titlePinned', + value: titlePinned, + onChanged: (v) => setState(() => titlePinned = v), + ), + MyBoolProperty( + label: 'descriptionPinned', + value: descriptionPinned, + onChanged: (v) => setState(() => descriptionPinned = v), + ), + MyBoolProperty( + label: 'actionsPinned', + value: actionsPinned, + onChanged: (v) => setState(() => actionsPinned = v), + ), + ], + children: [ + ShadButton.outline( + child: const Text('Open'), + onPressed: () { + showShadSheet( + context: context, + side: side, + builder: (context) { + return ShadSheet( + draggable: draggable, + constraints: + side == ShadSheetSide.left || side == ShadSheetSide.right + ? const BoxConstraints(maxWidth: 512) + : null, + title: const Text('Edit Profile'), + description: const Text( + "Make changes to your profile here. Click save when you're done", + ), + actions: const [ShadButton(child: Text('Save changes'))], + titlePinned: titlePinned, + descriptionPinned: descriptionPinned, + actionsPinned: actionsPinned, + child: Padding( + padding: const EdgeInsets.symmetric(vertical: 20), + child: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + spacing: 16, + children: + (profile.map( + (p) => Row( + children: [ + Expanded( + child: Text( + p.title, + textAlign: TextAlign.end, + style: theme.textTheme.small, + ), + ), + const SizedBox(width: 16), + Expanded( + flex: 5, + child: ShadInput( + initialValue: p.value, + ), + ), + ], + ), + ) * + 20) + .toList(), + ), + ), + ); + }, + ); + }, + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/slider.md b/.claude/skills/shadcn-ui-flutter/components/slider.md new file mode 100644 index 0000000..73a9d8e --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/slider.md @@ -0,0 +1,81 @@ +# Slider + +An input where the user selects a value from within a given range. + + + +```dart +ShadSlider( + initialValue: 33, + max: 100, +), +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:example/common/properties/string_property.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter/widgets.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class SliderPage extends StatefulWidget { + const SliderPage({super.key}); + + @override + State createState() => _SliderPageState(); +} + +class _SliderPageState extends State { + var enabled = true; + double value = 33; + ShadSliderInteraction sliderInteraction = ShadSliderInteraction.tapAndSlide; + int? divisions; + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Slider', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyEnumProperty( + label: 'Interaction', + value: sliderInteraction, + values: ShadSliderInteraction.values, + onChanged: (value) => setState(() => sliderInteraction = value!), + ), + MyStringProperty( + label: 'Divisions', + initialValue: divisions?.toString() ?? '', + onChanged: (v) { + setState(() { + final parsed = int.tryParse(v); + divisions = parsed; + }); + }, + inputFormatters: [FilteringTextInputFormatter.digitsOnly], + ), + ], + children: [ + ShadSlider( + initialValue: 33, + max: 100, + enabled: enabled, + onChanged: print, + allowedInteraction: sliderInteraction, + semanticFormatterCallback: (double value) => + '${value.round()}% volume level', + divisions: divisions, + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/sonner.md b/.claude/skills/shadcn-ui-flutter/components/sonner.md new file mode 100644 index 0000000..422af70 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/sonner.md @@ -0,0 +1,76 @@ +# Sonner + +An opinionated toast component. + + + +```dart +ShadButton.outline( + child: const Text('Show Toast'), + onPressed: () { + final sonner = ShadSonner.of(context); + final id = Random().nextInt(1000); + final now = DateTime.now(); + sonner.show( + ShadToast( + id: id, + title: const Text('Event has been created'), + description: Text(DateFormat.yMd().add_jms().format(now)), + action: ShadButton( + child: const Text('Undo'), + onPressed: () => sonner.hide(id), + ), + ), + ); + }, +), +``` + +## Example +```dart +import 'dart:math'; + +import 'package:example/common/base_scaffold.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/widgets.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class SonnerPage extends StatefulWidget { + const SonnerPage({super.key}); + + @override + State createState() => _SonnerPageState(); +} + +class _SonnerPageState extends State + with SingleTickerProviderStateMixin { + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Sonner', + children: [ + ShadButton.outline( + child: const Text('Show Toast'), + onPressed: () { + final sonner = ShadSonner.of(context); + final id = Random().nextInt(1000); + final now = DateTime.now(); + sonner.show( + ShadToast( + id: id, + title: const Text('Event has been created'), + description: Text(DateFormat.yMd().add_jms().format(now)), + action: ShadButton( + child: const Text('Undo'), + onPressed: () => sonner.hide(id), + ), + ), + ); + }, + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/switch.md b/.claude/skills/shadcn-ui-flutter/components/switch.md new file mode 100644 index 0000000..ba60c23 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/switch.md @@ -0,0 +1,240 @@ +# Switch + +A control that allows the user to toggle between checked and not checked. + + + +```dart +class SwitchExample extends StatefulWidget { + const SwitchExample({super.key}); + + @override + State createState() => _SwitchExampleState(); +} + +class _SwitchExampleState extends State { + bool value = false; + + @override + Widget build(BuildContext context) { + return ShadSwitch( + value: value, + onChanged: (v) => setState(() => value = v), + label: const Text('Airplane Mode'), + ); + } +} +``` + + + +## Form + + + +```dart +ShadSwitchFormField( + id: 'terms', + initialValue: false, + inputLabel: + const Text('I accept the terms and conditions'), + onChanged: (v) {}, + inputSublabel: + const Text('You agree to our Terms and Conditions'), + validator: (v) { + if (!v) { + return 'You must accept the terms and conditions'; + } + return null; + }, +) +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/widgets.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class SwitchPage extends StatefulWidget { + const SwitchPage({super.key}); + + @override + State createState() => _SwitchPageState(); +} + +class _SwitchPageState extends State { + bool value = false; + bool enabled = true; + final focusNode = FocusNode(); + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Switch', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyBoolProperty( + label: 'Focused', + value: focusNode.hasFocus, + onChanged: enabled + ? (value) { + setState(() { + if (value) { + focusNode.requestFocus(); + } else { + focusNode.unfocus(); + } + }); + } + : null, + ), + ], + children: [ + ShadSwitch( + value: value, + focusNode: focusNode, + enabled: enabled, + onChanged: (v) { + setState(() => value = v); + }, + label: const Text('Airplane Mode'), + ), + ], + ); + } +} + +``` + +## Form Example +```dart +// ignore_for_file: avoid_print + +import 'dart:convert'; + +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class SwitchFormFieldPage extends StatefulWidget { + const SwitchFormFieldPage({super.key}); + + @override + State createState() => _SwitchFormFieldPageState(); +} + +class _SwitchFormFieldPageState extends State { + bool enabled = true; + var autovalidateMode = ShadAutovalidateMode.alwaysAfterFirstValidation; + bool initialValue = false; + Map formValue = {}; + final formKey = GlobalKey(); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ShadForm( + key: formKey, + enabled: enabled, + autovalidateMode: autovalidateMode, + initialValue: {'terms': initialValue}, + child: BaseScaffold( + appBarTitle: 'SwitchFormField', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyEnumProperty( + label: 'autovalidateMode', + value: autovalidateMode, + values: ShadAutovalidateMode.values, + onChanged: (value) { + if (value != null) { + setState(() => autovalidateMode = value); + } + }, + ), + MyBoolProperty( + label: 'Form Initial Value', + value: initialValue, + onChanged: (value) { + formKey.currentState!.setFieldValue('terms', value); + setState(() { + initialValue = value; + }); + }, + ), + ], + children: [ + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 350), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + ShadSwitchFormField( + id: 'terms', + initialValue: initialValue, + inputLabel: const Text('I accept the terms and conditions'), + onChanged: (v) {}, + inputSublabel: const Text( + 'You agree to our Terms and Conditions', + ), + validator: (v) { + if (!v) { + return 'You must accept the terms and conditions'; + } + return null; + }, + ), + const SizedBox(height: 16), + ShadButton( + child: const Text('Submit'), + onPressed: () { + print('submitted'); + if (formKey.currentState!.saveAndValidate()) { + setState(() { + formValue = formKey.currentState!.value; + }); + } else { + print('validation failed'); + } + }, + ), + if (formValue.isNotEmpty) + Padding( + padding: const EdgeInsets.only(top: 24, left: 12), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('FormValue', style: theme.textTheme.p), + const SizedBox(height: 4), + SelectableText( + const JsonEncoder.withIndent( + ' ', + ).convert(formValue), + style: theme.textTheme.small, + ), + ], + ), + ), + ], + ), + ), + ], + ), + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/table.md b/.claude/skills/shadcn-ui-flutter/components/table.md new file mode 100644 index 0000000..1117211 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/table.md @@ -0,0 +1,490 @@ +# Table + +A responsive table component. + +## List + +Use the `ShadTable.list` widget to create a table from a two dimensional array of children. +Use it just for **small** tables, because every child will be created. + + + +```dart + + + +const invoices = [ + ( + invoice: "INV001", + paymentStatus: "Paid", + totalAmount: r"$250.00", + paymentMethod: "Credit Card", + ), + ( + invoice: "INV002", + paymentStatus: "Pending", + totalAmount: r"$150.00", + paymentMethod: "PayPal", + ), + ( + invoice: "INV003", + paymentStatus: "Unpaid", + totalAmount: r"$350.00", + paymentMethod: "Bank Transfer", + ), + ( + invoice: "INV004", + paymentStatus: "Paid", + totalAmount: r"$450.00", + paymentMethod: "Credit Card", + ), + ( + invoice: "INV005", + paymentStatus: "Paid", + totalAmount: r"$550.00", + paymentMethod: "PayPal", + ), + ( + invoice: "INV006", + paymentStatus: "Pending", + totalAmount: r"$200.00", + paymentMethod: "Bank Transfer", + ), + ( + invoice: "INV007", + paymentStatus: "Unpaid", + totalAmount: r"$300.00", + paymentMethod: "Credit Card", + ), +]; + +class TablePage extends StatelessWidget { + const TablePage({ + super.key, + }); + + @override + Widget build(BuildContext context) { + return Scaffold( + body: Center( + child: ConstrainedBox( + constraints: const BoxConstraints( + maxWidth: 600, + // added just to center the table vertically + maxHeight: 450, + ), + child: ShadTable.list( + header: const [ + ShadTableCell.header(child: Text('Invoice')), + ShadTableCell.header(child: Text('Status')), + ShadTableCell.header(child: Text('Method')), + ShadTableCell.header( + alignment: Alignment.centerRight, + child: Text('Amount'), + ), + ], + footer: const [ + ShadTableCell.footer(child: Text('Total')), + ShadTableCell.footer(child: Text('')), + ShadTableCell.footer(child: Text('')), + ShadTableCell.footer( + alignment: Alignment.centerRight, + child: Text(r'$2500.00'), + ), + ], + columnSpanExtent: (index) { + if (index == 2) return const FixedTableSpanExtent(130); + if (index == 3) { + return const MaxTableSpanExtent( + FixedTableSpanExtent(120), + RemainingTableSpanExtent(), + ); + } + // uses the default value + return null; + }, + children: invoices + .map( + (invoice) => [ + ShadTableCell( + child: Text( + invoice.invoice, + style: const TextStyle( + fontWeight: FontWeight.w500, + ), + ), + ), + ShadTableCell(child: Text(invoice.paymentStatus)), + ShadTableCell(child: Text(invoice.paymentMethod)), + ShadTableCell( + alignment: Alignment.centerRight, + child: Text( + invoice.totalAmount, + ), + ), + ], + ), + ), + ), + ), + ); + } +} +``` + + + +## Builder + +You can also use a builder to create the table. +This method is preferred for **large** tables because widgets are created on demand. +Here it is the same table as above, but using a builder. + +```dart +const invoices = [ + [ + "INV001", + "Paid", + "Credit Card", + r"$250.00", + ], + [ + "INV002", + "Pending", + "PayPal", + r"$150.00", + ], + [ + "INV003", + "Unpaid", + "Bank Transfer", + r"$350.00", + ], + [ + "INV004", + "Paid", + "Credit Card", + r"$450.00", + ], + [ + "INV005", + "Paid", + "PayPal", + r"$550.00", + ], + [ + "INV006", + "Pending", + "Bank Transfer", + r"$200.00", + ], + [ + "INV007", + "Unpaid", + "Credit Card", + r"$300.00", + ], +]; + +final headings = [ + 'Invoice', + 'Status', + 'Method', + 'Amount', +]; + +class TableExample extends StatelessWidget { + const TableExample({super.key}); + + @override + Widget build(BuildContext context) { + return ShadTable( + columnCount: invoices[0].length, + rowCount: invoices.length, + header: (context, column) { + final isLast = column == headings.length - 1; + return ShadTableCell.header( + alignment: isLast ? Alignment.centerRight : null, + child: Text(headings[column]), + ); + }, + columnSpanExtent: (index) { + if (index == 2) return const FixedTableSpanExtent(150); + if (index == 3) { + return const MaxTableSpanExtent( + FixedTableSpanExtent(120), + RemainingTableSpanExtent(), + ); + } + return null; + }, + builder: (context, index) { + final invoice = invoices[index.row]; + return ShadTableCell( + alignment: index.column == invoice.length - 1 + ? Alignment.centerRight + : Alignment.centerLeft, + child: Text( + invoice[index.column], + style: index.column == 0 + ? const TextStyle(fontWeight: FontWeight.w500) + : null, + ), + ); + }, + footer: (context, column) { + if (column == 0) { + return const ShadTableCell.footer( + child: Text( + 'Total', + style: TextStyle(fontWeight: FontWeight.w500), + ), + ); + } + if (column == 3) { + return const ShadTableCell.footer( + alignment: Alignment.centerRight, + child: Text( + r'$2500.00', + ), + ); + } + return const ShadTableCell(child: SizedBox()); + }, + ); + } +} +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +const invoices = [ + ( + invoice: "INV001", + paymentStatus: "Paid", + totalAmount: r"$250.00", + paymentMethod: "Credit Card", + ), + ( + invoice: "INV002", + paymentStatus: "Pending", + totalAmount: r"$150.00", + paymentMethod: "PayPal", + ), + ( + invoice: "INV003", + paymentStatus: "Unpaid", + totalAmount: r"$350.00", + paymentMethod: "Bank Transfer", + ), + ( + invoice: "INV004", + paymentStatus: "Paid", + totalAmount: r"$450.00", + paymentMethod: "Credit Card", + ), + ( + invoice: "INV005", + paymentStatus: "Paid", + totalAmount: r"$550.00", + paymentMethod: "PayPal", + ), + ( + invoice: "INV006", + paymentStatus: "Pending", + totalAmount: r"$200.00", + paymentMethod: "Bank Transfer", + ), + ( + invoice: "INV007", + paymentStatus: "Unpaid", + totalAmount: r"$300.00", + paymentMethod: "Credit Card", + ), +]; + +class TablePage extends StatelessWidget { + const TablePage({super.key}); + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Table', + wrapChildrenInScrollable: false, + wrapSingleChildInColumn: false, + children: [ + Center( + child: ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: ShadTable.list( + header: const [ + ShadTableCell.header(child: Text('Invoice')), + ShadTableCell.header(child: Text('Status')), + ShadTableCell.header(child: Text('Method')), + ShadTableCell.header( + alignment: Alignment.centerRight, + child: Text('Amount'), + ), + ], + footer: const [ + ShadTableCell.footer(child: Text('Total')), + ShadTableCell.footer(child: Text('')), + ShadTableCell.footer(child: Text('')), + ShadTableCell.footer( + alignment: Alignment.centerRight, + child: Text(r'$2500.00'), + ), + ], + columnSpanExtent: (index) { + if (index == 2) return const FixedTableSpanExtent(130); + if (index == 3) { + return const MaxTableSpanExtent( + FixedTableSpanExtent(120), + RemainingTableSpanExtent(), + ); + } + // uses the default value + return null; + }, + children: invoices.map( + (invoice) => [ + ShadTableCell( + child: Text( + invoice.invoice, + style: const TextStyle( + fontWeight: FontWeight.w500, + ), + ), + ), + ShadTableCell(child: Text(invoice.paymentStatus)), + ShadTableCell(child: Text(invoice.paymentMethod)), + ShadTableCell( + alignment: Alignment.centerRight, + child: Text( + invoice.totalAmount, + ), + ), + ], + ), + ), + ), + ), + ], + ); + } +} + +/* With builder it will be +const invoices = [ + [ + "INV001", + "Paid", + "Credit Card", + r"$250.00", + ], + [ + "INV002", + "Pending", + "PayPal", + r"$150.00", + ], + [ + "INV003", + "Unpaid", + "Bank Transfer", + r"$350.00", + ], + [ + "INV004", + "Paid", + "Credit Card", + r"$450.00", + ], + [ + "INV005", + "Paid", + "PayPal", + r"$550.00", + ], + [ + "INV006", + "Pending", + "Bank Transfer", + r"$200.00", + ], + [ + "INV007", + "Unpaid", + "Credit Card", + r"$300.00", + ], +]; + +final headings = [ + 'Invoice', + 'Status', + 'Method', + 'Amount', +]; + +ShadTable( + columnCount: invoices[0].length, + rowCount: invoices.length, + header: (context, column) { + final isLast = column == headings.length - 1; + return ShadTableCell.header( + alignment: isLast ? Alignment.centerRight : null, + child: Text(headings[column]), + ); + }, + columnSpanExtent: (index) { + if (index == 2) return const FixedTableSpanExtent(150); + if (index == 3) { + return const MaxTableSpanExtent( + FixedTableSpanExtent(120), + RemainingTableSpanExtent(), + ); + } + return null; + }, + builder: (context, index) { + final invoice = invoices[index.row]; + return ShadTableCell( + alignment: index.column == invoice.length - 1 + ? Alignment.centerRight + : Alignment.centerLeft, + child: Text( + invoice[index.column], + style: index.column == 0 + ? const TextStyle(fontWeight: FontWeight.w500) + : null, + ), + ); + }, + footer: (context, column) { + if (column == 0) { + return const ShadTableCell.footer( + child: Text( + 'Total', + style: TextStyle(fontWeight: FontWeight.w500), + ), + ); + } + if (column == 3) { + return const ShadTableCell.footer( + alignment: Alignment.centerRight, + child: Text( + r'$2500.00', + ), + ); + } + return const ShadTableCell(child: SizedBox()); + }, +) +*/ + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/tabs.md b/.claude/skills/shadcn-ui-flutter/components/tabs.md new file mode 100644 index 0000000..68f6e39 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/tabs.md @@ -0,0 +1,158 @@ +# Tabs + +A set of layered sections of content—known as tab panels—that are displayed one at a time. + + + +```dart +class TabsExample extends StatelessWidget { + const TabsExample({super.key}); + + @override + Widget build(BuildContext context) { + return ShadTabs( + value: 'account', + tabBarConstraints: const BoxConstraints(maxWidth: 400), + contentConstraints: const BoxConstraints(maxWidth: 400), + tabs: [ + ShadTab( + value: 'account', + content: ShadCard( + title: const Text('Account'), + description: const Text( + "Make changes to your account here. Click save when you're done."), + footer: const ShadButton(child: Text('Save changes')), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + const SizedBox(height: 16), + ShadInputFormField( + label: const Text('Name'), + initialValue: 'Ale', + ), + const SizedBox(height: 8), + ShadInputFormField( + label: const Text('Username'), + initialValue: 'nank1ro', + ), + const SizedBox(height: 16), + ], + ), + ), + child: const Text('Account'), + ), + ShadTab( + value: 'password', + content: ShadCard( + title: const Text('Password'), + description: const Text( + "Change your password here. After saving, you'll be logged out."), + footer: const ShadButton(child: Text('Save password')), + child: Column( + children: [ + const SizedBox(height: 16), + ShadInputFormField( + label: const Text('Current password'), + obscureText: true, + ), + const SizedBox(height: 8), + ShadInputFormField( + label: const Text('New password'), + obscureText: true, + ), + const SizedBox(height: 16), + ], + ), + ), + child: const Text('Password'), + ), + ], + ); + } +} +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class TabsPage extends StatelessWidget { + const TabsPage({super.key}); + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: "Tabs", + wrapChildrenInScrollable: false, + wrapSingleChildInColumn: false, + alignment: Alignment.topCenter, + children: [ + ShadTabs( + value: 'account', + tabBarConstraints: const BoxConstraints(maxWidth: 400), + contentConstraints: const BoxConstraints(maxWidth: 400), + onChanged: (value) => print(value), + tabs: [ + ShadTab( + value: 'account', + content: ShadCard( + title: const Text('Account'), + description: const Text( + "Make changes to your account here. Click save when you're done.", + ), + footer: const ShadButton(child: Text('Save changes')), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + const SizedBox(height: 16), + ShadInputFormField( + label: const Text('Name'), + initialValue: 'Ale', + ), + const SizedBox(height: 8), + ShadInputFormField( + label: const Text('Username'), + initialValue: 'nank1ro', + ), + const SizedBox(height: 16), + ], + ), + ), + child: const Text('Account'), + ), + ShadTab( + value: 'password', + content: ShadCard( + title: const Text('Password'), + description: const Text( + "Change your password here. After saving, you'll be logged out.", + ), + footer: const ShadButton(child: Text('Save password')), + child: Column( + children: [ + const SizedBox(height: 16), + ShadInputFormField( + label: const Text('Current password'), + obscureText: true, + ), + const SizedBox(height: 8), + ShadInputFormField( + label: const Text('New password'), + obscureText: true, + ), + const SizedBox(height: 16), + ], + ), + ), + child: const Text('Password'), + ), + ], + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/textarea.md b/.claude/skills/shadcn-ui-flutter/components/textarea.md new file mode 100644 index 0000000..11b8120 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/textarea.md @@ -0,0 +1,216 @@ +# Textarea + +Displays a form textarea or a component that looks like a textarea. + + + +```dart +ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 400), + child: const ShadTextarea( + placeholder: Text('Type your message here'), + ), +), +``` + + + +## Form + + + +```dart +ShadTextareaFormField( + id: 'bio', + label: const Text('Bio'), + placeholder: + const Text('Tell us a little bit about yourself'), + description: const Text( + 'You can @mention other users and organizations.'), + validator: (v) { + if (v.length < 10) { + return 'Bio must be at least 10 characters.'; + } + return null; + }, +) +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class TextareaPage extends StatefulWidget { + const TextareaPage({super.key}); + + @override + State createState() => _TextareaPageState(); +} + +class _TextareaPageState extends State { + bool enabled = true; + bool resizable = true; + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Textarea', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (v) => setState(() => enabled = v), + ), + MyBoolProperty( + label: 'Resizable', + value: resizable, + onChanged: (v) => setState(() => resizable = v), + ), + ], + children: [ + ShadTextarea( + placeholder: const Text('Type your message here...'), + enabled: enabled, + resizable: resizable, + onChanged: (v) => print('Value changed: $v'), + ), + ], + ); + } +} + +``` + +## Form Example +```dart +// ignore_for_file: avoid_print + +import 'dart:convert'; + +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:example/common/properties/string_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class TextareaFormFieldPage extends StatefulWidget { + const TextareaFormFieldPage({super.key}); + + @override + State createState() => _TextareaFormFieldPageState(); +} + +class _TextareaFormFieldPageState extends State { + bool enabled = true; + var autovalidateMode = ShadAutovalidateMode.alwaysAfterFirstValidation; + String? initialValue; + Map formValue = {}; + final formKey = GlobalKey(); + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + + return ShadForm( + key: formKey, + enabled: enabled, + autovalidateMode: autovalidateMode, + initialValue: {if (initialValue != null) 'bio': initialValue}, + child: BaseScaffold( + appBarTitle: 'TextareaFormField', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyEnumProperty( + label: 'autovalidateMode', + value: autovalidateMode, + values: ShadAutovalidateMode.values, + onChanged: (value) { + if (value != null) { + setState(() => autovalidateMode = value); + } + }, + ), + MyStringProperty( + label: 'Form Initial Value', + initialValue: initialValue, + placeholder: const Text('Enter your bio...'), + onChanged: (value) { + formKey.currentState!.setFieldValue('bio', value); + }, + ), + ], + children: [ + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 500), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + ShadTextareaFormField( + id: 'bio', + label: const Text('Bio'), + placeholder: const Text('Tell us about yourself...'), + minHeight: 100, + maxHeight: 250, + validator: (v) { + if (v.trim().isEmpty) return 'Bio cannot be empty.'; + if (v.length < 10) { + return 'Bio must be at least 10 characters.'; + } + return null; + }, + ), + const SizedBox(height: 16), + ShadButton( + child: const Text('Submit'), + onPressed: () { + if (formKey.currentState!.saveAndValidate()) { + ShadToaster.of(context).show( + ShadToast(title: Text('Form submitted successfully')), + ); + setState(() { + formValue = formKey.currentState!.value; + }); + } else { + ShadToaster.of(context).show( + ShadToast.destructive( + title: Text('Please correct the errors in the form'), + ), + ); + } + }, + ), + if (formValue.isNotEmpty) + Padding( + padding: const EdgeInsets.only(top: 24, left: 12), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('FormValue', style: theme.textTheme.p), + const SizedBox(height: 4), + SelectableText( + const JsonEncoder.withIndent( + ' ', + ).convert(formValue), + style: theme.textTheme.small, + ), + ], + ), + ), + ], + ), + ), + ], + ), + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/time-picker.md b/.claude/skills/shadcn-ui-flutter/components/time-picker.md new file mode 100644 index 0000000..438bd32 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/time-picker.md @@ -0,0 +1,263 @@ +# Time Picker + +A time picker component. + + + +```dart +class PrimaryTimePicker extends StatelessWidget { + const PrimaryTimePicker({super.key}); + + @override + Widget build(BuildContext context) { + return ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: const ShadTimePicker( + trailing: Padding( + padding: EdgeInsets.only(left: 8, top: 14), + child: Icon(LucideIcons.clock4), + ), + ), + ); + } +} +``` + + + +## Form + + + +```dart +ShadTimePickerFormField( + label: const Text('Pick a time'), + onChanged: print, + description: + const Text('The time of the day you want to pick'), + validator: (v) => v == null ? 'A time is required' : null, +) +``` + + + +## ShadTimePickerFormField.period + + + +```dart +ShadTimePickerFormField.period( + label: const Text('Pick a time'), + onChanged: print, + description: + const Text('The time of the day you want to pick'), + validator: (v) => v == null ? 'A time is required' : null, +), +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class TimePickerPage extends StatefulWidget { + const TimePickerPage({super.key}); + + @override + State createState() => _TimePickerPageState(); +} + +class _TimePickerPageState extends State { + bool showHours = true; + bool showMinutes = true; + bool showSeconds = true; + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'TimePicker', + editable: [ + MyBoolProperty( + label: ' Show Hours', + value: showHours, + enabled: showMinutes || showSeconds, + onChanged: (v) => setState(() => showHours = v), + ), + MyBoolProperty( + label: ' Show Minutes', + value: showMinutes, + enabled: showHours || showSeconds, + onChanged: (v) => setState(() => showMinutes = v), + ), + MyBoolProperty( + label: ' Show Seconds', + enabled: showHours || showMinutes, + value: showSeconds, + onChanged: (v) => setState(() => showSeconds = v), + ), + ], + children: [ + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: ShadTimePicker( + showHours: showHours, + showMinutes: showMinutes, + showSeconds: showSeconds, + trailing: const Padding( + padding: EdgeInsets.only(left: 8, top: 14), + child: Icon(LucideIcons.clock4), + ), + onChanged: (time) { + print('time: $time'); + }, + ), + ), + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 600), + child: ShadTimePicker.period( + showHours: showHours, + showMinutes: showMinutes, + showSeconds: showSeconds, + crossAxisAlignment: WrapCrossAlignment.end, + onChanged: (time) { + print('time: $time'); + }, + ), + ), + ], + ); + } +} + +``` + +## Form Example +```dart +// ignore_for_file: avoid_print + +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/bool_property.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:flutter/material.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class TimePickerFormFieldPage extends StatefulWidget { + const TimePickerFormFieldPage({super.key}); + + @override + State createState() => + _TimePickerFormFieldPageState(); +} + +class _TimePickerFormFieldPageState extends State { + bool enabled = true; + var autovalidateMode = ShadAutovalidateMode.alwaysAfterFirstValidation; + Map formValue = {}; + final formKey = GlobalKey(); + bool showHours = true; + bool showMinutes = true; + bool showSeconds = true; + + @override + Widget build(BuildContext context) { + final theme = ShadTheme.of(context); + return ShadForm( + key: formKey, + enabled: enabled, + autovalidateMode: autovalidateMode, + child: BaseScaffold( + appBarTitle: 'TimePickerFormField', + editable: [ + MyBoolProperty( + label: 'Enabled', + value: enabled, + onChanged: (value) => setState(() => enabled = value), + ), + MyEnumProperty( + label: 'autovalidateMode', + value: autovalidateMode, + values: ShadAutovalidateMode.values, + onChanged: (value) { + if (value != null) { + setState(() => autovalidateMode = value); + } + }, + ), + MyBoolProperty( + label: ' Show Hours', + value: showHours, + enabled: showMinutes || showSeconds, + onChanged: (v) => setState(() => showHours = v), + ), + MyBoolProperty( + label: ' Show Minutes', + value: showMinutes, + enabled: showHours || showSeconds, + onChanged: (v) => setState(() => showMinutes = v), + ), + MyBoolProperty( + label: ' Show Seconds', + enabled: showHours || showMinutes, + value: showSeconds, + onChanged: (v) => setState(() => showSeconds = v), + ), + ], + children: [ + ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 350), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + ShadTimePickerFormField( + id: 'time', + showHours: showHours, + showMinutes: showMinutes, + showSeconds: showSeconds, + label: const Text('Pick a time'), + onChanged: print, + description: const Text( + 'The time of the day you want to pick', + ), + validator: (v) => v == null ? 'A time is required' : null, + ), + const SizedBox(height: 16), + ShadButton( + child: const Text('Submit'), + onPressed: () { + print('submitted'); + if (formKey.currentState!.saveAndValidate()) { + setState(() { + formValue = formKey.currentState!.value; + }); + } else { + print('validation failed'); + } + }, + ), + if (formValue.isNotEmpty) + Padding( + padding: const EdgeInsets.only(top: 24, left: 12), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('FormValue', style: theme.textTheme.p), + const SizedBox(height: 4), + SelectableText( + formValue.toString(), + style: theme.textTheme.small, + ), + ], + ), + ), + ], + ), + ), + ], + ), + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/toast.md b/.claude/skills/shadcn-ui-flutter/components/toast.md new file mode 100644 index 0000000..9aea2ab --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/toast.md @@ -0,0 +1,208 @@ +# Toast + +A succinct message that is displayed temporarily. + + + +```dart +ShadButton.outline( + child: const Text('Add to calendar'), + onPressed: () { + ShadToaster.of(context).show( + ShadToast( + title: const Text('Scheduled: Catch up'), + description: + const Text('Friday, February 10, 2023 at 5:57 PM'), + action: ShadButton.outline( + child: const Text('Undo'), + onPressed: () => ShadToaster.of(context).hide(), + ), + ), + ); + }, +), +``` + + + +## Simple + + + +```dart +ShadButton.outline( + child: const Text('Show Toast'), + onPressed: () { + ShadToaster.of(context).show( + const ShadToast( + description: Text('Your message has been sent.'), + ), + ); + }, +), +``` + + + +## With Title + + + +```dart +ShadButton.outline( + child: const Text('Show Toast'), + onPressed: () { + ShadToaster.of(context).show( + const ShadToast( + title: Text('Uh oh! Something went wrong'), + description: + Text('There was a problem with your request'), + ), + ); + }, +), +``` + + + +## With Action + + + +```dart +ShadButton.outline( + child: const Text('Show Toast'), + onPressed: () { + ShadToaster.of(context).show( + ShadToast( + title: const Text('Uh oh! Something went wrong'), + description: + const Text('There was a problem with your request'), + action: ShadButton.outline( + child: const Text('Try again'), + onPressed: () => ShadToaster.of(context).hide(), + ), + ), + ); + }, +), +``` + + + +## Destructive + + + +```dart +final theme = ShadTheme.of(context); + +ShadButton.outline( + child: const Text('Show Toast'), + onPressed: () { + ShadToaster.of(context).show( + ShadToast.destructive( + title: const Text('Uh oh! Something went wrong'), + description: + const Text('There was a problem with your request'), + action: ShadButton.destructive( + child: const Text('Try again'), + decoration: ShadDecoration( + border: ShadBorder.all( + color: theme.colorScheme.destructiveForeground, + width: 1, + ), + ), + onPressed: () => ShadToaster.of(context).hide(), + ), + ), + ); + }, +), +``` + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:example/common/properties/enum_property.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/widgets.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +enum Alignm { + topRight, + topLeft, + bottomRight, + bottomLeft, + centerRight, + centerLeft, + center, + topCenter, + bottomCenter; + + Alignment toAlignment() { + return switch (this) { + topRight => Alignment.topRight, + topLeft => Alignment.topLeft, + bottomRight => Alignment.bottomRight, + bottomLeft => Alignment.bottomLeft, + centerRight => Alignment.centerRight, + centerLeft => Alignment.centerLeft, + center => Alignment.center, + topCenter => Alignment.topCenter, + bottomCenter => Alignment.bottomCenter, + }; + } +} + +class ToastPage extends StatefulWidget { + const ToastPage({super.key}); + + @override + State createState() => _ToastPageState(); +} + +class _ToastPageState extends State { + var alignment = Alignm.bottomRight; + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Toast', + editable: [ + MyEnumProperty( + label: 'Alignment', + value: alignment, + values: Alignm.values, + onChanged: (v) { + if (v != null) { + setState(() { + alignment = v; + }); + } + }, + ), + ], + children: [ + ShadButton.outline( + child: const Text('Add to calendar'), + onPressed: () { + final toaster = ShadToaster.of(context); + toaster.show( + ShadToast( + alignment: alignment.toAlignment(), + title: const Text('Scheduled: Catch up'), + description: const Text('Friday, February 10, 2023 at 5:57 PM'), + action: ShadButton.outline( + child: const Text('Undo'), + onPressed: () => toaster.hide(), + ), + ), + ); + }, + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/components/tooltip.md b/.claude/skills/shadcn-ui-flutter/components/tooltip.md new file mode 100644 index 0000000..020eb0a --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/components/tooltip.md @@ -0,0 +1,60 @@ +# Tooltip + +A popup that displays information related to an element when the element receives keyboard focus or the mouse hovers over it. + + +```dart +ShadTooltip( + builder: (context) => const Text('Add to library'), + child: ShadButton.outline( + child: const Text('Hover/Focus'), + onPressed: () {}, + ), +), +``` + + +The tooltip works on hover only if the child uses a `ShadGestureDetector`. If you don't use a `ShadButton` or something similar that implements `ShadGestureDetector` hover will not work. +If, for example, you want to just show an image as child, wrap it with `ShadGestureDetector` to make it working. + +## Example +```dart +import 'package:example/common/base_scaffold.dart'; +import 'package:flutter/widgets.dart'; +import 'package:shadcn_ui/shadcn_ui.dart'; + +class TooltipPage extends StatefulWidget { + const TooltipPage({super.key}); + + @override + State createState() => _TooltipPageState(); +} + +class _TooltipPageState extends State { + final focusNode = FocusNode(); + + @override + void dispose() { + focusNode.dispose(); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + return BaseScaffold( + appBarTitle: 'Tooltip', + children: [ + ShadTooltip( + focusNode: focusNode, + builder: (context) => const Text('Add to library'), + child: ShadButton.outline( + focusNode: focusNode, + child: const Text('Hover/Focus'), + ), + ), + ], + ); + } +} + +``` diff --git a/.claude/skills/shadcn-ui-flutter/guides/decorator.md b/.claude/skills/shadcn-ui-flutter/guides/decorator.md new file mode 100644 index 0000000..ca922b8 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/guides/decorator.md @@ -0,0 +1,49 @@ +Decorates most of the components of the library using a `ShadDecoration` handled by the `ShadDecorator` component. + +## Default + +```dart +ShadDecoration( + secondaryBorder: ShadBorder.all( + padding: const EdgeInsets.all(4), + width: 0, + ), + secondaryFocusedBorder: ShadBorder.all( + width: 2, + color: colorScheme.ring, + radius: radius.add(radius / 2), + padding: const EdgeInsets.all(2), + ), + labelStyle: textTheme.muted.copyWith( + fontWeight: FontWeight.w500, + color: colorScheme.foreground, + ), + errorStyle: textTheme.muted.copyWith( + fontWeight: FontWeight.w500, + color: colorScheme.destructive, + ), + labelPadding: const EdgeInsets.only(bottom: 8), + descriptionStyle: textTheme.muted, + descriptionPadding: const EdgeInsets.only(top: 8), + errorPadding: const EdgeInsets.only(top: 8), + errorLabelStyle: textTheme.muted.copyWith( + fontWeight: FontWeight.w500, + color: colorScheme.destructive, + ), +); +``` + +## Secondary Border + +By default, a secondary border is drawn around the focusable components. +If you want to disable it and instead make bolder the primary border, you just need to add the `disableSecondaryBorder` property to the theme. + +```dart +ShadThemeData( + // Disables the secondary border + disableSecondaryBorder: true, +), +``` + +Be aware, this change is not recommended, as it may lead to accessibility issues. +The secondary border is there to help users understand which component is focused. \ No newline at end of file diff --git a/.claude/skills/shadcn-ui-flutter/guides/interop.md b/.claude/skills/shadcn-ui-flutter/guides/interop.md new file mode 100644 index 0000000..43798fe --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/guides/interop.md @@ -0,0 +1,161 @@ +# Interoperability + +## Shadcn + Material + +We are the first Flutter UI library to allow shadcn components to be used simultaneously with Material components. +The setup is simple: + +```diff lang="dart" +import 'package:shadcn_ui/shadcn_ui.dart'; ++ import 'package:flutter/material.dart'; + +void main() { + runApp(const MyApp()); +} + +class MyApp extends StatelessWidget { + const MyApp({super.key}); + + @override + Widget build(BuildContext context) { +- return ShadApp(); ++ return ShadApp.custom( ++ themeMode: ThemeMode.dark, ++ darkTheme: ShadThemeData( ++ brightness: Brightness.dark, ++ colorScheme: const ShadSlateColorScheme.dark(), ++ ), ++ appBuilder: (context) { ++ return MaterialApp( ++ theme: Theme.of(context), ++ localizationsDelegates: const [ ++ GlobalShadLocalizations.delegate, ++ GlobalMaterialLocalizations.delegate, ++ GlobalCupertinoLocalizations.delegate, ++ GlobalWidgetsLocalizations.delegate, ++ ], ++ builder: (context, child) { ++ return ShadAppBuilder(child: child!); ++ }, ++ ); ++ }, ++ ); + } +``` + +:::tip +If you need to use the `Router` instead of the `Navigator`, use `MaterialApp.router`. +::: + +--- + +The default Material `ThemeData` created by `ShadApp` is: + +```dart +ThemeData( + fontFamily: themeData.textTheme.family, + extensions: themeData.extensions, + colorScheme: ColorScheme( + brightness: themeData.brightness, + primary: themeData.colorScheme.primary, + onPrimary: themeData.colorScheme.primaryForeground, + secondary: themeData.colorScheme.secondary, + onSecondary: themeData.colorScheme.secondaryForeground, + error: themeData.colorScheme.destructive, + onError: themeData.colorScheme.destructiveForeground, + surface: themeData.colorScheme.background, + onSurface: themeData.colorScheme.foreground, + ), + scaffoldBackgroundColor: themeData.colorScheme.background, + brightness: themeData.brightness, + dividerTheme: DividerThemeData( + color: themeData.colorScheme.border, + thickness: 1, + ), + textSelectionTheme: TextSelectionThemeData( + cursorColor: themeData.colorScheme.primary, + selectionColor: themeData.colorScheme.selection, + selectionHandleColor: themeData.colorScheme.primary, + ), + iconTheme: IconThemeData( + size: 16, + color: themeData.colorScheme.foreground, + ), + scrollbarTheme: ScrollbarThemeData( + crossAxisMargin: 1, + mainAxisMargin: 1, + thickness: const WidgetStatePropertyAll(8), + radius: const Radius.circular(999), + thumbColor: WidgetStatePropertyAll(themeData.colorScheme.border), + ), +), +``` + +:::note +Use `Theme.of(context).copyWith(...)` to override the default theme, without losing the default values provided by shadcn_ui. +::: + +## Shadcn + Cupertino + +If you need to use shadcn components with Cupertino components, use `CupertinoApp` instead of `MaterialApp`, like you are already used to. + +```diff lang="dart" +import 'package:shadcn_ui/shadcn_ui.dart'; ++ import 'package:flutter/cupertino.dart'; ++ import 'package:flutter_localizations/flutter_localizations.dart'; + +void main() { + runApp(const MyApp()); +} + +class MyApp extends StatelessWidget { + const MyApp({super.key}); + + @override + Widget build(BuildContext context) { +- return ShadApp(); ++ return ShadApp.custom( ++ themeMode: ThemeMode.dark, ++ darkTheme: ShadThemeData( ++ brightness: Brightness.dark, ++ colorScheme: const ShadSlateColorScheme.dark(), ++ ), ++ appBuilder: (context) { ++ return CupertinoApp( ++ theme: CupertinoTheme.of(context), ++ localizationsDelegates: const [ ++ GlobalShadLocalizations.delegate, ++ DefaultMaterialLocalizations.delegate, ++ DefaultCupertinoLocalizations.delegate, ++ DefaultWidgetsLocalizations.delegate, ++ ], ++ builder: (context, child) { ++ return ShadAppBuilder(child: child!); ++ }, ++ ); ++ }, ++ ); + } +``` + +:::tip +If you need to use the `Router` instead of the `Navigator`, use `CupertinoApp.router`. +::: + +--- + +The default `CupertinoThemeData` created by `ShadApp` is: + +```dart +CupertinoThemeData( + primaryColor: themeData.colorScheme.primary, + primaryContrastingColor: themeData.colorScheme.primaryForeground, + scaffoldBackgroundColor: themeData.colorScheme.background, + barBackgroundColor: themeData.colorScheme.primary, + brightness: themeData.brightness, +), +``` + +:::note +Use `CupertinoTheme.of(context).copyWith(...)` to override the default theme, without losing the default values provided by shadcn_ui. +::: \ No newline at end of file diff --git a/.claude/skills/shadcn-ui-flutter/guides/responsive.md b/.claude/skills/shadcn-ui-flutter/guides/responsive.md new file mode 100644 index 0000000..eb2678e --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/guides/responsive.md @@ -0,0 +1,65 @@ +In *shadcn_ui* the responsiveness is an important part of the library. + +The `ShadTheme` supports a customizable set of breakpoints. + +## Default + +```dart +ShadThemeData( + breakpoints: ShadBreakpoints( + tn: 0, // tiny + sm: 640, // small + md: 768, // medium + lg: 1024, // large + xl: 1280, // extra large + xxl: 1536, // extra extra large + ), +); +``` + +## Current breakpoint + +To get the current breakpoint you can use `ShadResponsiveBuilder` or `context.breakpoint`, eg: + +```dart + +ShadResponsiveBuilder( + builder: (context, breakpoint) { + final sm = breakpoint >= ShadTheme.of(context).breakpoints.sm; + ... + }, +), +``` + +which is equivalent to: + +```dart +final sm = context.breakpoint >= ShadTheme.of(context).breakpoints.sm; + +``` + +In Tailwind CSS, it's common to say that *sm* is not for small screens, but will target also the largest sizes if you don't provide a larger breakpoint. + +That's why I'm using the `>=` operator. + +If you just want to check if you're in a specific breakpoint, use the `==` operator. + +## Sealed class + +The breakpoint returned is a sealed class so you can switch any size. + +```dart + +ShadResponsiveBuilder( + builder: (context, breakpoint) { + return switch (breakpoint) { + ShadBreakpointTN() => const Text('Tiny'), + ShadBreakpointSM() => const Text('Small'), + ShadBreakpointMD() => const Text('Medium'), + ShadBreakpointLG() => const Text('Large'), + ShadBreakpointXL() => const Text('Extra Large'), + ShadBreakpointXXL() => const Text('Extra Extra Large'), + }; + }, +), +``` \ No newline at end of file diff --git a/.claude/skills/shadcn-ui-flutter/guides/theming.md b/.claude/skills/shadcn-ui-flutter/guides/theming.md new file mode 100644 index 0000000..318f41f --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/guides/theming.md @@ -0,0 +1,141 @@ +Defines the theme and color scheme for the app. + +The supported color schemes are: + +- blue +- gray +- green +- neutral +- orange +- red +- rose +- slate +- stone +- violet +- yellow +- zinc + +## Usage + +```diff lang="dart" + + +@override +Widget build(BuildContext context) { + return ShadApp( ++ darkTheme: ShadThemeData( ++ brightness: Brightness.dark, ++ colorScheme: const ShadSlateColorScheme.dark(), ++ ), + child: ... + ); +} +``` + +You can override specific properties of the selected theme/color scheme: + +```diff lang="dart" + + +@override +Widget build(BuildContext context) { + return ShadApp( + darkTheme: ShadThemeData( + brightness: Brightness.dark, + colorScheme: const ShadSlateColorScheme.dark( ++ background: Colors.blue, + ), ++ primaryButtonTheme: const ShadButtonTheme( ++ backgroundColor: Colors.cyan, ++ ), + ), + ), + child: ... + ); +} +``` + +You can also create your custom color scheme, just extend the `ShadColorScheme` class and pass all the properties. + + +## ShadColorScheme.fromName + +If you want to allow the user to change the default shadcn themes, I suggest using `ShadColorScheme.fromName`. + +```dart +// available color scheme names +final shadThemeColors = [ + 'blue', + 'gray', + 'green', + 'neutral', + 'orange', + 'red', + 'rose', + 'slate', + 'stone', + 'violet', + 'yellow', + 'zinc', +]; + +final lightColorScheme = ShadColorScheme.fromName('blue'); +final darkColorScheme = ShadColorScheme.fromName('slate', brightness: Brightness.dark); +``` + +In this way you can easily create a select to change the color scheme, for example: + +```dart + + + +// Somewhere in your app +ShadSelect( + initialValue: 'slate', + maxHeight: 200, + options: shadThemeColors.map( + (option) => ShadOption( + value: option, + child: Text( + option.capitalizeFirst(), + ), + ), + ), + selectedOptionBuilder: (context, value) { + return Text(value.capitalizeFirst()); + }, + onChanged: (value) { + // rebuild the app using your state management solution + }, +), +``` + +For example I'm using solidart as state management, here it is the example code used to rebuild the app widget when the user changes the theme mode. Check the "Toggle Theme" example at + +The same can be done for the color scheme, using a `Signal()` + +## Extend with custom colors + +You can extend the `ShadColorScheme` with your own custom colors by using the `custom` parameter. +```diff lang="dart" +return ShadApp( + theme: ShadThemeData( ++ colorScheme: const ShadZincColorScheme.light( ++ custom: { ++ 'myCustomColor': Color.fromARGB(255, 177, 4, 196), ++ }, ++ ), + ), +); +``` + +Then you can access it like this `ShadTheme.of(context).colorScheme.custom['myCustomColor']!`. + +Or you can create an extension on `ShadColorScheme` to make it easier to access: +```dart +extension CustomColorExtension on ShadColorScheme { + Color get myCustomColor => custom['myCustomColor']!; +} +``` + +In this way you can access it like other colors `ShadTheme.of(context).colorScheme.myCustomColor`. \ No newline at end of file diff --git a/.claude/skills/shadcn-ui-flutter/guides/typography.md b/.claude/skills/shadcn-ui-flutter/guides/typography.md new file mode 100644 index 0000000..23c2be3 --- /dev/null +++ b/.claude/skills/shadcn-ui-flutter/guides/typography.md @@ -0,0 +1,231 @@ +Styles for headings, paragraphs, lists...etc + +## h1Large + + + ```dart +Text( + 'Taxing Laughter: The Joke Tax Chronicles', + style: ShadTheme.of(context).textTheme.h1Large, +) +``` + + +## h1 + + + ```dart +Text( + 'Taxing Laughter: The Joke Tax Chronicles', + style: ShadTheme.of(context).textTheme.h1, +) +``` + + +## h2 + + + ```dart +Text( + 'The People of the Kingdom', + style: ShadTheme.of(context).textTheme.h2, +) +``` + + +## h3 + + + ```dart +Text( + 'The Joke Tax', + style: ShadTheme.of(context).textTheme.h3, +) +``` + + +## h4 + + + ```dart +Text( + 'The king, seeing how much happier his subjects were, realized the error of his ways and repealed the joke tax.', + style: ShadTheme.of(context).textTheme.h4, +) +``` + + +## p + + + ```dart +Text( + 'The king, seeing how much happier his subjects were, realized the error of his ways and repealed the joke tax.', + style: ShadTheme.of(context).textTheme.p, +) +``` + + +## Blockquote + + + ```dart +Text( + '"After all," he said, "everyone enjoys a good joke, so it\'s only fair that they should pay for the privilege."', + style: ShadTheme.of(context).textTheme.blockquote, +) +``` + + +## Table + + + ```dart +Text( + "King's Treasury", + style: ShadTheme.of(context).textTheme.table, +) +``` + + +## List + + + ```dart +Text( + '1st level of puns: 5 gold coins', + style: ShadTheme.of(context).textTheme.list, +) +``` + + +## Lead + + + ```dart +Text( + 'A modal dialog that interrupts the user with important content and expects a response.', + style: ShadTheme.of(context).textTheme.lead, +) +``` + + +## Large + + + ```dart +Text( + 'Are you absolutely sure?', + style: ShadTheme.of(context).textTheme.large, +) +``` + + +## Small + + + ```dart +Text( + 'Email address', + style: ShadTheme.of(context).textTheme.small, +) +``` + + +## Muted + + + ```dart +Text( + 'Enter your email address.', + style: ShadTheme.of(context).textTheme.muted, +) +``` + + +## Custom font family + +By default Shadcn UI uses [Geist](https://vercel.com/font) as default font family. +To change it, add the local font to your project, for example in the `/fonts` directory. +Then update your `pubspec.yaml` with something like this: + +```diff lang="yaml" +flutter: ++ fonts: ++ - family: UbuntuMono ++ fonts: ++ - asset: fonts/UbuntuMono-Regular.ttf ++ - asset: fonts/UbuntuMono-Italic.ttf ++ style: italic ++ - asset: fonts/UbuntuMono-Bold.ttf ++ weight: 700 ++ - asset: fonts/UbuntuMono-BoldItalic.ttf ++ weight: 700 ++ style: italic +``` + +Then in your `ShadApp` update the `ShadTextTheme`: +```diff lang="dart" +return ShadApp( + debugShowCheckedModeBanner: false, + themeMode: themeMode, + routes: routes, + theme: ShadThemeData( + brightness: Brightness.light, + colorScheme: const ShadZincColorScheme.light(), ++ textTheme: ShadTextTheme( ++ colorScheme: const ShadZincColorScheme.light(), ++ family: 'UbuntuMono', ++ ), + ), + ... +); +``` + +## Google font + +Install the [google_fonts](https://pub.dev/packages/google_fonts) package. +Then add the google font to your `ShadApp`: +```diff lang="dart" +return ShadApp( + debugShowCheckedModeBanner: false, + themeMode: themeMode, + routes: routes, + theme: ShadThemeData( + brightness: Brightness.light, + colorScheme: const ShadZincColorScheme.light(), ++ textTheme: ShadTextTheme.fromGoogleFont(GoogleFonts.poppins), + ), + ... +); +``` + +## Extend with custom styles + +You can extend the `ShadTextTheme` with your own custom styles by using the `custom` parameter. +```diff lang="dart" +return ShadApp( + theme: ShadThemeData( ++ textTheme: ShadTextTheme( ++ custom: { ++ 'myCustomStyle': const TextStyle( ++ fontSize: 16, ++ fontWeight: FontWeight.w400, ++ color: Colors.blue, ++ ), ++ }, ++ ), + ), +); +``` + +Then you can access it like this `ShadTheme.of(context).textTheme.custom['myCustomStyle']!`. + +Or you can create an extension on `ShadTextTheme` to make it easier to access: +```dart +extension CustomStyleExtension on ShadTextTheme { + TextStyle get myCustomStyle => custom['myCustomStyle']!; +} +``` + +In this way you can access it like other styles `ShadTheme.of(context).textTheme.myCustomStyle`. \ No newline at end of file diff --git a/.claude/skills/spark-python-data-source/SKILL.md b/.claude/skills/spark-python-data-source/SKILL.md index 898b9d2..4f90c60 100644 --- a/.claude/skills/spark-python-data-source/SKILL.md +++ b/.claude/skills/spark-python-data-source/SKILL.md @@ -1,238 +1,99 @@ --- name: spark-python-data-source -description: Use when building custom Spark data source connectors for external systems (databases, APIs, message queues), implementing batch/streaming readers/writers, or creating data source plugins for systems without native Spark support. Triggers - "build Spark data source", "create Spark connector", "implement Spark reader/writer", "connect Spark to [system]", "streaming data source" +description: Build custom Python data sources for Apache Spark using the PySpark DataSource API — batch and streaming readers/writers for external systems. Use this skill whenever someone wants to connect Spark to an external system (database, API, message queue, custom protocol), build a Spark connector or plugin in Python, implement a DataSourceReader or DataSourceWriter, pull data from or push data to a system via Spark, or work with the PySpark DataSource API in any way. Even if they just say "read from X in Spark" or "write DataFrame to Y" and there's no native connector, this skill applies. --- # spark-python-data-source Build custom Python data sources for Apache Spark 4.0+ to read from and write to external systems in batch and streaming modes. -## When to use - -Use when building Spark connectors for external systems that lack native support: -- External databases, APIs, message queues -- Custom file formats or protocols -- Real-time streaming data sources -- Systems requiring specialized authentication or protocols - -Triggers: "build Spark data source", "create Spark connector", "implement Spark reader/writer", "connect Spark to [system]", "streaming data source" - ## Instructions -You are an experienced Spark developer building custom Python data sources following the PySpark DataSource API. Follow these principles and patterns: +You are an experienced Spark developer building custom Python data sources using the PySpark DataSource API. Follow these principles and patterns. ### Core Architecture Each data source follows a flat, single-level inheritance structure: -1. **DataSource class** - Entry point returning readers/writers -2. **Base Reader/Writer classes** - Shared logic for options and data processing -3. **Batch classes** - Inherit from base + `DataSourceReader`/`DataSourceWriter` -4. **Stream classes** - Inherit from base + `DataSourceStreamReader`/`DataSourceStreamWriter` - -### Critical Design Principles - -**SIMPLE over CLEVER** - These are non-negotiable: - -✅ REQUIRED: -- Flat single-level inheritance only -- Direct implementations, no abstractions -- Explicit imports, explicit control flow -- Standard library first, minimal dependencies -- Simple classes with single responsibilities - -❌ FORBIDDEN: -- Abstract base classes or complex inheritance -- Factory patterns or dependency injection -- Decorators for cross-cutting concerns -- Complex configuration classes -- Async/await (unless absolutely necessary) -- Connection pooling or caching (unless critical) -- Generic "framework" code -- Premature optimization - -### Implementation Pattern - -```python -from pyspark.sql.datasource import ( - DataSource, DataSourceReader, DataSourceWriter, - DataSourceStreamReader, DataSourceStreamWriter -) - -# 1. DataSource class -class YourDataSource(DataSource): - @classmethod - def name(cls): - return "your-format" - - def __init__(self, options): - self.options = options - - def schema(self): - return self._infer_or_return_schema() - - def reader(self, schema): - return YourBatchReader(self.options, schema) - - def streamReader(self, schema): - return YourStreamReader(self.options, schema) - - def writer(self, schema, overwrite): - return YourBatchWriter(self.options, schema) - - def streamWriter(self, schema, overwrite): - return YourStreamWriter(self.options, schema) - -# 2. Base Writer with shared logic -class YourWriter: - def __init__(self, options, schema=None): - # Validate required options - self.url = options.get("url") - assert self.url, "url is required" - self.batch_size = int(options.get("batch_size", "50")) - self.schema = schema - - def write(self, iterator): - # Import libraries here for partition execution - import requests - from pyspark import TaskContext - - context = TaskContext.get() - partition_id = context.partitionId() - - msgs = [] - cnt = 0 - - for row in iterator: - cnt += 1 - msgs.append(row.asDict()) - - if len(msgs) >= self.batch_size: - self._send_batch(msgs) - msgs = [] +1. **DataSource class** — entry point that returns readers/writers +2. **Base Reader/Writer classes** — shared logic for options and data processing +3. **Batch classes** — inherit from base + `DataSourceReader`/`DataSourceWriter` +4. **Stream classes** — inherit from base + `DataSourceStreamReader`/`DataSourceStreamWriter` - if msgs: - self._send_batch(msgs) +See [implementation-template.md](references/implementation-template.md) for the full annotated skeleton covering all four modes (batch read/write, stream read/write). - return SimpleCommitMessage(partition_id=partition_id, count=cnt) +### Spark-Specific Design Constraints - def _send_batch(self, msgs): - # Implement send logic - pass +These are specific to the PySpark DataSource API and its driver/executor architecture — general Python best practices (clean code, minimal dependencies, no premature abstraction) still apply but aren't repeated here. -# 3. Batch Writer -class YourBatchWriter(YourWriter, DataSourceWriter): - pass +**Flat single-level inheritance only.** PySpark serializes reader/writer instances to ship them to executors. Complex inheritance hierarchies and abstract base classes break serialization and make cross-process debugging painful. Use one shared base class mixed with the PySpark interface (e.g., `class YourBatchWriter(YourWriter, DataSourceWriter)`). -# 4. Stream Writer -class YourStreamWriter(YourWriter, DataSourceStreamWriter): - def commit(self, messages, batchId): - pass +**Import third-party libraries inside executor methods.** The `read()` and `write()` methods run on remote executor processes that don't share the driver's Python environment. Top-level imports from the driver won't be available on executors — always import libraries like `requests` or database drivers inside the methods that run on workers. - def abort(self, messages, batchId): - pass +**Minimize dependencies.** Every package you add must be installed on all executor nodes in the cluster, not just the driver. Prefer the standard library; when external packages are needed, keep them few and well-known. -# 5. Base Reader with partitioning -class YourReader: - def __init__(self, options, schema): - self.url = options.get("url") - assert self.url, "url is required" - self.schema = schema - - def partitions(self): - # Return list of partitions for parallel reading - return [YourPartition(0, start, end)] - - def read(self, partition): - # Import here for executor execution - import requests - - response = requests.get(f"{self.url}?start={partition.start}") - for item in response.json(): - yield tuple(item.values()) - -# 6. Batch Reader -class YourBatchReader(YourReader, DataSourceReader): - pass - -# 7. Stream Reader -class YourStreamReader(YourReader, DataSourceStreamReader): - def initialOffset(self): - return {"offset": "0"} - - def latestOffset(self): - return {"offset": str(self._get_latest())} - - def partitions(self, start, end): - return [YourPartition(0, start["offset"], end["offset"])] - - def commit(self, end): - pass -``` +**No async/await** unless the external system's SDK is async-only. The PySpark DataSource API is synchronous, so async adds complexity with no benefit. ### Project Setup +Create a Python project using a packaging tool such as `uv`, `poetry`, or `hatch`. Examples use `uv` (substitute your tool of choice): + ```bash -# Create project -poetry new your-datasource +uv init your-datasource cd your-datasource -poetry add pyspark pytest pytest-spark - -# Development commands - CRITICAL: Always use 'poetry run' -poetry run pytest # Run tests -poetry run ruff check src/ # Lint -poetry run ruff format src/ # Format -poetry build # Build wheel +uv add pyspark pytest pytest-spark ``` -### Registration and Usage - -```python -# Register -from your_package import YourDataSource -spark.dataSource.register(YourDataSource) - -# Batch read -df = spark.read.format("your-format").option("url", "...").load() - -# Batch write -df.write.format("your-format").option("url", "...").save() +``` +your-datasource/ +├── pyproject.toml +├── src/ +│ └── your_datasource/ +│ ├── __init__.py +│ └── datasource.py +└── tests/ + ├── conftest.py + └── test_datasource.py +``` -# Streaming read -df = spark.readStream.format("your-format").option("url", "...").load() +Run all commands through the packaging tool so they execute within the correct virtual environment: -# Streaming write -df.writeStream.format("your-format").option("url", "...").start() +```bash +uv run pytest # Run tests +uv run ruff check src/ # Lint +uv run ruff format src/ # Format +uv build # Build wheel ``` ### Key Implementation Decisions -**Partitioning Strategy**: Choose based on data source characteristics -- Time-based: For APIs with temporal data (see [partitioning-patterns.md](references/partitioning-patterns.md)) -- Token-range: For distributed databases (see [partitioning-patterns.md](references/partitioning-patterns.md)) -- ID-range: For paginated APIs +**Partitioning Strategy** — choose based on data source characteristics: +- Time-based: for APIs with temporal data +- Token-range: for distributed databases +- ID-range: for paginated APIs +- See [partitioning-patterns.md](references/partitioning-patterns.md) for implementations of each strategy -**Authentication**: Support multiple methods in priority order +**Authentication** — support multiple methods in priority order: - Databricks Unity Catalog credentials - Cloud default credentials (managed identity) - Explicit credentials (service principal, API key, username/password) -- See [authentication-patterns.md](references/authentication-patterns.md) +- See [authentication-patterns.md](references/authentication-patterns.md) for patterns with fallback chains -**Type Conversion**: Map between Spark and external types +**Type Conversion** — map between Spark and external types: - Handle nulls, timestamps, UUIDs, collections -- See [type-conversion.md](references/type-conversion.md) +- See [type-conversion.md](references/type-conversion.md) for bidirectional mapping tables and helpers -**Streaming Offsets**: Design for exactly-once semantics +**Streaming Offsets** — design for exactly-once semantics: - JSON-serializable offset class - Non-overlapping partition boundaries -- See [streaming-patterns.md](references/streaming-patterns.md) +- See [streaming-patterns.md](references/streaming-patterns.md) for offset tracking and watermark patterns -**Error Handling**: Implement retries and resilience -- Exponential backoff for retryable errors +**Error Handling** — implement retries and resilience: +- Exponential backoff for transient failures (network, rate limits) - Circuit breakers for cascading failures -- See [error-handling.md](references/error-handling.md) +- See [error-handling.md](references/error-handling.md) for retry decorators and failure classification -### Testing Approach +### Testing ```python import pytest @@ -256,33 +117,17 @@ def test_writer_sends_data(spark): assert mock_post.called ``` -### Code Review Checklist - -Before implementing, ask: -1. Is this the simplest way to solve this problem? -2. Would a new developer understand this immediately? -3. Am I adding abstraction for real needs vs hypothetical flexibility? -4. Can I solve this with standard library? -5. Does this follow the established flat pattern? - -### Common Mistakes to Avoid - -- Creating abstract base classes for "reusability" -- Adding configuration frameworks or dependency injection -- Premature optimization before measuring performance -- Complex error handling hierarchies -- Importing heavy libraries at module level (import in methods) -- Using `python` command directly (always use `poetry run`) +See [testing-patterns.md](references/testing-patterns.md) for unit/integration test patterns, fixtures, and running tests. ### Reference Implementations Study these for real-world patterns: -- [cyber-spark-data-connectors](https://github.com/alexott/cyber-spark-data-connectors) - Sentinel, Splunk, REST -- [spark-cassandra-data-source](https://github.com/alexott/spark-cassandra-data-source) - Token-range partitioning -- [pyspark-hubspot](https://github.com/dgomez04/pyspark-hubspot) - REST API pagination -- [pyspark-mqtt](https://github.com/databricks-industry-solutions/python-data-sources/tree/main/mqtt) - Streaming with TLS +- [cyber-spark-data-connectors](https://github.com/alexott/cyber-spark-data-connectors) — Sentinel, Splunk, REST +- [spark-cassandra-data-source](https://github.com/alexott/spark-cassandra-data-source) — Token-range partitioning +- [pyspark-hubspot](https://github.com/dgomez04/pyspark-hubspot) — REST API pagination +- [pyspark-mqtt](https://github.com/databricks-industry-solutions/python-data-sources/tree/main/mqtt) — Streaming with TLS -## Usage +## Example Prompts ``` Create a Spark data source for reading from MongoDB with sharding support @@ -299,13 +144,14 @@ Write a data source for REST API with OAuth2 authentication and pagination ## References -- [partitioning-patterns.md](references/partitioning-patterns.md) - Parallel reading strategies -- [authentication-patterns.md](references/authentication-patterns.md) - Multi-method auth implementations -- [type-conversion.md](references/type-conversion.md) - Bidirectional type mapping -- [streaming-patterns.md](references/streaming-patterns.md) - Offset management and watermarking -- [error-handling.md](references/error-handling.md) - Retries, circuit breakers, resilience -- [testing-patterns.md](references/testing-patterns.md) - Unit and integration testing -- [production-patterns.md](references/production-patterns.md) - Observability, security, validation +- [implementation-template.md](references/implementation-template.md) — Full annotated skeleton; read when starting a new data source +- [partitioning-patterns.md](references/partitioning-patterns.md) — Read when the source supports parallel reads and you need to split work across executors +- [authentication-patterns.md](references/authentication-patterns.md) — Read when the external system requires credentials or tokens +- [type-conversion.md](references/type-conversion.md) — Read when mapping between Spark types and the external system's type system +- [streaming-patterns.md](references/streaming-patterns.md) — Read when implementing `DataSourceStreamReader` or `DataSourceStreamWriter` +- [error-handling.md](references/error-handling.md) — Read when adding retry logic or handling transient failures +- [testing-patterns.md](references/testing-patterns.md) — Read when writing tests; covers unit, integration, and performance testing +- [production-patterns.md](references/production-patterns.md) — Read when hardening for production: observability, security, input validation - [Official Databricks Documentation](https://docs.databricks.com/aws/en/pyspark/datasources) - [Apache Spark Python DataSource Tutorial](https://spark.apache.org/docs/latest/api/python/tutorial/sql/python_data_source.html) -- [awesome-python-datasources](https://github.com/allisonwang-db/awesome-python-datasources) - directory of available implementations. +- [awesome-python-datasources](https://github.com/allisonwang-db/awesome-python-datasources) — Directory of community implementations diff --git a/.claude/skills/spark-python-data-source/references/implementation-template.md b/.claude/skills/spark-python-data-source/references/implementation-template.md new file mode 100644 index 0000000..045fe94 --- /dev/null +++ b/.claude/skills/spark-python-data-source/references/implementation-template.md @@ -0,0 +1,141 @@ +# Implementation Template + +Full skeleton for a Python data source covering all four modes: batch read, batch write, stream read, stream write. Adapt to your needs — most connectors only implement a subset. + +```python +from pyspark.sql.datasource import ( + DataSource, DataSourceReader, DataSourceWriter, + DataSourceStreamReader, DataSourceStreamWriter +) + +# 1. DataSource class — entry point that returns readers/writers +class YourDataSource(DataSource): + @classmethod + def name(cls): + return "your-format" + + def __init__(self, options): + self.options = options + + def schema(self): + return self._infer_or_return_schema() + + def reader(self, schema): + return YourBatchReader(self.options, schema) + + def streamReader(self, schema): + return YourStreamReader(self.options, schema) + + def writer(self, schema, overwrite): + return YourBatchWriter(self.options, schema) + + def streamWriter(self, schema, overwrite): + return YourStreamWriter(self.options, schema) + +# 2. Base Writer — shared logic for batch and stream writing +# Plain class (not a DataSourceWriter yet) so batch/stream +# subclasses can mix it in with the right PySpark base. +class YourWriter: + def __init__(self, options, schema=None): + self.url = options.get("url") + assert self.url, "url is required" + self.batch_size = int(options.get("batch_size", "50")) + self.schema = schema + + def write(self, iterator): + # Import here — this runs on executors, not the driver. + # Executor processes don't share the driver's module state. + import requests + from pyspark import TaskContext + + context = TaskContext.get() + partition_id = context.partitionId() + + msgs = [] + cnt = 0 + + for row in iterator: + cnt += 1 + msgs.append(row.asDict()) + + if len(msgs) >= self.batch_size: + self._send_batch(msgs) + msgs = [] + + if msgs: + self._send_batch(msgs) + + return SimpleCommitMessage(partition_id=partition_id, count=cnt) + + def _send_batch(self, msgs): + # Implement send logic + pass + +# 3. Batch Writer — inherits shared logic + PySpark interface +class YourBatchWriter(YourWriter, DataSourceWriter): + pass + +# 4. Stream Writer — adds commit/abort for micro-batch semantics +class YourStreamWriter(YourWriter, DataSourceStreamWriter): + def commit(self, messages, batchId): + pass + + def abort(self, messages, batchId): + pass + +# 5. Base Reader — shared logic for batch and stream reading +class YourReader: + def __init__(self, options, schema): + self.url = options.get("url") + assert self.url, "url is required" + self.schema = schema + + def partitions(self): + return [YourPartition(0, start, end)] + + def read(self, partition): + # Import here — runs on executors + import requests + + response = requests.get(f"{self.url}?start={partition.start}") + for item in response.json(): + yield tuple(item.values()) + +# 6. Batch Reader +class YourBatchReader(YourReader, DataSourceReader): + pass + +# 7. Stream Reader — adds offset tracking for incremental reads +class YourStreamReader(YourReader, DataSourceStreamReader): + def initialOffset(self): + return {"offset": "0"} + + def latestOffset(self): + return {"offset": str(self._get_latest())} + + def partitions(self, start, end): + return [YourPartition(0, start["offset"], end["offset"])] + + def commit(self, end): + pass +``` + +## Registration and Usage + +```python +# Register +from your_package import YourDataSource +spark.dataSource.register(YourDataSource) + +# Batch read +df = spark.read.format("your-format").option("url", "...").load() + +# Batch write +df.write.format("your-format").option("url", "...").save() + +# Streaming read +df = spark.readStream.format("your-format").option("url", "...").load() + +# Streaming write +df.writeStream.format("your-format").option("url", "...").start() +``` diff --git a/.claude/skills/spark-python-data-source/references/production-patterns.md b/.claude/skills/spark-python-data-source/references/production-patterns.md index 71928ca..6dfbd8a 100644 --- a/.claude/skills/spark-python-data-source/references/production-patterns.md +++ b/.claude/skills/spark-python-data-source/references/production-patterns.md @@ -146,146 +146,55 @@ class LoggingWriter: ## Security Validation -Input validation and sanitization: +Input validation and sanitization for production data sources: ```python import re import ipaddress class SecureDataSource: - """Data source with security validation.""" - - # Sensitive keys that should never be logged - SENSITIVE_KEYS = { - "password", "api_key", "client_secret", "token", - "access_token", "refresh_token", "bearer_token" - } + """Data source with input validation.""" def __init__(self, options): - # Validate and sanitize options self._validate_options(options) self.options = options - # Create sanitized version for logging - self._safe_options = self._sanitize_for_logging(options) - def _validate_options(self, options): - """Comprehensive option validation.""" - # Validate required options + """Validate options at system boundary.""" required = ["host", "database", "table"] missing = [opt for opt in required if opt not in options] if missing: raise ValueError(f"Missing required options: {', '.join(missing)}") - # Validate host (IP or hostname) self._validate_host(options["host"]) - # Validate port range if "port" in options: port = int(options["port"]) if port < 1 or port > 65535: raise ValueError(f"Port must be 1-65535, got {port}") - # Validate table name (prevent SQL injection) self._validate_identifier(options["table"], "table") - # Validate numeric options - if "batch_size" in options: - batch_size = int(options["batch_size"]) - if batch_size < 1 or batch_size > 10000: - raise ValueError(f"batch_size must be 1-10000, got {batch_size}") - def _validate_host(self, host): """Validate host is valid IP or hostname.""" try: - # Try as IP address ipaddress.ip_address(host) return except ValueError: pass - - # Validate as hostname if not re.match(r'^[a-zA-Z0-9][a-zA-Z0-9-\.]*[a-zA-Z0-9]$', host): raise ValueError(f"Invalid host format: {host}") def _validate_identifier(self, identifier, name): - """Validate SQL identifier (table, column name).""" - # Prevent SQL injection + """Validate SQL identifier to prevent injection.""" if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', identifier): raise ValueError( f"Invalid {name} identifier: {identifier}. " - f"Must contain only letters, numbers, and underscores, " - f"and start with a letter or underscore." + f"Must contain only letters, numbers, and underscores." ) - - def _sanitize_for_logging(self, options): - """Mask sensitive values for logging.""" - safe = {} - for key, value in options.items(): - if key.lower() in self.SENSITIVE_KEYS: - safe[key] = "***REDACTED***" - else: - safe[key] = value - return safe - - def __repr__(self): - return f"SecureDataSource({self._safe_options})" ``` -## Secrets Management - -Load credentials from secure storage: - -```python -def load_secrets_from_databricks(scope, keys): - """Load secrets from Databricks secrets.""" - try: - from pyspark.dbutils import DBUtils - from pyspark.sql import SparkSession - - spark = SparkSession.getActiveSession() - if not spark: - raise ValueError("No active Spark session") - - dbutils = DBUtils(spark) - secrets = {} - - for key in keys: - try: - secrets[key] = dbutils.secrets.get(scope=scope, key=key) - except Exception as e: - raise ValueError(f"Failed to load secret '{key}' from scope '{scope}': {e}") - - return secrets - - except Exception as e: - raise ValueError(f"Failed to access Databricks secrets: {e}") - -class SecureCredentialLoader: - """Load credentials securely.""" - - @staticmethod - def load_credentials(options): - """Load credentials from secure storage.""" - # Priority 1: Databricks secrets - if "secret_scope" in options: - secret_keys = [ - "username", "password", "api_key", "client_secret" - ] - secrets = load_secrets_from_databricks( - options["secret_scope"], - secret_keys - ) - options.update(secrets) - - # Priority 2: Environment variables - elif options.get("use_env_vars", "false").lower() == "true": - import os - options["username"] = os.environ.get("DB_USERNAME") - options["password"] = os.environ.get("DB_PASSWORD") - - return options -``` +For credential sanitization in logs and secrets management, see [authentication-patterns.md](authentication-patterns.md) — the "Security Best Practices" and "Use Secrets Management" sections. ## Configuration Validation diff --git a/.claude/skills/spark-python-data-source/references/streaming-patterns.md b/.claude/skills/spark-python-data-source/references/streaming-patterns.md index 6f00ddd..66b9e8e 100644 --- a/.claude/skills/spark-python-data-source/references/streaming-patterns.md +++ b/.claude/skills/spark-python-data-source/references/streaming-patterns.md @@ -396,5 +396,5 @@ class MonitoredStreamReader(DataSourceStreamReader): 3. **State Management**: Store offsets in Spark checkpoints 4. **Watermarking**: Support event-time processing for late data 5. **Monitoring**: Track batch progress and lag metrics -6. **Error Handling**: Implement retry logic for transient failures +6. **Error Handling**: Streaming writers are especially susceptible to transient failures (network blips, rate limits) since they run continuously. Use retry with exponential backoff from [error-handling.md](error-handling.md) in your `write()` methods. 7. **Backpressure**: Respect rate limits with appropriate partition sizing diff --git a/.claude/skills/spark-python-data-source/references/testing-patterns.md b/.claude/skills/spark-python-data-source/references/testing-patterns.md index 96e2e28..1b4aeb2 100644 --- a/.claude/skills/spark-python-data-source/references/testing-patterns.md +++ b/.claude/skills/spark-python-data-source/references/testing-patterns.md @@ -415,25 +415,27 @@ tests/ ## Running Tests +Run tests through your packaging tool (e.g., `uv run`, `poetry run`, `hatch run`). Examples use `uv`: + ```bash # Run all tests -poetry run pytest +uv run pytest # Run specific test file -poetry run pytest tests/unit/test_writer.py +uv run pytest tests/unit/test_writer.py # Run specific test -poetry run pytest tests/unit/test_writer.py::test_writer_sends_batch +uv run pytest tests/unit/test_writer.py::test_writer_sends_batch # Run with coverage -poetry run pytest --cov=your_package --cov-report=html +uv run pytest --cov=your_package --cov-report=html # Run only unit tests -poetry run pytest tests/unit/ +uv run pytest tests/unit/ # Run with verbose output -poetry run pytest -v +uv run pytest -v # Run with print statements -poetry run pytest -s +uv run pytest -s ``` diff --git a/CLAUDE.md b/CLAUDE.md index 5ccac7f..b301e11 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,17 +1,17 @@ # Claude Code on Databricks -Welcome! This environment comes pre-configured with 5 AI coding agents, 39 skills, and 2 MCP servers. Hermes Agent is available alongside Claude Code, Codex, Gemini CLI, and OpenCode — launch it with `hermes chat`. +Welcome! This environment comes pre-configured with 5 AI coding agents, 50 skills, and 3 MCP servers. Hermes Agent is available alongside Claude Code, Codex, Gemini CLI, and OpenCode — launch it with `hermes chat`. -## Skills (30 total) +## Skills (50 total) ### Databricks Skills (16) | Category | Skills | |----------|--------| -| AI & Agents | agent-bricks, databricks-genie, mlflow-evaluation, model-serving | +| AI & Agents | agent-bricks, databricks-genie, databricks-ai-functions, mlflow-evaluation, model-serving | | Analytics | aibi-dashboards, databricks-unity-catalog | -| Data Engineering | spark-declarative-pipelines, databricks-jobs, synthetic-data-generation | -| Development | asset-bundles, databricks-app-apx, databricks-app-python, databricks-python-sdk, databricks-config | +| Data Engineering | spark-declarative-pipelines, databricks-jobs, synthetic-data-gen | +| Development | databricks-bundles, databricks-apps-python, databricks-python-sdk, databricks-config | | Reference | databricks-docs, unstructured-pdf-generation | ### Development Workflow Skills (14) @@ -35,10 +35,52 @@ From [obra/superpowers](https://github.com/obra/superpowers): | writing-skills | Create new skills | | using-superpowers | Introduction to available skills | +### Flutter Skills (10) + +From [flutter/skills](https://github.com/flutter/skills): + +| Skill | Purpose | +|-------|---------| +| flutter-add-integration-test | Add integration tests | +| flutter-add-widget-preview | Add widget previews | +| flutter-add-widget-test | Add widget tests | +| flutter-apply-architecture-best-practices | Apply Flutter architecture best practices | +| flutter-build-responsive-layout | Build adaptive/responsive layouts | +| flutter-fix-layout-issues | Diagnose and fix layout issues | +| flutter-implement-json-serialization | Implement JSON serialization | +| flutter-setup-declarative-routing | Set up declarative routing (go_router) | +| flutter-setup-localization | Set up localization (i18n) | +| flutter-use-http-package | Networking with the http package | + +### Dart Skills (9) + +From [dart-lang/skills](https://github.com/dart-lang/skills): + +| Skill | Purpose | +|-------|---------| +| dart-add-unit-test | Add unit tests | +| dart-build-cli-app | Build a Dart CLI app | +| dart-collect-coverage | Collect test coverage | +| dart-fix-runtime-errors | Diagnose and fix runtime errors | +| dart-generate-test-mocks | Generate test mocks (mockito/mocktail) | +| dart-migrate-to-checks-package | Migrate assertions to the checks package | +| dart-resolve-package-conflicts | Resolve package version conflicts | +| dart-run-static-analysis | Run static analysis | +| dart-use-pattern-matching | Use Dart pattern matching | + +### Flutter UI Skills (1) + +From [nank1ro/flutter-shadcn-ui](https://github.com/nank1ro/flutter-shadcn-ui): + +| Skill | Purpose | +|-------|---------| +| shadcn-ui-flutter | Build Flutter UIs with shadcn_ui components | + ## MCP Servers - **DeepWiki** - AI-powered documentation for any GitHub repository - **Exa** - Web search and code context retrieval +- **CoDA** (exposed at `/mcp`) - Delegate coding tasks to AI agents via MCP. Any MCP client (Genie Code, Claude Desktop, Cursor) can call `coda_run`, `coda_inbox`, and `coda_get_result` to submit background tasks, check status, and retrieve results. See `docs/mcp-v2-background-execution.md`. ## Databricks CLI diff --git a/README.md b/README.md index aca755c..1e80205 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Use this template](https://img.shields.io/badge/Use%20this%20template-2ea44f?logo=github)](https://github.com/datasciencemonkey/coding-agents-databricks-apps/generate) [![Deploy to Databricks](https://img.shields.io/badge/Deploy-Databricks%20Apps-FF3621?logo=databricks&logoColor=white)](docs/deployment.md) [![Agents](https://img.shields.io/badge/Agents-5%20included-green)](#whats-inside) -[![Skills](https://img.shields.io/badge/Skills-39%20built--in-blue)](#-all-39-skills) +[![Skills](https://img.shields.io/badge/Skills-43%20built--in-blue)](#-all-43-skills) > Run Claude Code, Codex, Gemini CLI, Hermes Agent, and OpenCode in your browser — zero setup, wired to your Databricks workspace. @@ -68,7 +68,7 @@ This isn't just a terminal in the cloud. Running coding agents on Databricks giv | ✂️ **Split Panes** | Run two sessions side by side with a draggable divider | | 🌐 **WebSocket I/O** | Real-time terminal output over WebSocket — zero-latency, eliminates polling delay | | 🔁 **HTTP Polling Fallback** | Automatic fallback via Web Worker when WebSocket is unavailable | -| 🚀 **Parallel Setup** | 7 agent setups run in parallel (~5x faster startup) | +| 🚀 **Parallel Setup** | 6 agent setups run in parallel (~5x faster startup) | | 🔍 **Search** | Find anything in your terminal history (Ctrl+Shift+F) | | 🎤 **Voice Input** | Dictate commands with your mic (Option+V) | | 📋 **Image Paste** | Paste or drag-and-drop images into the terminal — saved to `~/uploads/`, path inserted automatically | @@ -179,7 +179,7 @@ This template repo opens that vision up for every Databricks user — no IDE set ---
-🧠 All 39 Skills +🧠 All 43 Skills ### Databricks Skills (25) — [ai-dev-kit](https://github.com/databricks-solutions/ai-dev-kit) @@ -204,16 +204,100 @@ This template repo opens that vision up for every Databricks user — no IDE set | Ship | finishing-branch, git-worktrees | | Meta | dispatching-agents, writing-skills, using-superpowers | +### BDD Skills (4) + +| Category | Skills | +|----------|--------| +| Testing | bdd-features, bdd-run, bdd-scaffold, bdd-steps | +
-🔌 2 MCP Servers +🔌 MCP Servers + +### Built-in MCP Clients | Server | What it does | |--------|-------------| | **DeepWiki** | Ask questions about any GitHub repo — gets AI-powered answers from the codebase | | **Exa** | Web search and code context retrieval for up-to-date information | +### CoDA MCP Server (exposed at `/mcp`) + +CoDA itself exposes an **MCP server** that any MCP-compatible client can connect to — delegate coding tasks to AI agents running on Databricks, without needing the terminal UI. + +| Tool | Purpose | +|------|---------| +| `coda_run` | Fire-and-forget: submit a coding task, get back immediately | +| `coda_inbox` | Dashboard: see all running/completed/failed tasks at a glance | +| `coda_get_result` | Pull the full structured result of a completed task | + +**Why this matters:** Any tool that speaks MCP can use your Databricks-hosted coding agents — no custom integration needed. + +#### Example: Databricks Genie Code + +Genie Code connects to CoDA's MCP endpoint and delegates coding work to agents running in the background: + +``` +User → Genie Code: "Build me a sales pipeline using the transactions table" + +Genie Code calls coda_run(prompt="Build a sales pipeline...", email="user@company.com", + context='{"tables": ["sales.transactions"]}') + +→ Returns immediately: {task_id: "task-abc", status: "running"} +→ User keeps chatting with Genie Code while the agent works + +User → Genie Code: "How's my pipeline coming?" + +Genie Code calls coda_inbox() +→ {tasks: [{task_id: "task-abc", status: "completed", summary: "Built pipeline.py..."}]} + +Genie Code calls coda_get_result(task_id="task-abc", session_id="sess-123") +→ {summary: "Created pipeline.py with 3 stages", files_changed: ["pipeline.py"], ...} +``` + +#### Connecting MCP Clients (Claude Code, Claude Desktop, Cursor, etc.) + +Databricks Apps use OAuth — not PATs — for authentication. A static `Authorization: Bearer ` header will get a `302` redirect to the OAuth login page. To connect any MCP client, use the **stdio bridge** (`tools/coda-bridge.py`) which injects fresh OAuth tokens automatically via `databricks auth token`. + +**1. Copy the bridge script:** + +```bash +mkdir -p ~/.claude/mcp-bridges +cp tools/coda-bridge.py ~/.claude/mcp-bridges/ +``` + +**2. Add to your MCP client settings** (e.g. `~/.claude/settings.json`): + +```json +"coda-mcp": { + "type": "stdio", + "command": "python3", + "args": ["/path/to/.claude/mcp-bridges/coda-bridge.py"], + "env": { + "CODA_MCP_URL": "https://your-app.databricksapps.com/mcp", + "DATABRICKS_PROFILE": "your-profile" + } +} +``` + +**3. Restart your MCP client.** + +The bridge reads `CODA_MCP_URL` and `DATABRICKS_PROFILE` from environment — no hardcoded values. If you redeploy the app or switch workspaces, just update the `env` block. + +**Prerequisites:** `databricks` CLI installed and authenticated (`databricks auth login -p `), Python 3.8+, no pip dependencies. + +**Troubleshooting:** Bridge logs go to stderr. If you see `Auth failed (302)`, refresh your CLI session with `databricks auth login -p `. See [full setup guide](docs/mcp-client-setup.md) for details. + +#### Task Chaining + +Chain tasks by passing `previous_session_id` — the new agent reads the prior task's results for context: + +``` +coda_run(prompt="Add monitoring to the pipeline", previous_session_id="sess-123") +``` + +See [MCP v2 Design Doc](docs/mcp-v2-background-execution.md) for the full protocol reference.
@@ -221,13 +305,14 @@ This template repo opens that vision up for every Databricks user — no IDE set 🏗️ Architecture ``` -┌─────────────────────┐ WebSocket ┌─────────────────────┐ -│ Browser Client │◄═══════════►│ Gunicorn + Flask │ -│ (xterm.js) │ (primary) │ + Flask-SocketIO │ -│ │───────────►│ (PTY Manager) │ -│ │ HTTP Poll │ │ -│ │ (fallback) │ │ -└─────────────────────┘ └─────────────────────┘ +┌─────────────────────┐ WebSocket ┌──────────────────────────────────┐ +│ Browser Client │◄═══════════►│ uvicorn (ASGI) │ +│ (xterm.js) │ (fallback) │ ├─ python-socketio (Socket.IO) │ +│ │───────────►│ ├─ FastMCP /mcp │ +│ │ HTTP Poll │ └─ WSGIMiddleware(Flask + PTY) │ +│ │ (primary │ │ +│ │ under uvicorn) │ +└─────────────────────┘ └──────────────────────────────────┘ │ │ │ on first load │ on startup ▼ ▼ @@ -245,9 +330,9 @@ This template repo opens that vision up for every Databricks user — no IDE set ### Startup Flow -1. Gunicorn starts, calls `initialize_app()` via `post_worker_init` hook +1. uvicorn starts `coda_mcp.mcp_asgi:app`, which calls `initialize_app()` during ASGI lifespan startup (Flask mounted via `WSGIMiddleware`; MCP mounted at `/mcp` via native ASGI; Socket.IO wraps both) 2. App serves the terminal UI with inline setup progress -3. Background thread runs setup: 5 sequential steps (git config, micro editor, GitHub CLI, Databricks CLI upgrade, content-filter proxy), then 6 agent setups (Claude, Codex, OpenCode, Gemini, Databricks CLI config, MLflow) run in parallel via `ThreadPoolExecutor` +3. Background thread runs setup: 5 sequential steps (git config, micro editor, GitHub CLI, Databricks CLI upgrade, content-filter proxy), then 6 agent setups (`setup/setup_claude.py`, `setup/setup_codex.py`, etc.) run in parallel via `ThreadPoolExecutor` 4. `/api/setup-status` endpoint reports progress to the UI 5. Once complete, the terminal becomes interactive @@ -267,6 +352,7 @@ This template repo opens that vision up for every Databricks user — no IDE set | `/api/resize` | POST | Resize terminal dimensions | | `/api/upload` | POST | Upload file (clipboard image paste) | | `/api/session/close` | POST | Close terminal session | +| `/mcp` | POST | MCP JSON-RPC endpoint (CoDA tools) | ### WebSocket Events (Socket.IO) @@ -303,9 +389,9 @@ This template repo opens that vision up for every Databricks user — no IDE set Single-user app — the owner is resolved via the app's service principal and Apps API (`app.creator`), with no PAT required at deploy time. Authorization checks `X-Forwarded-Email` against `app.creator`. On first terminal session, the user pastes a short-lived PAT interactively. Tokens auto-rotate every 10 minutes (15-minute lifetime), with old tokens proactively revoked. On restart, the user re-pastes (no persistence by design). -### Gunicorn +### Server -Production uses `workers=1` (PTY state is process-local), `threads=16` (concurrent polling + WebSocket), `gthread` worker class, `timeout=60` (long-lived WebSocket connections). +Production uses `uvicorn` (single worker — PTY state is process-local) serving `coda_mcp.mcp_asgi:app`. The ASGI stack composes `python-socketio.ASGIApp` → MCP Streamable HTTP at `/mcp` → `WSGIMiddleware(Flask)` for the terminal UI. WebSocket transport falls back to HTTP polling under uvicorn — the `static/poll-worker.js` Web Worker already handles this transparently. `gunicorn.conf.py` is retained for reference and local WSGI-only dev; it is **not** used in production. @@ -316,27 +402,36 @@ Production uses `workers=1` (PTY state is process-local), `threads=16` (concurre coding-agents-databricks-apps/ ├── app.py # Flask backend + PTY management + setup orchestration ├── app_state.py # Shared app state (setup progress, session registry) -├── app.yaml.template # Databricks Apps deployment config template +├── app.yaml # Databricks Apps deployment config (uvicorn entrypoint) ├── cli_auth.py # Interactive PAT setup + CLI credential writer ├── content_filter_proxy.py # Proxy that sanitises empty-content blocks for OpenCode -├── gunicorn.conf.py # Gunicorn production server config +├── gunicorn.conf.py # Legacy WSGI-only config (unused in production; uvicorn is the entrypoint) ├── pat_rotator.py # Background PAT auto-rotation (10-min cycle) ├── pyproject.toml # Package metadata + uv config (supply-chain guardrails) ├── requirements.txt # Compiled from pyproject.toml (Dependabot compatibility) ├── requirements.lock # Hash-pinned lockfile (auto-regenerated by CI) ├── Makefile # Deploy, redeploy, status, and cleanup targets -├── setup_claude.py # Claude Code CLI + MCP configuration -├── setup_codex.py # Codex CLI configuration -├── setup_gemini.py # Gemini CLI configuration -├── setup_opencode.py # OpenCode configuration -├── setup_databricks.py # Databricks CLI configuration -├── setup_mlflow.py # MLflow tracing auto-configuration -├── setup_proxy.py # Content-filter proxy startup ├── sync_to_workspace.py # Post-commit hook: sync to Workspace -├── install_micro.sh # Micro editor installer -├── install_gh.sh # GitHub CLI installer (OS/arch-aware) -├── install_databricks_cli.sh # Databricks CLI upgrade script -├── utils.py # Utility functions (ensure_https) +├── utils.py # Utility functions (ensure_https, gateway discovery) +├── coda_mcp/ # MCP server package (CoDA — Coding Agents) +│ ├── __init__.py +│ ├── mcp_server.py # FastMCP tool definitions (coda_run, coda_inbox, coda_get_result) +│ ├── mcp_endpoint.py # Flask Blueprint: JSON-RPC /mcp endpoint +│ ├── mcp_asgi.py # ASGI bridge (optional, for native MCP SDK transport) +│ └── task_manager.py # Disk-based session/task state manager +├── setup/ # Agent setup scripts (run at boot) +│ ├── setup_claude.py # Claude Code CLI + MCP configuration +│ ├── setup_codex.py # Codex CLI configuration +│ ├── setup_gemini.py # Gemini CLI configuration +│ ├── setup_opencode.py # OpenCode configuration +│ ├── setup_hermes.py # Hermes Agent configuration +│ ├── setup_databricks.py # Databricks CLI configuration +│ ├── setup_mlflow.py # MLflow tracing auto-configuration +│ └── setup_proxy.py # Content-filter proxy startup +├── scripts/ # Shell scripts +│ ├── install_micro.sh # Micro editor installer +│ ├── install_gh.sh # GitHub CLI installer (OS/arch-aware) +│ └── install_databricks_cli.sh # Databricks CLI upgrade script ├── static/ │ ├── index.html # Terminal UI (xterm.js + split panes + WebSocket) │ ├── favicon.svg # App favicon @@ -350,8 +445,12 @@ coding-agents-databricks-apps/ │ └── workflows/ │ ├── dependency-audit.yml # Weekly CVE audit + lockfile drift check │ └── update-lockfile.yml # Auto-regenerate requirements.lock on push +├── tools/ +│ └── coda-bridge.py # Stdio-to-HTTP MCP bridge (OAuth token injection) └── docs/ ├── deployment.md # Full Databricks Apps deployment guide + ├── mcp-client-setup.md # MCP client setup guide (bridge config) + ├── mcp-v2-background-execution.md # MCP server design doc ├── prd/ # Product requirement documents └── plans/ # Design documentation ``` @@ -362,4 +461,4 @@ coding-agents-databricks-apps/ ## Technologies -Flask · Flask-SocketIO · Socket.IO · Gunicorn · xterm.js · Python PTY · uv · Databricks SDK · Databricks AI Gateway · MLflow +Flask · Flask-SocketIO · Socket.IO · uvicorn · MCP (Streamable HTTP) · xterm.js · Python PTY · uv · Databricks SDK · Databricks AI Gateway · MLflow diff --git a/app.py b/app.py index b5acb65..9a39beb 100644 --- a/app.py +++ b/app.py @@ -1,3 +1,4 @@ +import asyncio import os import pty import fcntl @@ -46,6 +47,7 @@ CLEANUP_INTERVAL_SECONDS = 900 # Check for stale sessions every 15 min GRACEFUL_SHUTDOWN_WAIT = 3 # Seconds to wait after SIGHUP before SIGKILL MAX_CONCURRENT_SESSIONS = int(os.environ.get("MAX_CONCURRENT_SESSIONS", "5")) +TRANSCRIPT_CAP_BYTES = 10 * 1024 * 1024 # 10 MB soft cap per transcript # Logging setup logging.basicConfig(level=logging.INFO) @@ -58,8 +60,46 @@ app.config['MAX_CONTENT_LENGTH'] = 32 * 1024 * 1024 # 32 MB — aligned with Claude Code's 30 MB file limit # WebSocket support via Flask-SocketIO (simple-websocket transport, threading mode) +# Used for local dev (python app.py). Under uvicorn/ASGI, the AsyncServer in +# mcp_asgi.py intercepts /socket.io/ before WSGIMiddleware, so these handlers +# are only active in WSGI mode. socketio = SocketIO(app, async_mode='threading', cors_allowed_origins=[], logger=False, engineio_logger=False) +# ── ASGI WebSocket support (python-socketio AsyncServer) ───────────── +# Set by mcp_asgi.py at startup. Background threads use _emit_from_thread() +# which routes to the async server (ASGI) or Flask-SocketIO (WSGI) automatically. +_async_sio = None +_event_loop = None + + +def set_async_sio(sio_instance, loop): + """Called by mcp_asgi.py to wire up the ASGI Socket.IO server.""" + global _async_sio, _event_loop + _async_sio = sio_instance + _event_loop = loop + + +def _emit_from_thread(event, data, room=None): + """Thread-safe emit for background threads (PTY reader, cleanup, SIGTERM). + + Routes to AsyncServer (ASGI mode) or Flask-SocketIO (WSGI mode) automatically. + """ + if _async_sio and _event_loop and _event_loop.is_running(): + try: + asyncio.run_coroutine_threadsafe( + _async_sio.emit(event, data, room=room), + _event_loop, + ) + except Exception: + pass + else: + # WSGI mode (local dev) — use Flask-SocketIO directly + try: + socketio.emit(event, data, room=room) + except Exception: + pass + + # Store sessions: {session_id: {"master_fd": fd, "pid": pid, "output_buffer": deque, "lock": Lock, ...}} # sessions_lock guards dict-level ops (add/remove/iterate); each session["lock"] guards per-session state sessions = {} @@ -86,10 +126,7 @@ def handle_sigterm(signum, frame): shutting_down = True logger.info("SIGTERM received — setting shutting_down flag for clients") # Notify WS clients immediately (HTTP poll clients will see shutting_down on next poll) - try: - socketio.emit('shutting_down', {}) - except Exception: - pass + _emit_from_thread('shutting_down', {}) # NOTE: Do not register SIGTERM handler at module level. # It is installed in initialize_app() for gunicorn only. @@ -150,6 +187,11 @@ def _run_step(step_id, command): env.pop("DATABRICKS_CLIENT_ID", None) env.pop("DATABRICKS_CLIENT_SECRET", None) + # Ensure setup scripts can still import from repo root (e.g. `from utils import ...`) + app_dir = os.path.dirname(os.path.abspath(__file__)) + existing_pp = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = f"{app_dir}:{existing_pp}" if existing_pp else app_dir + result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=300) if result.returncode == 0: _update_step(step_id, status="complete", completed_at=time.time()) @@ -370,8 +412,14 @@ def _configure_all_cli_auth(token): # 3. Re-run Codex, OpenCode, Gemini setup scripts with token in env # They are idempotent: detect CLI already installed, just write config files - env = {**os.environ, "DATABRICKS_TOKEN": token} - for script in ["setup_codex.py", "setup_opencode.py", "setup_gemini.py", "setup_hermes.py"]: + app_dir = os.path.dirname(os.path.abspath(__file__)) + existing_pp = os.environ.get("PYTHONPATH", "") + env = { + **os.environ, + "DATABRICKS_TOKEN": token, + "PYTHONPATH": f"{app_dir}:{existing_pp}" if existing_pp else app_dir, + } + for script in ["setup/setup_codex.py", "setup/setup_opencode.py", "setup/setup_gemini.py", "setup/setup_hermes.py"]: try: result = subprocess.run( ["uv", "run", "python", script], @@ -410,26 +458,26 @@ def run_setup(): _update_step("git", status="error", completed_at=time.time(), error=str(e)) _run_step("micro", ["bash", "-c", - "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true"]) + "mkdir -p ~/.local/bin && bash scripts/install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true"]) - _run_step("gh", ["bash", "install_gh.sh"]) + _run_step("gh", ["bash", "scripts/install_gh.sh"]) # --- Upgrade Databricks CLI (runtime image ships an older version) --- - _run_step("dbcli", ["bash", "install_databricks_cli.sh"]) + _run_step("dbcli", ["bash", "scripts/install_databricks_cli.sh"]) # --- Content-filter proxy (must be running before OpenCode starts) --- # Sanitizes requests/responses between OpenCode and Databricks # (see OpenCode #5028, docs/plans/2026-03-11-litellm-empty-content-blocks-design.md) - _run_step("proxy", ["uv", "run", "python", "setup_proxy.py"]) + _run_step("proxy", ["uv", "run", "python", "setup/setup_proxy.py"]) # --- Parallel agent setup (all independent of each other) --- parallel_steps = [ - ("claude", ["uv", "run", "python", "setup_claude.py"]), - ("codex", ["uv", "run", "python", "setup_codex.py"]), - ("opencode", ["uv", "run", "python", "setup_opencode.py"]), - ("gemini", ["uv", "run", "python", "setup_gemini.py"]), - ("hermes", ["uv", "run", "python", "setup_hermes.py"]), - ("databricks", ["uv", "run", "python", "setup_databricks.py"]), + ("claude", ["uv", "run", "python", "setup/setup_claude.py"]), + ("codex", ["uv", "run", "python", "setup/setup_codex.py"]), + ("opencode", ["uv", "run", "python", "setup/setup_opencode.py"]), + ("gemini", ["uv", "run", "python", "setup/setup_gemini.py"]), + ("hermes", ["uv", "run", "python", "setup/setup_hermes.py"]), + ("databricks", ["uv", "run", "python", "setup/setup_databricks.py"]), ] with ThreadPoolExecutor(max_workers=len(parallel_steps)) as executor: @@ -442,7 +490,7 @@ def run_setup(): # --- MLflow setup runs AFTER claude setup to avoid settings.json race --- # setup_mlflow.py merges env vars into ~/.claude/settings.json which # setup_claude.py also writes; running sequentially prevents clobbering. - _run_step("mlflow", ["uv", "run", "python", "setup_mlflow.py"]) + _run_step("mlflow", ["uv", "run", "python", "setup/setup_mlflow.py"]) # Sync latest token into all CLI configs — covers the race where PAT # rotation happened while a setup script was still installing (the @@ -580,7 +628,132 @@ def _check_ws_authorization(): return True -# ── WebSocket Event Handlers ────────────────────────────────────────────── +def _check_ws_authorization_from_environ(environ): + """Check authorization from WSGI environ dict (for ASGI WebSocket via python-socketio). + + Same logic as _check_ws_authorization() but reads headers from the environ + dict instead of Flask's request context. WSGI environ stores HTTP headers as + HTTP_X_FORWARDED_EMAIL (uppercase, underscores, HTTP_ prefix). + """ + if not app_owner: + if _is_databricks_apps(): + logger.error("SECURITY: app_owner not resolved — denying WebSocket (fail-closed)") + return False + return True # Local dev only + + raw_user = ( + environ.get("HTTP_X_FORWARDED_EMAIL") + or environ.get("HTTP_X_FORWARDED_USER") + or environ.get("HTTP_X_DATABRICKS_USER_EMAIL") + ) + current_user = raw_user.lower() if raw_user else raw_user + + if not current_user: + if _is_databricks_apps(): + logger.warning("No user identity in WebSocket request on Databricks Apps — denying") + return False + return True # Local dev only + + if current_user != app_owner: + logger.warning(f"WebSocket unauthorized: {current_user} (owner: {app_owner})") + return False + return True + + +def register_sio_handlers(sio): + """Register Socket.IO event handlers on an AsyncServer for ASGI mode. + + Called by mcp_asgi.py. The handlers mirror the Flask-SocketIO handlers below + but use python-socketio's async API (explicit sid, enter_room/leave_room, + async def, ConnectionRefusedError for auth denial). + """ + + @sio.on('connect') + async def handle_connect(sid, environ, auth): + # Capture event loop on first connection for _emit_from_thread() + set_async_sio(sio, asyncio.get_running_loop()) + + # Diagnostic: log transport and header presence for debugging proxy behavior + transport = environ.get('QUERY_STRING', '') + has_email = bool(environ.get('HTTP_X_FORWARDED_EMAIL')) + has_user = bool(environ.get('HTTP_X_FORWARDED_USER')) + logger.info(f"WS connect: sid={sid}, qs={transport}, " + f"has_email={has_email}, has_user={has_user}") + + if not _check_ws_authorization_from_environ(environ): + raise ConnectionRefusedError('unauthorized') + logger.info("WebSocket client connected (ASGI)") + + @sio.on('join_session') + async def handle_join_session(sid, data): + session_id = data.get('session_id') + if not session_id: + return {'status': 'error', 'message': 'session_id required'} + sess = _get_session(session_id) + if not sess: + return {'status': 'error', 'message': 'Session not found'} + with sess["lock"]: + sess["last_poll_time"] = time.time() + sess["output_buffer"].clear() + await sio.enter_room(sid, session_id) + logger.info(f"WebSocket client joined session room {session_id}") + return {'status': 'ok'} + + @sio.on('leave_session') + async def handle_leave_session(sid, data): + session_id = data.get('session_id') + if session_id: + await sio.leave_room(sid, session_id) + logger.info(f"WebSocket client left session room {session_id}") + + @sio.on('terminal_input') + async def handle_terminal_input(sid, data): + session_id = data.get('session_id') + input_data = data.get('input', '') + sess = _get_session(session_id) + if not sess: + return + with sess["lock"]: + sess["last_poll_time"] = time.time() + fd = sess["master_fd"] + try: + os.write(fd, input_data.encode()) + except OSError as e: + logger.warning(f"WebSocket input write error for {session_id}: {e}") + + @sio.on('terminal_resize') + async def handle_terminal_resize(sid, data): + session_id = data.get('session_id') + cols = data.get('cols', 80) + rows = data.get('rows', 24) + sess = _get_session(session_id) + if not sess: + return + with sess["lock"]: + sess["last_poll_time"] = time.time() + fd = sess["master_fd"] + try: + winsize = struct.pack("HHHH", rows, cols, 0, 0) + fcntl.ioctl(fd, termios.TIOCSWINSZ, winsize) + except OSError as e: + logger.warning(f"WebSocket resize error for {session_id}: {e}") + + @sio.on('heartbeat') + async def handle_heartbeat(sid, data): + session_ids = data.get('session_ids', []) + now = time.time() + for s_id in session_ids: + sess = _get_session(s_id) + if sess: + with sess["lock"]: + sess["last_poll_time"] = now + + @sio.on('disconnect') + async def handle_disconnect(sid): + logger.info("WebSocket client disconnected (ASGI)") + + +# ── WebSocket Event Handlers (Flask-SocketIO — WSGI/local dev only) ────── @socketio.on('connect') def handle_ws_connect(): @@ -686,6 +859,42 @@ def _get_session(session_id): return sessions.get(session_id) +def _tee_transcript_chunk(session, output: bytes, cap: int = TRANSCRIPT_CAP_BYTES) -> None: + """Append PTY output to the transcript file. Single-writer (read_pty_output). + + All file-handle access is under ``session["lock"]`` so we never race the + Timer-driven close path in ``terminate_session``. The ``ValueError`` catch + is belt-and-suspenders for the tiny window where the handle is closed + between the ``is not None`` check and the actual ``write`` call (the lock + prevents this, but be defensive). + """ + with session["lock"]: + fh = session.get("transcript_fh") + written = session.get("transcript_bytes", 0) + if fh is None: + return + remaining = cap - written + if remaining <= 0: + return + chunk = output[:remaining] + try: + fh.write(chunk) + fh.flush() + session["transcript_bytes"] = written + len(chunk) + if len(chunk) < len(output): + fh.write(b"\n[transcript truncated at %d bytes]\n" % cap) + fh.flush() + fh.close() + session["transcript_fh"] = None + except (OSError, ValueError) as exc: + logger.warning("transcript write failed: %s", exc) + try: + fh.close() + except Exception: + pass + session["transcript_fh"] = None + + def read_pty_output(session_id, fd): """Background thread to read PTY output into buffer and push via WebSocket.""" session = _get_session(session_id) @@ -711,12 +920,11 @@ def read_pty_output(session_id, fd): session["output_buffer"].append(decoded) session["last_poll_time"] = time.time() # Keep session alive during WS output # Push via WebSocket to the session room (AC-8) - try: - socketio.emit('terminal_output', + _emit_from_thread('terminal_output', {'session_id': session_id, 'output': decoded}, room=session_id) - except Exception: - pass # No WebSocket clients — HTTP polling handles it + # Tee to transcript file if enabled for this session + _tee_transcript_chunk(session, output) else: # select timed out — check if process is still alive try: @@ -731,10 +939,7 @@ def read_pty_output(session_id, fd): break # Process exited or fd closed — notify WebSocket clients (AC-9) - try: - socketio.emit('session_exited', {'session_id': session_id}, room=session_id) - except Exception: - pass + _emit_from_thread('session_exited', {'session_id': session_id}, room=session_id) logger.info(f"Session {session_id} process exited") @@ -744,14 +949,38 @@ def read_pty_output(session_id, fd): def terminate_session(session_id, pid, master_fd): - """Gracefully terminate a session: SIGHUP -> wait -> SIGKILL -> cleanup.""" + """Gracefully terminate a session: SIGHUP -> wait -> SIGKILL -> cleanup. + + Idempotent. Both the explicit close path (``mcp_close_pty_session``) and the + read-thread exit path (``read_pty_output``) call this for the same session. + We atomically *claim* the session by popping it from ``sessions`` — only the + caller that wins the pop kills the process and closes ``master_fd``. This + guarantees ``os.close()`` runs exactly once: a second close could land on a + since-reused fd (e.g. an asyncio event loop's self-pipe allocated by a later + test) and corrupt unrelated I/O, surfacing as intermittent EBADF. + """ + # Atomically claim the session. If it's already gone, the other teardown + # path handled it — bail out WITHOUT touching the (possibly reused) fd. + with sessions_lock: + sess = sessions.pop(session_id, None) + if sess is None: + return + logger.info(f"Terminating stale session {session_id} (pid={pid})") # Notify WebSocket clients that the session is closed - try: - socketio.emit('session_closed', {'session_id': session_id}, room=session_id) - except Exception: - pass + _emit_from_thread('session_closed', {'session_id': session_id}, room=session_id) + + # Close transcript handle (if any) under per-session lock; swap-then-close + # outside the lock to avoid blocking on slow filesystems. + with sess["lock"]: + transcript_fh = sess.get("transcript_fh") + sess["transcript_fh"] = None + if transcript_fh is not None: + try: + transcript_fh.close() + except Exception: + pass try: os.kill(pid, signal.SIGHUP) @@ -769,8 +998,21 @@ def terminate_session(session_id, pid, master_fd): except OSError: pass # Process or fd already gone - with sessions_lock: - sessions.pop(session_id, None) + # Clean up the project dir if coda_interactive created one. + # Done here (not in mcp_close_pty_session) so BOTH the graceful close + # path AND the idle reaper (which calls terminate_session directly) hit + # this cleanup. Safe for HTTP-created sessions too — they never planted + # a dir at this path, so os.path.isdir short-circuits. + import shutil + project_dir = os.path.join( + os.path.expanduser("~/.coda/projects"), + session_id, + ) + if os.path.isdir(project_dir): + try: + shutil.rmtree(project_dir) + except OSError as e: + logger.warning("Failed to clean up project dir %s: %s", project_dir, e) def _get_session_process(pid): @@ -858,7 +1100,7 @@ def cleanup_stale_sessions(): def authorize_request(): """Check authorization before processing any request.""" # Skip auth for health check, setup status, and Socket.IO (has own auth via connect event) - if request.path in ("/health", "/api/setup-status", "/api/pat-status", "/api/configure-pat", "/api/app-state") or request.path.startswith("/socket.io"): + if request.path in ("/health", "/api/setup-status", "/api/pat-status", "/api/configure-pat", "/api/app-state") or request.path.startswith("/socket.io") or request.path.startswith("/mcp"): return None authorized, user = check_authorization() @@ -873,6 +1115,10 @@ def authorize_request(): @app.after_request def set_security_headers(response): + # MCP endpoint handles its own CORS/headers — skip security headers + # that might interfere (CSP connect-src, X-Frame-Options, etc.) + if request.path.startswith("/mcp"): + return response response.headers["X-Content-Type-Options"] = "nosniff" response.headers["X-Frame-Options"] = "DENY" response.headers["X-XSS-Protection"] = "1; mode=block" @@ -934,17 +1180,58 @@ def list_sessions(): return jsonify(result) +def _serve_transcript_replay(session_id: str): + """Serve the on-disk transcript for a PTY session as a replay response. + + Used by attach_session() in two cases: + 1. The PTY is gone (transcript-fallback path). + 2. The PTY exists but is replay_only=True (no live attach allowed). + + Returns either a Flask JSON response with replay=True, or a 404 if no + transcript exists for this pty_session_id. + """ + from coda_mcp import task_manager as _tm + tdir = _tm.find_task_dir_by_pty_session(session_id) + if tdir: + transcript = os.path.join(tdir, "transcript.log") + if os.path.isfile(transcript): + try: + with open(transcript, "rb") as f: + content = f.read() + return jsonify({ + "session_id": session_id, + "label": "hermes-mcp (replay)", + "output": [content.decode("utf-8", errors="replace")], + "replay": True, + "process": None, + "created_at": None, + }) + except OSError: + pass + return jsonify({"error": "Session not found or exited"}), 404 + + @app.route("/api/session/attach", methods=["POST"]) def attach_session(): - """Reattach to an existing session — returns buffered output for replay.""" + """Reattach to an existing session — returns buffered output for replay. + + If the live PTY is gone but an on-disk transcript exists for this + pty_session_id, return the transcript as ``output`` with ``replay: True``. + """ data = request.get_json(silent=True) or {} session_id = data.get("session_id", "") sess = _get_session(session_id) + + # Replay-only sessions (e.g. those created by coda_run) always serve the + # transcript-from-disk, even when the PTY is still alive. + if sess and sess.get("replay_only"): + return _serve_transcript_replay(session_id) + if not sess or sess.get("exited"): - return jsonify({"error": "Session not found or exited"}), 404 + return _serve_transcript_replay(session_id) - # Reset idle clock so the 24h reaper starts fresh + # Existing live-attach path sess["last_poll_time"] = time.time() return jsonify({ @@ -1083,7 +1370,8 @@ def create_session(): """Create a new terminal session.""" # Quick reject before forking a PTY (approximate — authoritative check below) with sessions_lock: - if len(sessions) >= MAX_CONCURRENT_SESSIONS: + active = len(sessions) + if active >= MAX_CONCURRENT_SESSIONS: return jsonify({"error": f"Maximum {MAX_CONCURRENT_SESSIONS} concurrent sessions reached. Close an existing session first."}), 429 data = request.get_json(silent=True) or {} @@ -1122,7 +1410,8 @@ def create_session(): with sessions_lock: # Authoritative check under the same lock as insertion — prevents # TOCTOU race where two concurrent requests both pass the early check. - if len(sessions) >= MAX_CONCURRENT_SESSIONS: + active = len(sessions) + if active >= MAX_CONCURRENT_SESSIONS: os.close(master_fd) try: os.kill(pid, signal.SIGKILL) @@ -1151,6 +1440,133 @@ def create_session(): return jsonify({"error": str(e)}), 500 +# ── MCP Integration Helpers ────────────────────────────────────────── + + +def mcp_create_pty_session( + label: str = "hermes-mcp", + transcript_path: str | None = None, + replay_only: bool = False, + cwd: str | None = None, +) -> str: + """Create a PTY session for MCP use. Returns the PTY session_id.""" + with sessions_lock: + active = len(sessions) + if active >= MAX_CONCURRENT_SESSIONS: + raise RuntimeError( + f"Maximum {MAX_CONCURRENT_SESSIONS} concurrent sessions reached." + ) + + master_fd, slave_fd = pty.openpty() + + # Strip PAT, SP creds, registry tokens, and other secrets that must not be + # readable from the agent's PTY. See _build_terminal_shell_env docstring + # for the full list. (F-01) + shell_env = _build_terminal_shell_env(os.environ) + if not shell_env.get("HOME") or shell_env["HOME"] == "/": + shell_env["HOME"] = "/app/python/source_code" + local_bin = f"{shell_env['HOME']}/.local/bin" + shell_env["PATH"] = f"{local_bin}:{shell_env.get('PATH', '')}" + + projects_dir = os.path.join(shell_env["HOME"], "projects") + os.makedirs(projects_dir, exist_ok=True) + + # When caller passes cwd, use it; otherwise fall back to projects_dir + # (preserves current behavior for existing callers that don't pass cwd). + spawn_cwd = cwd if cwd is not None else projects_dir + + pid = subprocess.Popen( + ["/bin/bash"], + stdin=slave_fd, + stdout=slave_fd, + stderr=slave_fd, + preexec_fn=os.setsid, + env=shell_env, + cwd=spawn_cwd, + ).pid + os.close(slave_fd) + + # Open transcript file (if requested) before locking the session dict. + transcript_fh = None + if transcript_path: + try: + parent_dir = os.path.dirname(transcript_path) + if parent_dir: + os.makedirs(parent_dir, exist_ok=True) + transcript_fh = open(transcript_path, "ab", buffering=0) + os.fchmod(transcript_fh.fileno(), 0o600) + except OSError as exc: + logger.warning("Could not open transcript at %s: %s", transcript_path, exc) + transcript_fh = None + + session_id = str(uuid.uuid4()) + + try: + with sessions_lock: + active = len(sessions) + if active >= MAX_CONCURRENT_SESSIONS: + os.close(master_fd) + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + raise RuntimeError( + f"Maximum {MAX_CONCURRENT_SESSIONS} concurrent sessions reached." + ) + sessions[session_id] = { + "master_fd": master_fd, + "pid": pid, + "output_buffer": deque(maxlen=1000), + "lock": threading.Lock(), + "last_poll_time": time.time(), + "created_at": time.time(), + "label": label, + "transcript_path": transcript_path if transcript_fh else None, + "transcript_fh": transcript_fh, + "transcript_bytes": 0, + "replay_only": replay_only, + "cwd": cwd, + } + + thread = threading.Thread( + target=read_pty_output, args=(session_id, master_fd), daemon=True + ) + thread.start() + except BaseException: + # Roll back transcript open if anything below it raises before the + # session is fully wired. The PTY itself is cleaned up by existing + # error paths; this is just the transcript handle. + if transcript_fh is not None: + try: + transcript_fh.close() + except Exception: + pass + raise + + return session_id + + +def mcp_send_input(session_id: str, data: str): + """Send input to a PTY session.""" + session = _get_session(session_id) + if not session: + raise RuntimeError(f"Session {session_id} not found") + with session["lock"]: + os.write(session["master_fd"], data.encode()) + + +def mcp_close_pty_session(session_id: str): + """Close a PTY session. + + Project-dir cleanup (for coda_interactive sessions) lives inside + terminate_session so the idle reaper hits it too. + """ + session = _get_session(session_id) + if not session: + return + terminate_session(session_id, session["pid"], session["master_fd"]) + + @app.route("/api/input", methods=["POST"]) def send_input(): """Send input to the terminal.""" @@ -1368,6 +1784,20 @@ def initialize_app(local_dev=False): logger.info(f"Started session cleanup thread (timeout={SESSION_TIMEOUT_SECONDS}s, interval={CLEANUP_INTERVAL_SECONDS}s)") +# ── MCP Endpoint ───────────────────────────────────────────────────── +from coda_mcp.mcp_endpoint import mcp_bp +from coda_mcp.mcp_server import set_app_hooks + +app.register_blueprint(mcp_bp) + +# Wire MCP tools to PTY infrastructure +set_app_hooks( + create_session_fn=mcp_create_pty_session, + send_input_fn=mcp_send_input, + close_session_fn=mcp_close_pty_session, +) + + if __name__ == "__main__": # Local dev — no SIGTERM handler (SIG_DFL), no shutting_down flag initialize_app(local_dev=True) diff --git a/app.yaml b/app.yaml index 4d20047..b84a8bc 100644 --- a/app.yaml +++ b/app.yaml @@ -1,6 +1,17 @@ +# Production entrypoint is uvicorn (ASGI), not gunicorn. Required because +# the MCP server at /mcp uses FastMCP.streamable_http_app(), a native ASGI +# transport that cannot be served by gunicorn's WSGI workers. Flask is +# mounted via WSGIMiddleware inside coda_mcp.mcp_asgi alongside MCP and +# Socket.IO. WebSocket transport falls back to HTTP polling under uvicorn — +# acceptable because static/poll-worker.js already implements the fallback. +# gunicorn.conf.py is retained for legacy WSGI-only local dev; not used here. command: - - gunicorn - - app:app + - uvicorn + - coda_mcp.mcp_asgi:app + - --host + - 0.0.0.0 + - --port + - "8000" env: - name: HOME value: /app/python/source_code diff --git a/cli_auth.py b/cli_auth.py index 61c9f25..53c2a25 100644 --- a/cli_auth.py +++ b/cli_auth.py @@ -35,6 +35,7 @@ def _update_claude(token): settings["env"]["ANTHROPIC_AUTH_TOKEN"] = token with open(path, "w") as f: json.dump(settings, f, indent=2) + os.chmod(path, 0o600) except (OSError, json.JSONDecodeError): pass # file doesn't exist yet — initial setup hasn't run @@ -59,6 +60,7 @@ def _update_opencode(token): if changed: with open(path, "w") as f: json.dump(auth, f, indent=2) + os.chmod(path, 0o600) except (OSError, json.JSONDecodeError): pass @@ -84,6 +86,7 @@ def _update_hermes(token): if new_content != content: with open(path, "w") as f: f.write(new_content) + os.chmod(path, 0o600) except OSError: pass @@ -102,5 +105,6 @@ def _replace_dotenv_key(path, key, value): if new_content != content: with open(path, "w") as f: f.write(new_content) + os.chmod(path, 0o600) except OSError: pass diff --git a/coda_mcp/__init__.py b/coda_mcp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/coda_mcp/databricks_preamble.py b/coda_mcp/databricks_preamble.py new file mode 100644 index 0000000..46d2528 --- /dev/null +++ b/coda_mcp/databricks_preamble.py @@ -0,0 +1,124 @@ +"""Builders for the CoDA prompt envelope's CAPABILITIES and WORKFLOW PROTOCOL sections. + +These are injected into prompt.txt by ``task_manager.wrap_prompt`` when +``workflow_protocol=True``. Pure functions — no side effects, no I/O. +""" +from __future__ import annotations + + +_DATABRICKS_SKILLS: tuple[str, ...] = ( + "agent-bricks", + "databricks-genie", + "databricks-apps-python", + "databricks-ai-functions", + "databricks-jobs", + "databricks-unity-catalog", + "spark-declarative-pipelines", + "aibi-dashboards", + "model-serving", + "mlflow-evaluation", + "databricks-bundles", + "databricks-python-sdk", + "databricks-config", + "databricks-docs", + "synthetic-data-gen", + "unstructured-pdf-generation", +) + + +def get_databricks_skills() -> tuple[str, ...]: + """Return the canonical Databricks skill list. Tests pin this against CLAUDE.md.""" + return _DATABRICKS_SKILLS + + +def build_capabilities() -> str: + """Orientation block: CLI, skills, MCP servers, when to prefer Databricks-native paths.""" + skills_lines = [] + # Pack 4 skills per line for readability in prompt.txt. + for i in range(0, len(_DATABRICKS_SKILLS), 4): + chunk = _DATABRICKS_SKILLS[i:i + 4] + skills_lines.append("- " + ", ".join(chunk)) + skills_block = "\n".join(skills_lines) + return ( + "You are running inside CoDA on a Databricks-authenticated host.\n" + "\n" + "Databricks CLI: pre-configured. `databricks current-user me` confirms auth.\n" + "Use it for jobs, workspace, clusters, warehouses, Unity Catalog operations.\n" + "\n" + "Skills available at ~/.claude/skills/ — read each skill's SKILL.md before\n" + "invoking. Relevant Databricks skills:\n" + f"{skills_block}\n" + "\n" + "MCP servers wired:\n" + "- DeepWiki — ask_question, read_wiki_contents for any GitHub repo\n" + "- Exa — web_search_exa, web_fetch_exa for live web context\n" + "- CoDA — chain follow-up tasks via previous_session_id\n" + "\n" + "When the task touches Databricks data, pipelines, jobs, dashboards, agents,\n" + "or model serving, DEFAULT to the skill / CLI / SDK path above instead of\n" + "generic Python or web search." + ) + + +def build_workflow_protocol() -> str: + """3-phase workflow with critique at each phase + info_needed escape hatch.""" + return ( + "You MUST process this task in three phases. Emit status.jsonl events as\n" + "you go (one JSON object per line, format below).\n" + "\n" + "PHASE 1 — PLAN\n" + "- Write a step-by-step plan as a status.jsonl line with step=\"plan\" and\n" + " message containing the numbered steps.\n" + "- Then critique your own plan as if you were a separate reviewer.\n" + " (Spawn a sub-agent for the critique if your agent supports it; otherwise\n" + " write the critique inline as a self-review.) Emit step=\"critique_plan\"\n" + " with the verdict (APPROVE / BLOCK / APPROVE-WITH-FIXES) and findings.\n" + "- If the critique surfaces blockers, revise the plan once and re-emit\n" + " step=\"plan\". Maximum 2 plan iterations total.\n" + "- If after 2 attempts you still cannot produce a viable plan, write\n" + " result.json with status=\"info_needed\" (see below) and stop.\n" + "\n" + "PHASE 2 — EXECUTE\n" + "- Work the plan. Emit step=\"execute_\" lines after completing each plan\n" + " step (n is 1-indexed, matches the plan's numbering).\n" + "- After execution, emit step=\"critique_execute\" with a review of what got\n" + " built vs what the plan said. APPROVE / BLOCK / APPROVE-WITH-FIXES.\n" + "- If the critique surfaces correctness or scope gaps, fix them and re-emit\n" + " step=\"critique_execute\". Maximum 2 execute iterations total.\n" + "- If you hit a hard blocker (missing access, missing data, ambiguous\n" + " requirements that the plan revealed only mid-execution), write\n" + " result.json with status=\"info_needed\" and stop.\n" + "\n" + "PHASE 3 — SYNTHESIZE\n" + "- Write result.json with status=\"completed\".\n" + "- Emit step=\"critique_synthesize\" with a review of the result against the\n" + " original TASK.\n" + "- If the critique surfaces gaps, revise result.json. Maximum 2 synthesis\n" + " iterations total.\n" + "\n" + "If at any phase you cannot proceed, use the INFO_NEEDED escape hatch:\n" + "- Set status=\"info_needed\" in result.json.\n" + "- Set \"feedback\" to a precise, actionable string naming exactly what is\n" + " missing (a table name, a decision, an access grant, a clarification).\n" + " The calling client will read this and resubmit with the missing context.\n" + "- \"info_needed\" is NOT a failure — it is a structured request for\n" + " iteration. Use it whenever you would otherwise have to guess.\n" + "\n" + "If you encounter a hard, unrecoverable failure (a command crashed, an SDK\n" + "returned 500, a file is corrupt), use status=\"failed\" with a description\n" + "in \"errors\".\n" + "\n" + "DISAMBIGUATION — two soft statuses already exist and they mean different\n" + "things; use the right one:\n" + "- \"info_needed\" — the CALLER must add missing context (table name,\n" + " business decision, file contents, access grant) before the task can\n" + " proceed. Used when ambiguity or missing input blocks you.\n" + "- \"needs_approval\" — you have a concrete plan to do something destructive\n" + " (drop a table, delete a job, modify permissions). You will execute it\n" + " if and only if the caller explicitly approves. Used at the SAFETY\n" + " boundary, never for ambiguity. See SAFETY section below.\n" + "\n" + "If both apply (e.g. \"I'd drop a table but I'm not sure which one\"), prefer\n" + "\"info_needed\" — resolving the ambiguity first is cheaper than approving\n" + "the wrong destructive action." + ) diff --git a/coda_mcp/mcp_asgi.py b/coda_mcp/mcp_asgi.py new file mode 100644 index 0000000..f745e32 --- /dev/null +++ b/coda_mcp/mcp_asgi.py @@ -0,0 +1,122 @@ +"""Native MCP ASGI app with WebSocket support for terminal I/O. + +Architecture (all on one port, one uvicorn process): + + socketio.ASGIApp ← /socket.io/ → native ASGI WebSocket (terminal) + └── mcp_starlette ← /mcp → FastMCP Streamable HTTP (Genie Code) + └── WSGI(Flask) ← /* → REST API, static files (HTTP only) + +Usage in app.yaml:: + + command: ["uvicorn", "coda_mcp.mcp_asgi:app", "--host", "0.0.0.0", "--port", "8000"] +""" + +import os +import logging +import warnings + +import socketio as socketio_lib +from starlette.middleware.cors import CORSMiddleware + +with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + from starlette.middleware.wsgi import WSGIMiddleware + +from coda_mcp.mcp_server import mcp as mcp_instance, set_app_hooks +from coda_mcp import url_builder +from utils import ensure_https + +logger = logging.getLogger(__name__) + + +class AppUrlCaptureMiddleware: + """Capture X-Forwarded-Host (or Host) from every inbound HTTP request and + populate url_builder._app_url_cache. Used so MCP tools can return a + working viewer_url without manual configuration. + + Caveat: /socket.io/ traffic is intercepted by socketio.ASGIApp *before* + reaching mcp_starlette, so WebSocket connect requests never hit this + middleware. This is fine in practice — every HTTP request to /mcp and to + Flask routes does hit it, which is enough to keep the cache hot. + """ + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope.get("type") == "http": + headers = dict(scope.get("headers") or []) + host_bytes = headers.get(b"x-forwarded-host") or headers.get(b"host") + if host_bytes: + try: + url_builder.capture_from_headers(host_bytes.decode("latin-1")) + except Exception: + pass + await self.app(scope, receive, send) + +# ── Build allowed origins ───────────────────────────────────────── +# The browser connects from the app's own URL (e.g. mcp-test-coda-*.databricksapps.com) +# which differs from DATABRICKS_HOST (workspace URL). Databricks proxy handles auth, +# so Socket.IO CORS can safely allow all origins. Starlette CORSMiddleware below +# uses the same list for MCP/Flask routes. +_databricks_host = os.environ.get("DATABRICKS_HOST", "") +ALLOWED_ORIGINS = [] +if _databricks_host: + ALLOWED_ORIGINS.append(ensure_https(_databricks_host).rstrip("/")) + +# ── Import and initialize Flask app ──────────────────────────────── +from app import ( + app as flask_app, + initialize_app, + mcp_create_pty_session, + mcp_send_input, + mcp_close_pty_session, + register_sio_handlers, +) + +initialize_app() + +# Wire MCP tools to PTY infrastructure +set_app_hooks( + create_session_fn=mcp_create_pty_session, + send_input_fn=mcp_send_input, + close_session_fn=mcp_close_pty_session, +) + +# ── Async Socket.IO server (native ASGI WebSocket) ─────────────── +# python-socketio AsyncServer handles /socket.io/ with real WebSocket, +# eliminating the WSGIMiddleware limitation that forced HTTP polling fallback. +sio = socketio_lib.AsyncServer( + async_mode='asgi', + cors_allowed_origins='*', # App URL differs from DATABRICKS_HOST; proxy handles auth + logger=False, + engineio_logger=False, +) + +# Register terminal I/O event handlers (connect, join_session, terminal_input, etc.) +register_sio_handlers(sio) + +# ── Build the ASGI app per Genie Code docs ───────────────────────── +mcp_starlette = mcp_instance.streamable_http_app() + +# Mount Flask as catch-all via WSGI adapter (HTTP routes only) +flask_asgi = WSGIMiddleware(flask_app.wsgi_app) +mcp_starlette.mount("/", app=flask_asgi) + +# CORS for MCP and Flask routes +mcp_starlette.add_middleware( + CORSMiddleware, + allow_origins=ALLOWED_ORIGINS or ["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Capture X-Forwarded-Host into url_builder cache (for MCP viewer_url). +# Added AFTER CORS so it wraps the CORS-handled request. +mcp_starlette.add_middleware(AppUrlCaptureMiddleware) + +# ── Top-level ASGI app ──────────────────────────────────────────── +# socketio.ASGIApp intercepts /socket.io/ for WebSocket + polling, +# passes everything else to mcp_starlette (MCP at /mcp, Flask at /) +app = socketio_lib.ASGIApp(sio, other_asgi_app=mcp_starlette) diff --git a/coda_mcp/mcp_endpoint.py b/coda_mcp/mcp_endpoint.py new file mode 100644 index 0000000..d7730f1 --- /dev/null +++ b/coda_mcp/mcp_endpoint.py @@ -0,0 +1,179 @@ +"""Flask Blueprint fallback for MCP JSON-RPC. + +NOTE: This is NOT the production path. Production deployment uses +`coda_mcp.mcp_asgi:app` served by uvicorn, which mounts the native MCP +SDK Streamable HTTP transport at /mcp. This module is a Flask-native +JSON-RPC fallback used only under WSGI runtimes (gunicorn local dev, +tests that exercise the Flask test client without spinning up ASGI). + +Both paths expose the same four tools (coda_run, coda_inbox, +coda_get_result, coda_interactive) and produce equivalent JSON-RPC responses, +so switching between them is transparent to MCP clients. +""" +import asyncio +import json +import logging +from flask import Blueprint, request, jsonify + +logger = logging.getLogger(__name__) + +mcp_bp = Blueprint("mcp", __name__) + +# Import tool functions from mcp_server.py +from coda_mcp.mcp_server import ( + mcp as mcp_instance, + coda_run, + coda_inbox, + coda_get_result, + coda_interactive, +) + +# Tool function dispatch +_TOOL_DISPATCH = { + "coda_run": coda_run, + "coda_inbox": coda_inbox, + "coda_get_result": coda_get_result, + "coda_interactive": coda_interactive, +} + +SERVER_INFO = { + "name": "coda", + "version": "1.0.0", +} + +CAPABILITIES = { + "tools": {"listChanged": False}, +} + + + +def _cors_headers(): + """Build CORS response headers. + + Permissive CORS for /mcp — the Databricks Apps proxy handles auth. + """ + headers = {} + origin = request.headers.get("Origin", "") + if origin: + headers["Access-Control-Allow-Origin"] = origin + headers["Access-Control-Allow-Methods"] = "GET, POST, DELETE, OPTIONS" + # Explicitly list all headers Genie Code might send + # (wildcard * is incompatible with credentials=true per CORS spec) + allowed_headers = ", ".join([ + "Content-Type", "Authorization", "Accept", + "Mcp-Session-Id", "X-Request-Id", "X-Requested-With", + "X-Forwarded-Email", "X-Forwarded-User", "X-Databricks-User-Email", + "Cookie", "Origin", "Referer", + ]) + headers["Access-Control-Allow-Headers"] = allowed_headers + headers["Access-Control-Allow-Credentials"] = "true" + headers["Access-Control-Max-Age"] = "86400" + return headers + + +@mcp_bp.route("/mcp", methods=["POST", "OPTIONS", "GET"]) +def mcp_handler(): + # Handle CORS preflight + if request.method == "OPTIONS": + resp = jsonify({}) + resp.status_code = 204 + for k, v in _cors_headers().items(): + resp.headers[k] = v + return resp + + # Handle GET for SSE (not supported in stateless mode) + if request.method == "GET": + resp = jsonify({"error": "SSE not supported. Use POST."}) + resp.status_code = 405 + return resp + + # Origin validation skipped — Databricks Apps proxy handles auth. + + data = request.get_json(silent=True) or {} + method = data.get("method", "") + req_id = data.get("id") + params = data.get("params", {}) + + # Route by method + if method == "initialize": + result = { + "protocolVersion": params.get("protocolVersion", "2025-03-26"), + "capabilities": CAPABILITIES, + "serverInfo": SERVER_INFO, + "instructions": mcp_instance._instructions if hasattr(mcp_instance, '_instructions') else "", + } + resp = jsonify({"jsonrpc": "2.0", "id": req_id, "result": result}) + + elif method == "notifications/initialized": + # No-op acknowledgment — return empty OK + resp = jsonify({}) + resp.status_code = 200 + + elif method == "tools/list": + tools = _build_tools_list() + resp = jsonify({"jsonrpc": "2.0", "id": req_id, "result": {"tools": tools}}) + + elif method == "tools/call": + tool_name = params.get("name", "") + arguments = params.get("arguments", {}) + tool_fn = _TOOL_DISPATCH.get(tool_name) + if not tool_fn: + resp = jsonify({ + "jsonrpc": "2.0", "id": req_id, + "error": {"code": -32601, "message": f"Unknown tool: {tool_name}"} + }) + else: + try: + # Tool functions are async — run them + result_str = asyncio.run(tool_fn(**arguments)) + result_data = json.loads(result_str) + resp = jsonify({ + "jsonrpc": "2.0", "id": req_id, + "result": { + "content": [{"type": "text", "text": result_str}], + "isError": "error" in result_data, + } + }) + except Exception as e: + resp = jsonify({ + "jsonrpc": "2.0", "id": req_id, + "error": {"code": -32603, "message": str(e)} + }) + + elif method == "ping": + resp = jsonify({"jsonrpc": "2.0", "id": req_id, "result": {}}) + + else: + resp = jsonify({ + "jsonrpc": "2.0", "id": req_id, + "error": {"code": -32601, "message": f"Method not found: {method}"} + }) + + # Add CORS headers + for k, v in _cors_headers().items(): + resp.headers[k] = v + + return resp + + +def _build_tools_list(): + """Extract tool definitions from FastMCP registry.""" + tools = [] + # Access FastMCP's internal tool manager + tool_manager = mcp_instance._tool_manager + for name, tool in tool_manager._tools.items(): + tool_dict = { + "name": tool.name, + "description": tool.description or "", + "inputSchema": tool.parameters if hasattr(tool, 'parameters') else {}, + } + if hasattr(tool, 'annotations') and tool.annotations: + tool_dict["annotations"] = {} + if tool.annotations.readOnlyHint is not None: + tool_dict["annotations"]["readOnlyHint"] = tool.annotations.readOnlyHint + if tool.annotations.destructiveHint is not None: + tool_dict["annotations"]["destructiveHint"] = tool.annotations.destructiveHint + if tool.annotations.idempotentHint is not None: + tool_dict["annotations"]["idempotentHint"] = tool.annotations.idempotentHint + tools.append(tool_dict) + return tools diff --git a/coda_mcp/mcp_server.py b/coda_mcp/mcp_server.py new file mode 100644 index 0000000..c1a812c --- /dev/null +++ b/coda_mcp/mcp_server.py @@ -0,0 +1,760 @@ +"""MCP server exposing CoDA session/task tools via FastMCP. + +v2: Background execution + inbox pattern. +- ``coda_run`` — fire-and-forget task submission (auto-creates ephemeral session) +- ``coda_inbox`` — dashboard of all background tasks +- ``coda_get_result`` — pull full structured result for a completed task + +Delegates all disk state to ``task_manager.py``. PTY operations are +handled through app hooks (create/send/close) set via ``set_app_hooks()``. + +Run standalone for testing:: + + python mcp_server.py # stdio transport +""" + +import asyncio +import json +import logging +import os +import re +import shlex +import shutil +import threading +import time + +from mcp.server.fastmcp import FastMCP +from mcp.server.fastmcp.server import TransportSecuritySettings +from mcp.types import ToolAnnotations + +from coda_mcp import task_manager +from coda_mcp import url_builder + +logger = logging.getLogger(__name__) + +# ── FastMCP instance ──────────────────────────────────────────────── + +# Build allowed origins from DATABRICKS_HOST for Genie Code requests +_databricks_host = os.environ.get("DATABRICKS_HOST", "") +_allowed_origins = [] +if _databricks_host: + # Ensure https:// prefix, strip trailing slash + origin = _databricks_host if _databricks_host.startswith("https://") else f"https://{_databricks_host}" + _allowed_origins.append(origin.rstrip("/")) + +mcp = FastMCP( + "coda", + instructions=( + "CoDA MCP server — delegate coding tasks to AI agents on Databricks.\n\n" + "CRITICAL — FIRE AND FORGET:\n" + "coda_run submits work and returns IMMEDIATELY. The task runs autonomously " + "in the background. After calling coda_run, DO NOT call coda_inbox or " + "coda_get_result to check on it. Do NOT loop, poll, or wait. Simply tell " + "the user the task was submitted and MOVE ON to their next request.\n\n" + "WHEN TO CHECK INBOX:\n" + "Call coda_inbox ONLY when the user explicitly asks about background tasks " + "(e.g. 'how's my task going?', 'check on that', 'what's in my inbox'). " + "Never call it proactively, automatically, or in a loop.\n\n" + "WORKFLOW:\n" + "1) coda_run — submit work, get back task_id. Tell user it's running. Stop.\n" + "2) Continue chatting about other topics — the task runs independently.\n" + "3) coda_inbox — ONLY when user asks. Shows all tasks from last 24h.\n" + "4) coda_get_result — for completed tasks, get full structured output.\n\n" + "CHAINING: pass previous_session_id from a completed task's session_id " + "to give the new task context of what was done before.\n\n" + "INFO_NEEDED HANDOFF: When coda_inbox shows a task with status='info_needed', " + "the agent could not proceed because of missing context. Call coda_get_result " + "to read the 'feedback' field — it tells you exactly what the agent needs (a " + "table name, a decision, a clarification). Add that context to the prompt and " + "resubmit via coda_run with previous_session_id set to the original task's " + "session_id so the agent has the prior attempt's context. 'needs_approval' is " + "similar but means the agent has a destructive plan and is waiting for the " + "caller's explicit go/no-go.\n\n" + "SHARE THE REPLAY URL: When coda_run returns a viewer_url field (non-null), " + "mention it to the user in plain text (e.g. \"you can view the session replay " + "at \"). The URL is a read-only static replay showing the prompt, the " + "agent's work, and the final output. It reflects the task's progress while " + "running, then the full transcript once it completes — and remains valid " + "indefinitely after that. It is safe to share: it points to the same " + "Databricks App the user is already authenticated against. Do this on the " + "first mention of the task and any time the user asks where the task is or " + "how to see what it did.\n\n" + "INTERACTIVE HANDOFF (coda_interactive): When the user wants a human to " + "drive a coding agent in CoDA — not autonomous execution — call " + "coda_interactive instead of coda_run. The tool reads files from a " + "directory that already exists in the Databricks Workspace (a Git " + "Folder or a plain Workspace folder — either works). IMPORTANT: this " + "tool runs inside CoDA on Databricks and reads ONLY from the Databricks " + "Workspace — it CANNOT see your local filesystem. If you are a LOCAL " + "agent (e.g. Claude Code or Codex on the user's machine) and the project " + "files for this task live locally, you MUST first copy them into the " + "Workspace, THEN pass that Workspace path. Easiest: run `databricks " + "workspace import-dir " + "/Workspace/Users//` (Databricks CLI; the SDK " + "or REST work too), then call coda_interactive with workspace_path set to " + "that /Workspace/Users/... path. The tool does NOT accept inline file " + "payloads. If the directory is a Git " + "Folder, ensure the desired branch is checked out first — " + "the pull is a point-in-time snapshot. The tool copies the directory " + "into a Coda-local working directory using your credentials (via " + "`databricks workspace export-dir`), launches the chosen agent " + "(claude default; also hermes, codex, gemini, opencode), and types " + "the prompt as the first user input. The return shape includes a " + "viewer_url the user opens to attach — share it immediately in plain " + "text; it is the only handle to the session, and the user drives it " + "until they exit. Interactive sessions do NOT appear in coda_inbox, " + "and coda_get_result returns nothing for them — do not try to poll " + "or fetch results. Note that git history is NOT available inside the " + "session (files-only export); if the user needs history context, " + "include a git log summary in the prompt string." + ), + stateless_http=True, + json_response=True, + transport_security=TransportSecuritySettings( + enable_dns_rebinding_protection=False, + ), +) + +# ── App hooks (PTY integration) ───────────────────────────────────── + +_app_create_session = None +_app_send_input = None +_app_close_session = None + + +def set_app_hooks( + create_session_fn, + send_input_fn, + close_session_fn, +): + """Wire up Flask app callbacks for PTY operations. + + Registers the create/send/close hooks that ``coda_run`` and ``_watch_task`` + use to drive the underlying PTY session. + """ + global _app_create_session, _app_send_input, _app_close_session + _app_create_session = create_session_fn + _app_send_input = send_input_fn + _app_close_session = close_session_fn + + +# ── Background watcher ────────────────────────────────────────────── + + +def _watch_task(session_id: str, task_id: str, timeout_s: int) -> None: + """Poll for result.json in a daemon thread. + + - Checks every 5 seconds for ``result.json`` in the task directory. + - If found, calls ``task_manager.complete_task()`` (which auto-closes session). + - Tracks last activity from ``status.jsonl`` mtime. + - Timeout: if wall clock exceeds *timeout_s* AND no status update + in the last 5 minutes, writes a timeout result and completes. + - On completion, closes the PTY if hooks are wired. + """ + tdir = task_manager._task_dir(session_id, task_id) + status_path = os.path.join(tdir, "status.jsonl") + start = time.time() + stale_threshold = 300 # 5 minutes + + while True: + time.sleep(5) + + # Check for result.json (may be at root or in results/ subdir) + result_path = task_manager._find_result_json(tdir) + if result_path: + try: + task_manager.complete_task(session_id, task_id) + _close_pty_immediately(session_id) + logger.info("Watcher: task %s completed (result found)", task_id) + except Exception: + logger.exception("Watcher: error completing task %s", task_id) + return + + # Check timeout + elapsed = time.time() - start + if elapsed > timeout_s: + # Check last activity + try: + last_activity = os.path.getmtime(status_path) + except OSError: + last_activity = start + + if (time.time() - last_activity) > stale_threshold: + # Write timeout result and complete + try: + timeout_result_path = os.path.join(tdir, "result.json") + task_manager._write_json(timeout_result_path, { + "status": "timeout", + "summary": "Task timed out", + "files_changed": [], + "artifacts": [], + "errors": [f"Timeout after {timeout_s}s with no activity for 5 min"], + }) + task_manager.complete_task(session_id, task_id) + _close_pty_immediately(session_id) + logger.warning("Watcher: task %s timed out", task_id) + except Exception: + logger.exception("Watcher: error timing out task %s", task_id) + return + + +def _close_pty_immediately(session_id: str) -> None: + """Close the PTY session associated with this task session immediately. + + Called by ``_watch_task`` as soon as the task transitions to completed + or failed. Reads ``pty_session_id`` from the task-manager's session.json + and calls the ``_app_close_session`` hook (i.e. ``mcp_close_pty_session`` + in production). + """ + if _app_close_session is None: + return + try: + session = task_manager._read_session(session_id) + pty_session_id = session.get("pty_session_id") + if pty_session_id: + _app_close_session(pty_session_id) + except Exception: + logger.debug("Could not close PTY for session %s", session_id, exc_info=True) + + +# ── Tool definitions ──────────────────────────────────────────────── + + +@mcp.tool( + annotations=ToolAnnotations( + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + ), +) +async def coda_run( + prompt: str, + email: str, + context: str = "{}", + previous_session_id: str = "", + permissions: str = "smart", + timeout_s: int = 3600, + workflow_protocol: bool = True, +) -> str: + """Submit a coding task — FIRE AND FORGET. + + Returns IMMEDIATELY with a task_id. The task runs autonomously in the + background. After receiving the response, tell the user the task was + submitted and move on. Do NOT follow up with coda_inbox or coda_get_result + unless the user explicitly asks to check status later. + + ``context`` is a JSON string with Unity Catalog metadata (tables, schemas). + ``previous_session_id`` chains to a prior task's session for context continuity. + ``permissions`` can be ``"smart"`` (default, safe) or ``"yolo"`` (auto-approve all). + + ``workflow_protocol`` defaults to True, which injects a Databricks + orientation block and a 3-phase workflow protocol (PLAN/EXECUTE/SYNTHESIZE + with critique at each phase) into the agent's prompt. The protocol also + defines the ``info_needed`` terminal status for clean handoff when the + agent is blocked. Set False to skip — useful for non-Databricks tasks. + + Returns JSON with ``task_id``, ``session_id``, and ``status: "running"``. + """ + try: + # Check concurrency limit + running = task_manager.count_running_tasks() + if running >= task_manager.MAX_CONCURRENT_TASKS: + return json.dumps({ + "status": "error", + "error": f"Concurrency limit reached ({task_manager.MAX_CONCURRENT_TASKS} " + f"tasks running). Try again when a task completes.", + }) + + # Parse context JSON + try: + ctx = json.loads(context) if context else None + except json.JSONDecodeError: + return json.dumps({ + "status": "error", + "error": f"Invalid JSON in context parameter: {context!r}", + }) + + # Auto-create ephemeral session + session_result = task_manager.create_session(email, "", label="hermes-mcp") + session_id = session_result["session_id"] + + # Create task first (we need task_id to compute transcript_path). + result = task_manager.create_task( + session_id=session_id, + prompt=prompt, + email=email, + context=ctx, + timeout_s=timeout_s, + permissions=permissions, + previous_session_id=previous_session_id or None, + workflow_protocol=workflow_protocol, + ) + task_id = result["task_id"] + + pty_session_id = None + if _app_create_session is not None: + transcript_path = os.path.join( + task_manager._task_dir(session_id, task_id), + "transcript.log", + ) + pty_session_id = _app_create_session( + label="hermes-mcp", + transcript_path=transcript_path, + replay_only=True, # coda_run URLs are post-hoc review only + ) + task_manager._update_session_field( + session_id, "pty_session_id", pty_session_id + ) + + # Send to PTY if hooks are wired + if _app_send_input is not None and pty_session_id is not None: + tdir = task_manager._task_dir(session_id, task_id) + prompt_path = os.path.join(tdir, "prompt.txt") + cmd = f'hermes -z "{prompt_path}"' + if permissions == "yolo": + cmd += " --yolo" + cmd += "\n" + _app_send_input(pty_session_id, cmd) + + # Start background watcher + t = threading.Thread( + target=_watch_task, + args=(session_id, task_id, timeout_s), + daemon=True, + ) + t.start() + + return json.dumps({ + "task_id": task_id, + "session_id": session_id, + "status": "running", + "viewer_url": url_builder.build_viewer_url(pty_session_id) if pty_session_id else None, + }) + + except Exception as exc: + return json.dumps({"status": "error", "error": str(exc)}) + + +def _safe_dirname(workspace_path: str) -> str: + """Local directory name for the pulled folder = sanitized basename.""" + base = os.path.basename(workspace_path.rstrip("/")) + safe = re.sub(r"[^A-Za-z0-9._-]", "_", base) + # Reject empty and the traversal names "." / ".." — `.` and `-` are allowed + # by the regex, so a basename of ".." would otherwise make ./ escape + # or alias the project dir. + if safe in ("", ".", ".."): + return "workspace" + return safe + + +def _normalize_workspace_path(workspace_path: str) -> str: + """Canonical Workspace API path: drop the /Workspace FUSE prefix if present. + + The deployed terminal's CLI uses the unprefixed form (/Users/...); REST + accepts both, but normalizing matches what the CLI expects and is harmless. + """ + p = workspace_path.rstrip("/") + if p.startswith("/Workspace/"): + p = p[len("/Workspace"):] + return p + + +_ALLOWED_AGENTS = {"claude", "hermes", "codex", "gemini", "opencode"} + +# Wait for the agent's TUI to settle by polling the PTY output buffer. Returns +# as soon as the buffer length stays constant for _PROMPT_SEED_STABILITY_S, or +# _PROMPT_SEED_MAX_WAIT_S elapses (whichever first). Replaces a brittle +# hardcoded sleep that didn't adapt to slow agent cold-starts. +_PROMPT_SEED_MAX_WAIT_S = 5.0 +_PROMPT_SEED_STABILITY_S = 1.0 +# Terminal-side `databricks workspace export-dir` pull (coda_interactive). We wait +# for an explicit shell completion marker, NOT for output to go quiet: the +# databricks CLI cold-starts SILENTLY for ~2s before writing any files, so an +# output-quiet heuristic declares "done" too early and the disk check finds +# nothing. The pull command's tail echoes one of these tokens; they are built +# from split string literals in the command (echo "CODA""_PULL_""OK") so the +# contiguous form here appears ONLY when the echo executes — never in the +# shell's echo of the typed command line. +_PULL_MAX_WAIT_S = 60.0 +_PULL_OK = "CODA_PULL_OK" +_PULL_FAIL = "CODA_PULL_FAIL" + + +async def _wait_for_output_stable( + pty_session_id: str, max_wait: float, stability: float +) -> None: + """Poll the PTY output buffer; return when it stabilizes or ``max_wait`` elapses. + + Stability = buffer length unchanged for ``stability`` seconds, after at least + one byte has appeared. If the session disappears mid-wait (PTY died), return. + """ + from app import sessions + loop = asyncio.get_running_loop() + deadline = loop.time() + max_wait + last_len = -1 + stable_since: float | None = None + poll_interval = 0.1 + + while loop.time() < deadline: + await asyncio.sleep(poll_interval) + sess = sessions.get(pty_session_id) + if sess is None: + return + current_len = sum(len(chunk) for chunk in sess.get("output_buffer", [])) + if current_len > 0 and current_len == last_len: + if stable_since is None: + stable_since = loop.time() + elif (loop.time() - stable_since) >= stability: + return + else: + stable_since = None + last_len = current_len + + +async def _wait_for_agent_ready(pty_session_id: str) -> None: + """Wait for an agent TUI to settle (prompt-seed budget). Wrapper for back-compat.""" + await _wait_for_output_stable( + pty_session_id, _PROMPT_SEED_MAX_WAIT_S, _PROMPT_SEED_STABILITY_S + ) + + +def _buffer_text(chunks) -> str: + """Decode a PTY output_buffer (list of bytes/str chunks) into one string.""" + parts = [] + for c in chunks: + parts.append(c.decode("utf-8", "replace") if isinstance(c, (bytes, bytearray)) else str(c)) + return "".join(parts) + + +async def _wait_for_pull(pty_session_id: str, target_dir: str) -> str: + """Wait for the terminal-side export-dir pull to finish. Returns 'ok'/'fail'/'timeout'. + + Watches the PTY output for the explicit completion marker echoed by the pull + command's ``&& echo OK || echo FAIL`` tail — robust against the databricks + CLI's silent cold-start (a "wait for output to go quiet" heuristic fires + during that silence, before any files exist). On the OK marker we also + confirm the files actually landed on disk. + """ + from app import sessions + loop = asyncio.get_running_loop() + deadline = loop.time() + _PULL_MAX_WAIT_S + poll_interval = 0.2 + + while loop.time() < deadline: + await asyncio.sleep(poll_interval) + sess = sessions.get(pty_session_id) + if sess is None: + return "fail" + text = _buffer_text(sess.get("output_buffer", [])) + if _PULL_OK in text: + if os.path.isdir(target_dir) and os.listdir(target_dir): + return "ok" + # Marker present but no files — treat as failure (shouldn't happen). + return "fail" + if _PULL_FAIL in text: + return "fail" + return "timeout" + + +_AGENT_LAUNCH_CMDS = { + "claude": "claude", + "hermes": "hermes chat", + "codex": "codex", + "gemini": "gemini", + "opencode": "opencode", +} + +# Agents that launch INTERACTIVELY with an auto-accept flag (no trust/permission +# dialog) and the kickoff prompt as a positional arg. For these, coda_interactive +# launches in one atomic command — no separate prompt-seeding, no TUI-ready wait. +# claude launches in a fresh per-session dir each time, which would otherwise trip +# its per-directory folder-trust dialog and swallow the prompt. Agents not listed +# fall back to launch -> wait-for-ready -> type the prompt. +_AGENT_AUTO_LAUNCH = { + "claude": "claude --enable-auto-mode", +} + + +@mcp.tool( + annotations=ToolAnnotations( + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + ), +) +async def coda_interactive( + prompt: str, + workspace_path: str, + agent: str = "claude", + email: str = "", +) -> str: + """Launch an interactive agent session in CoDA, handed off via a viewer URL. + + The MCP caller passes a Databricks Workspace directory path. This path must + already exist in the Databricks Workspace — the tool runs inside CoDA and + CANNOT read your local filesystem. If you are a local agent and the project + files live locally, FIRST upload them, e.g. + ``databricks workspace import-dir /Workspace/Users//``, + then pass that ``/Workspace/Users/...`` path. CoDA pulls that folder onto the + session's disk IN THE TERMINAL (authenticated as you) via + ``databricks workspace export-dir``, launches the chosen agent (claude + default) in the pulled directory, seeds ``prompt`` as the first user input, + and returns a ``viewer_url`` the calling user opens to drive it. + + If the pull produces no files (bad path or no read access) the tool returns + a ``status=error`` and does not launch the agent. + + Interactive sessions do NOT appear in ``coda_inbox`` and ``coda_get_result`` + will not return anything for them. The viewer URL is the only handle. + + ``email`` is accepted for forward-compatibility and is currently unused. + + Allowed agents: claude (default), hermes, codex, gemini, opencode. + """ + if agent not in _ALLOWED_AGENTS: + return json.dumps({ + "status": "error", + "error": f"Unknown agent: {agent!r}. Allowed: {sorted(_ALLOWED_AGENTS)}", + }) + + if _app_create_session is None or _app_send_input is None: + return json.dumps({ + "status": "error", + "error": "PTY hook not wired", + }) + + pty_session_id = None + project_dir = None + try: + # Create PTY FIRST so we have its session_id for the project_dir name. + pty_session_id = _app_create_session( + label=f"{agent}-interactive", + replay_only=False, + ) + project_dir = os.path.join( + os.path.expanduser("~/.coda/projects"), + pty_session_id, + ) + os.makedirs(project_dir, exist_ok=True) + + name = _safe_dirname(workspace_path) + source_path = _normalize_workspace_path(workspace_path) + + target_dir = os.path.join(project_dir, name) + + # Pull the Workspace folder into ./ AS THE USER (terminal creds). + # The tail echoes a completion marker so we detect success/failure WITHOUT + # relying on output timing — the databricks CLI cold-starts silently for + # ~2s before writing files, so a "wait for output to go quiet" heuristic + # races it and checks the disk too early. The marker tokens are split + # across string literals (echo "CODA""_PULL_""OK") so their contiguous + # form appears in the PTY output ONLY when the echo runs, never in the + # shell's echo of the typed command line. A failed export-dir + # short-circuits the && chain, so OK never prints and || echoes FAIL. + pull_cmd = ( + f"cd {shlex.quote(project_dir)} && " + f"databricks workspace export-dir {shlex.quote(source_path)} " + f"{shlex.quote('./' + name)} && " + f"cd {shlex.quote(name)} " + f'&& echo "CODA""_PULL_""OK" || echo "CODA""_PULL_""FAIL"' + ) + _app_send_input(pty_session_id, pull_cmd + "\n") + + outcome = await _wait_for_pull(pty_session_id, target_dir) + if outcome != "ok": + if _app_close_session is not None: + try: + _app_close_session(pty_session_id) + except Exception: + pass + if os.path.isdir(project_dir): + shutil.rmtree(project_dir, ignore_errors=True) + if outcome == "timeout": + msg = ( + f"Timed out pulling files from {workspace_path} after " + f"{int(_PULL_MAX_WAIT_S)}s — the export may be very large or " + f"`databricks workspace export-dir` is hung." + ) + else: + msg = ( + f"Failed to pull files from {workspace_path}. Check the path " + f"exists in the Workspace and that you have read access " + f"(ran `databricks workspace export-dir`)." + ) + return json.dumps({"status": "error", "error": msg}) + + # Kickoff prompt with a one-line context prefix naming the source. Kept + # to ONE line so it is safe both as a quoted CLI arg and as typed input + # (an embedded newline inside a quote would trigger shell line-continuation). + seeded_prompt = ( + f"Your working directory holds files exported from the Databricks " + f"Workspace path {workspace_path}. {prompt}" + ) + + # Launch the agent. Agents in _AGENT_AUTO_LAUNCH accept an auto-accept + # flag + the prompt as a positional arg, so we launch in ONE atomic + # command: no trust/permission dialog blocks the handoff, and the prompt + # isn't subject to TUI cold-start timing. Other agents fall back to + # launch -> wait-for-ready -> type the prompt. + auto_launch = _AGENT_AUTO_LAUNCH.get(agent) + if auto_launch is not None: + _app_send_input( + pty_session_id, f"{auto_launch} {shlex.quote(seeded_prompt)}\n" + ) + else: + _app_send_input(pty_session_id, _AGENT_LAUNCH_CMDS[agent] + "\n") + await _wait_for_agent_ready(pty_session_id) + _app_send_input(pty_session_id, seeded_prompt + "\n") + + viewer_url = url_builder.build_viewer_url(pty_session_id) + + return json.dumps({ + "status": "launched", + "viewer_url": viewer_url, + "agent": agent, + "project_dir": target_dir, + "workspace_path": workspace_path, + "instructions": ( + "Open viewer_url to attach. The agent is running in a directory " + "holding the files pulled from your Workspace folder, with your " + "kickoff prompt typed. Type the agent's quit command (e.g. /quit) " + "then `exit` to end the session. Note: files are a snapshot pulled " + "via 'databricks workspace export-dir' — git history is not included." + ), + }) + except Exception as e: + # Catch-all: ensure no resource leak. + if pty_session_id and _app_close_session is not None: + try: + _app_close_session(pty_session_id) + except Exception: + pass + if project_dir and os.path.isdir(project_dir): + shutil.rmtree(project_dir, ignore_errors=True) + return json.dumps({ + "status": "error", + "error": f"coda_interactive failed: {e}", + }) + + +@mcp.tool( + annotations=ToolAnnotations( + readOnlyHint=True, + destructiveHint=False, + idempotentHint=True, + ), +) +async def coda_inbox( + email: str = "", + status: str = "", +) -> str: + """Check status of all background tasks — your inbox. + + Call this instead of polling — it returns ALL tasks at once. + No need to track individual task_ids; the inbox shows everything + from the last 24 hours: running, completed, and failed tasks. + + By default returns all tasks. Filter by ``status`` to narrow: + ``"running"`` for in-progress only, ``"completed"`` for finished, + ``"failed"`` for errors, or ``""`` (default) for everything. + + Each task includes: ``task_id``, ``session_id``, ``status``, + ``elapsed_s``, ``prompt_summary`` (first 100 chars of what was asked), + ``previous_session_id`` (if chained from prior work). + Completed tasks also include ``summary`` (what was done). + Running tasks also include ``progress`` (latest agent step). + + Returns JSON with ``tasks`` (list sorted most recent first) + and ``counts`` (e.g. ``{"running": 1, "completed": 2, "failed": 0}``). + """ + try: + tasks = task_manager.list_all_tasks(email=email, status_filter=status) + # Decorate each task with its viewer URL (if available). + for t in tasks: + sess = task_manager._read_session_safe(t["session_id"]) + pty = sess.get("pty_session_id") if sess else None + if pty: + vu = url_builder.build_viewer_url(pty) + if vu: + t["viewer_url"] = vu + + counts = { + "running": 0, + "completed": 0, + "failed": 0, + "info_needed": 0, + "needs_approval": 0, + } + for t in tasks: + s = t.get("status", "") + if s in counts: + counts[s] += 1 + elif s == "done": + counts["completed"] += 1 + elif s == "timeout": + counts["failed"] += 1 + + return json.dumps({"tasks": tasks, "counts": counts}) + except Exception as exc: + return json.dumps({"status": "error", "error": str(exc)}) + + +@mcp.tool( + annotations=ToolAnnotations( + readOnlyHint=True, + destructiveHint=False, + idempotentHint=True, + ), +) +async def coda_get_result( + task_id: str, + session_id: str, +) -> str: + """Retrieve the structured result of a completed task. + + Call this AFTER coda_inbox shows a task as "completed", "failed", + "info_needed", or "needs_approval". + + Returns JSON with ``task_id``, ``session_id``, ``status``, ``summary`` + (what was done or why the agent stopped), ``files_changed`` (list of + modified files), ``artifacts`` (job IDs, commit hashes, etc.), + ``errors`` (if any), and — when status is "info_needed" — ``feedback`` + (a precise description of what context the caller must add before + resubmitting). + """ + try: + result = task_manager.get_task_result(task_id, session_id) + if result is None: + # No result yet — return current status + status = task_manager.get_task_status(task_id, session_id) + return json.dumps({ + "task_id": task_id, + "session_id": session_id, + "status": status.get("status", "unknown"), + "message": "Result not yet available — task is still in progress.", + }) + + result["task_id"] = task_id + result["session_id"] = session_id + # Ensure standard fields exist + result.setdefault("status", "done") + result.setdefault("summary", "") + result.setdefault("files_changed", []) + result.setdefault("artifacts", []) + result.setdefault("errors", []) + # Decorate with viewer_url if known + sess = task_manager._read_session_safe(session_id) + pty = sess.get("pty_session_id") if sess else None + if pty: + vu = url_builder.build_viewer_url(pty) + if vu: + result["viewer_url"] = vu + return json.dumps(result) + except Exception as exc: + return json.dumps({"status": "error", "task_id": task_id, "error": str(exc)}) + + +# ── Standalone entry point ────────────────────────────────────────── + +if __name__ == "__main__": + mcp.run() diff --git a/coda_mcp/task_manager.py b/coda_mcp/task_manager.py new file mode 100644 index 0000000..cd1cd5c --- /dev/null +++ b/coda_mcp/task_manager.py @@ -0,0 +1,644 @@ +"""Disk-based state manager for MCP sessions and tasks. + +Pure Python module — no Flask dependency. Just file I/O. + +Layout on disk +-------------- +~/.coda/sessions/{session-id}/ + session.json – session metadata + tasks/{task-id}/ + prompt.txt – wrapped prompt sent to the agent + meta.json – task metadata (email, timestamps, chaining) + status.jsonl – append-only progress log + result.json – final output (written by the agent) +""" + +import json +import os +import secrets +import time +import logging + +from coda_mcp.databricks_preamble import build_capabilities, build_workflow_protocol + +logger = logging.getLogger(__name__) + +# ── Root directory (patched in tests) ──────────────────────────────── + +SESSIONS_DIR = os.path.join( + os.environ.get("HOME", "/app/python/source_code"), ".coda", "sessions" +) + +# ── Concurrency limit ─────────────────────────────────────────────── + +MAX_CONCURRENT_TASKS = int(os.environ.get("CODA_MAX_CONCURRENT", "5")) + +# ── Task TTL (seconds) ────────────────────────────────────────────── + +TASK_TTL_S = int(os.environ.get("CODA_TASK_TTL", str(24 * 3600))) # 24h + +# ── PTY → task-dir reverse lookup (used by attach_session replay fallback) ── + +_pty_lookup_cache: dict[str, tuple[str, float]] = {} # pty_id -> (task_dir, ts) +_PTY_LOOKUP_TTL = 60.0 # seconds + +# ── Exceptions ─────────────────────────────────────────────────────── + + +class SessionBusyError(Exception): + """Raised when a task is submitted to a session that already has one running.""" + + +class SessionNotFoundError(Exception): + """Raised when the requested session does not exist or is closed.""" + + +class ConcurrencyLimitError(Exception): + """Raised when MAX_CONCURRENT_TASKS running tasks already exist.""" + + +# ── ID generators ──────────────────────────────────────────────────── + + +def _new_session_id() -> str: + return f"sess-{secrets.token_hex(6)}" + + +def _new_task_id() -> str: + return f"task-{secrets.token_hex(4)}" + + +# ── Low-level I/O ──────────────────────────────────────────────────── + + +def _session_dir(session_id: str) -> str: + return os.path.join(SESSIONS_DIR, session_id) + + +def _session_file(session_id: str) -> str: + return os.path.join(_session_dir(session_id), "session.json") + + +def _task_dir(session_id: str, task_id: str) -> str: + """Return the path to a task's directory.""" + return os.path.join(_session_dir(session_id), "tasks", task_id) + + +def _write_json(path: str, data: dict) -> None: + """Atomic write via tmp-then-rename.""" + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = path + ".tmp" + with open(tmp, "w") as f: + json.dump(data, f, indent=2) + os.replace(tmp, path) + + +def _read_session(session_id: str) -> dict: + """Read session.json or raise SessionNotFoundError.""" + path = _session_file(session_id) + try: + with open(path) as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + raise SessionNotFoundError(f"Session {session_id} not found or corrupt") + + +def _read_session_safe(session_id: str) -> dict | None: + """Read session.json, returning None on missing/corrupt instead of raising.""" + try: + return _read_session(session_id) + except SessionNotFoundError: + return None + + +def _update_session_field(session_id: str, key: str, value) -> None: + """Update a single field in session.json (read-modify-write).""" + data = _read_session(session_id) + data[key] = value + _write_json(_session_file(session_id), data) + + +# ── Session lifecycle ──────────────────────────────────────────────── + + +def create_session(email: str, user_id: str, label: str = "") -> dict: + """Create a new session directory with session.json. + + Returns ``{"session_id": "sess-…", "status": "ready"}``. + """ + session_id = _new_session_id() + data = { + "session_id": session_id, + "email": email, + "user_id": user_id, + "label": label, + "status": "ready", + "current_task": None, + "completed_tasks": [], + "created_at": time.time(), + } + _write_json(_session_file(session_id), data) + logger.info("Created session %s for %s", session_id, email) + return {"session_id": session_id, "status": "ready"} + + +def close_session(session_id: str) -> None: + """Mark a session as closed. Raises SessionNotFoundError if missing.""" + _read_session(session_id) # existence check + _update_session_field(session_id, "status", "closed") + logger.info("Closed session %s", session_id) + + +# ── Prompt wrapping ────────────────────────────────────────────────── + + +def wrap_prompt( + task_id: str, + session_id: str, + email: str, + prompt: str, + context: dict | None, + results_dir: str, + context_hint: str | None = None, + previous_session_id: str | None = None, + workflow_protocol: bool = True, +) -> str: + """Build the full prompt string written to ``prompt.txt``. + + Uses the ``---CODA-TASK---`` envelope convention so the agent can + parse metadata from the prompt deterministically. + + When ``workflow_protocol`` is True (default), inserts a CAPABILITIES + section (Databricks CLI, skills, MCP servers) and a WORKFLOW PROTOCOL + section (3-phase PLAN/EXECUTE/SYNTHESIZE with critique at each phase, + plus the info_needed escape hatch). Set False to skip both. + """ + context_block = "" + if context: + context_block = f"\nCONTEXT:\n{json.dumps(context, indent=2)}\n" + + hint_line = "" + if context_hint: + hint_line = f"context_hint: {context_hint}\n" + + prior_session_block = "" + if previous_session_id: + prior_dir = _session_dir(previous_session_id) + prior_session_block = ( + f"\nPRIOR SESSION: {previous_session_id}\n" + f"Read {prior_dir}/tasks/*/result.json for context on prior work.\n" + ) + + workflow_block = "" + if workflow_protocol: + workflow_block = ( + f"\nCAPABILITIES:\n" + f"{build_capabilities()}\n" + f"\n" + f"WORKFLOW PROTOCOL:\n" + f"{build_workflow_protocol()}\n" + ) + + return ( + f"---CODA-TASK---\n" + f"task_id: {task_id}\n" + f"session_id: {session_id}\n" + f"user: {email}\n" + f"{hint_line}" + f"{prior_session_block}" + f"{context_block}\n" + f"TASK:\n" + f"{prompt}\n" + f"{workflow_block}" + f"\n" + f"INSTRUCTIONS:\n" + f"1. As you work, append progress lines to {results_dir}/status.jsonl\n" + f' Each line must be valid JSON: {{"step": "label", "message": "what you are doing"}}\n' + f" Canonical step labels (use these when the workflow protocol is active):\n" + f" plan, critique_plan, execute_, critique_execute,\n" + f" synthesize, critique_synthesize, info_needed, failed\n" + f"\n" + f"2. When you are COMPLETELY DONE, write a SINGLE FILE at this exact path:\n" + f" {results_dir}/result.json\n" + f" It must contain this JSON structure (status is one of the four\n" + f" values listed below; the angle-bracketed placeholder is NOT literal\n" + f" JSON — pick exactly one of the four values):\n" + f" {{\n" + f' "status": "",\n' + f' "summary": "one paragraph describing what you did or why you stopped",\n' + f' "feedback": "REQUIRED if status=info_needed — what context the caller must add",\n' + f' "files_changed": ["list", "of", "file", "paths"],\n' + f' "artifacts": {{}},\n' + f' "errors": []\n' + f" }}\n" + f" - status=\"completed\": you finished the task.\n" + f" - status=\"failed\": unrecoverable hard error; describe in errors[].\n" + f" - status=\"info_needed\": you are blocked because something the CALLER must\n" + f" supply is missing. The feedback field is REQUIRED and must precisely\n" + f" name what is missing. The caller will resubmit with more context.\n" + f" - status=\"needs_approval\": you have a destructive action ready but need\n" + f" explicit caller approval before executing. See SAFETY section.\n" + f" IMPORTANT: result.json is a FILE not a directory. Write it with:\n" + f" echo '{{...}}' > {results_dir}/result.json\n" + f"\n" + f"3. If you delegate to a sub-agent, update status.jsonl with delegation steps.\n" + f"\n" + f"SAFETY:\n" + f"- Do NOT delete, drop, or truncate tables, schemas, catalogs, or volumes.\n" + f"- Do NOT delete files outside the current project directory.\n" + f"- Do NOT run destructive Databricks CLI commands (e.g. databricks clusters delete, " + f"databricks jobs delete, databricks pipelines delete).\n" + f"- Do NOT modify permissions, grants, or access controls unless explicitly requested.\n" + f"- Prefer CREATE OR REPLACE over DROP+CREATE. Prefer INSERT/MERGE over DELETE+INSERT.\n" + f"- If the task requires a destructive operation, describe what you would do in " + f"result.json with status \"needs_approval\" instead of executing it.\n" + f"---END-CODA-TASK---" + ) + + +# ── Task lifecycle ─────────────────────────────────────────────────── + + +def create_task( + session_id: str, + prompt: str, + email: str, + context: dict | None = None, + context_hint: str | None = None, + timeout_s: int | None = None, + permissions: str | None = None, + previous_session_id: str | None = None, + workflow_protocol: bool = True, +) -> dict: + """Create a task inside an existing session. + + Raises + ------ + SessionNotFoundError + If the session does not exist or is closed. + SessionBusyError + If the session already has a running task. + + Returns ``{"task_id": "task-…", "status": "running"}``. + """ + session = _read_session(session_id) + + if session.get("status") == "closed": + raise SessionNotFoundError(f"Session {session_id} is closed") + + if session.get("status") == "busy": + raise SessionBusyError( + f"Session {session_id} already has a running task: " + f"{session.get('current_task')}" + ) + + task_id = _new_task_id() + tdir = _task_dir(session_id, task_id) + os.makedirs(tdir, exist_ok=True) + + # Write wrapped prompt + results_dir = os.path.join(tdir, "results") + wrapped = wrap_prompt( + task_id=task_id, + session_id=session_id, + email=email, + prompt=prompt, + context=context, + results_dir=results_dir, + context_hint=context_hint, + previous_session_id=previous_session_id, + workflow_protocol=workflow_protocol, + ) + with open(os.path.join(tdir, "prompt.txt"), "w") as f: + f.write(wrapped) + + # Write meta.json for inbox scanning + now = time.time() + meta = { + "email": email, + "created_at": now, + "previous_session_id": previous_session_id or "", + "permissions": permissions or "smart", + "timeout_s": timeout_s or 3600, + "prompt_summary": prompt[:100], + } + _write_json(os.path.join(tdir, "meta.json"), meta) + + # Seed status log + with open(os.path.join(tdir, "status.jsonl"), "w") as f: + f.write(json.dumps({"status": "running", "ts": now}) + "\n") + + # Mark session busy + data = _read_session(session_id) + data["status"] = "busy" + data["current_task"] = task_id + _write_json(_session_file(session_id), data) + + logger.info("Created task %s in session %s", task_id, session_id) + return {"task_id": task_id, "status": "running"} + + +# ── Task queries ───────────────────────────────────────────────────── + + +def get_task_status(task_id: str, session_id: str) -> dict: + """Read the last line of status.jsonl for the task. + + Returns ``{"status": "not_found"}`` if the task directory is missing. + """ + status_path = os.path.join(_task_dir(session_id, task_id), "status.jsonl") + try: + last = None + with open(status_path) as f: + for line in f: + line = line.strip() + if line: + last = json.loads(line) + return last or {"status": "not_found"} + except (OSError, json.JSONDecodeError): + return {"status": "not_found"} + + +def _find_result_json(task_dir: str) -> str | None: + """Find result.json — agents may write it at root or in results/ subdir.""" + for candidate in [ + os.path.join(task_dir, "result.json"), + os.path.join(task_dir, "results", "result.json"), + ]: + if os.path.isfile(candidate): + return candidate + return None + + +def get_task_result(task_id: str, session_id: str) -> dict | None: + """Read result.json if it exists; otherwise return None.""" + result_path = _find_result_json(_task_dir(session_id, task_id)) + if not result_path: + return None + try: + with open(result_path) as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + return None + + +# ── Task completion ────────────────────────────────────────────────── + + +def complete_task(session_id: str, task_id: str) -> None: + """Mark a task as done and auto-close the session. + + Appends a ``done`` entry to status.jsonl, adds task_id to + ``completed_tasks``, and closes the session (v2: ephemeral sessions). + """ + session = _read_session(session_id) + + # Append done to status log + status_path = os.path.join(_task_dir(session_id, task_id), "status.jsonl") + with open(status_path, "a") as f: + f.write(json.dumps({"status": "done", "ts": time.time()}) + "\n") + + # Update session — auto-close (v2: sessions are ephemeral) + session["status"] = "closed" + session["current_task"] = None + session["closed_at"] = time.time() + if task_id not in session["completed_tasks"]: + session["completed_tasks"].append(task_id) + _write_json(_session_file(session_id), session) + + logger.info("Completed task %s in session %s (auto-closed)", task_id, session_id) + + +# ── Inbox: list all tasks across sessions ─────────────────────────── + + +def list_all_tasks(email: str = "", status_filter: str = "") -> list[dict]: + """Scan all sessions and return a flat list of tasks for the inbox. + + Returns tasks from the last ``TASK_TTL_S`` seconds, sorted most recent first. + Each entry includes task_id, session_id, status, elapsed_s, prompt_summary, + summary (if completed), progress (if running), previous_session_id, created_at. + """ + now = time.time() + cutoff = now - TASK_TTL_S + tasks = [] + + if not os.path.isdir(SESSIONS_DIR): + return tasks + + for sess_name in os.listdir(SESSIONS_DIR): + sess_dir = os.path.join(SESSIONS_DIR, sess_name) + if not os.path.isdir(sess_dir): + continue + + tasks_dir = os.path.join(sess_dir, "tasks") + if not os.path.isdir(tasks_dir): + continue + + for task_name in os.listdir(tasks_dir): + task_dir = os.path.join(tasks_dir, task_name) + if not os.path.isdir(task_dir): + continue + + # Read meta.json + meta_path = os.path.join(task_dir, "meta.json") + try: + with open(meta_path) as f: + meta = json.load(f) + except (OSError, json.JSONDecodeError): + # Legacy task without meta.json — skip or build minimal entry + meta = {} + + created_at = meta.get("created_at", 0) + if created_at < cutoff: + continue + + # Filter by email + if email and meta.get("email", "") != email: + continue + + # Determine task status from status.jsonl + task_status = _read_last_status(task_dir) + + # Check for result.json to determine completion + result_path = _find_result_json(task_dir) + summary = "" + if result_path: + try: + with open(result_path) as f: + result_data = json.load(f) + task_status = result_data.get("status", "completed") + summary = result_data.get("summary", "") + except (OSError, json.JSONDecodeError): + pass + + # Filter by status + if status_filter and task_status != status_filter: + continue + + # Get progress for running tasks + progress = "" + if task_status == "running": + progress = _read_last_progress(task_dir) + + elapsed_s = round(now - created_at, 1) + + entry = { + "task_id": task_name, + "session_id": sess_name, + "status": task_status, + "elapsed_s": elapsed_s, + "prompt_summary": meta.get("prompt_summary", ""), + "previous_session_id": meta.get("previous_session_id", ""), + "created_at": created_at, + } + if summary: + entry["summary"] = summary + if progress: + entry["progress"] = progress + + tasks.append(entry) + + # Sort most recent first + tasks.sort(key=lambda t: t["created_at"], reverse=True) + return tasks + + +def _read_last_status(task_dir: str) -> str: + """Read the last status from status.jsonl.""" + status_path = os.path.join(task_dir, "status.jsonl") + try: + last = None + with open(status_path) as f: + for line in f: + line = line.strip() + if line: + last = json.loads(line) + return (last or {}).get("status", "unknown") + except (OSError, json.JSONDecodeError): + return "unknown" + + +def _read_last_progress(task_dir: str) -> str: + """Read the last progress message from status.jsonl.""" + status_path = os.path.join(task_dir, "status.jsonl") + try: + last = None + with open(status_path) as f: + for line in f: + line = line.strip() + if line: + last = json.loads(line) + return (last or {}).get("message", "") + except (OSError, json.JSONDecodeError): + return "" + + +# ── Concurrency check ────────────────────────────────────────────── + + +def count_running_tasks() -> int: + """Count tasks currently in 'running' state across all sessions.""" + count = 0 + if not os.path.isdir(SESSIONS_DIR): + return count + + for sess_name in os.listdir(SESSIONS_DIR): + sess_file = os.path.join(SESSIONS_DIR, sess_name, "session.json") + try: + with open(sess_file) as f: + session = json.load(f) + if session.get("status") == "busy": + count += 1 + except (OSError, json.JSONDecodeError): + continue + return count + + +# ── PTY → task-dir reverse lookup ────────────────────────────────── + + +def find_task_dir_by_pty_session(pty_session_id: str) -> str | None: + """Find the task dir whose session.json carries this pty_session_id. + + Returns the path to the active task dir, or — if the session has completed — + the most recently completed task dir. Returns None on no match. + + Cached for ``_PTY_LOOKUP_TTL`` seconds to avoid disk scans on every browser + refresh. + + Invariant: CoDA MCP sessions are ephemeral — one task per session. If the + lifecycle ever changes to allow multiple tasks per session, this function + must be revisited to pick the in-progress task rather than + ``completed_tasks[-1]``. + """ + now = time.time() + cached = _pty_lookup_cache.get(pty_session_id) + if cached and (now - cached[1]) < _PTY_LOOKUP_TTL: + return cached[0] + + if not os.path.isdir(SESSIONS_DIR): + return None + + for sess_name in os.listdir(SESSIONS_DIR): + sess_file = os.path.join(SESSIONS_DIR, sess_name, "session.json") + try: + with open(sess_file) as f: + data = json.load(f) + except (OSError, json.JSONDecodeError): + continue + + if data.get("pty_session_id") != pty_session_id: + continue + + candidate = data.get("current_task") or ( + data["completed_tasks"][-1] if data.get("completed_tasks") else None + ) + if candidate: + tdir = os.path.join(SESSIONS_DIR, sess_name, "tasks", candidate) + _pty_lookup_cache[pty_session_id] = (tdir, now) + return tdir + + return None + + +# ── Cleanup expired sessions ──────────────────────────────────────── + + +def cleanup_expired_tasks() -> int: + """Remove session directories older than TASK_TTL_S. Returns count removed.""" + import shutil + + now = time.time() + cutoff = now - TASK_TTL_S + removed = 0 + + if not os.path.isdir(SESSIONS_DIR): + return removed + + for sess_name in os.listdir(SESSIONS_DIR): + sess_dir = os.path.join(SESSIONS_DIR, sess_name) + if not os.path.isdir(sess_dir): + continue + + sess_file = os.path.join(sess_dir, "session.json") + try: + with open(sess_file) as f: + session = json.load(f) + except (OSError, json.JSONDecodeError): + continue + + # Only clean closed sessions past TTL + if session.get("status") != "closed": + continue + + closed_at = session.get("closed_at", session.get("created_at", 0)) + if closed_at < cutoff: + try: + shutil.rmtree(sess_dir) + removed += 1 + logger.info("Cleaned up expired session %s", sess_name) + except OSError: + logger.warning("Failed to clean up session %s", sess_name) + + return removed diff --git a/coda_mcp/url_builder.py b/coda_mcp/url_builder.py new file mode 100644 index 0000000..c08d2ed --- /dev/null +++ b/coda_mcp/url_builder.py @@ -0,0 +1,46 @@ +"""Builds the viewer_url returned by CoDA MCP tools. + +Resolution order: +1. ``CODA_APP_URL`` env var (explicit override for local dev / power users). +2. Module-level cache populated by ``AppUrlCaptureMiddleware`` from the + ``X-Forwarded-Host`` header (officially provided by Databricks Apps). +3. ``None`` — caller omits the field entirely. + +The cache is process-global (single uvicorn worker per app) and refreshed +on every inbound HTTP request. +""" +from __future__ import annotations + +import os +from typing import Optional + +_app_url_cache: Optional[str] = None + + +def capture_from_headers(host: Optional[str]) -> None: + """Called by the ASGI middleware on every inbound HTTP request. + + No-op when ``host`` is falsy (None or empty) to avoid wiping a good + cache value with a missing header on a probe/CORS preflight. + + Strips any accidental ``https://`` / ``http://`` prefix on the way in + so build_viewer_url's unconditional ``https://`` prepend can't produce + a double-scheme URL. + """ + global _app_url_cache + if host: + host = host.removeprefix("https://").removeprefix("http://").strip("/") + if host: + _app_url_cache = host + + +def build_viewer_url(pty_session_id: str) -> Optional[str]: + """Return the full viewer URL for a PTY session, or None if no base is known.""" + override = os.environ.get("CODA_APP_URL", "").strip() + if override: + base = override.rstrip("/") + elif _app_url_cache: + base = f"https://{_app_url_cache}" + else: + return None + return f"{base}/?session={pty_session_id}" diff --git a/docs/coda-mcp-overview.html b/docs/coda-mcp-overview.html new file mode 100644 index 0000000..46e0969 --- /dev/null +++ b/docs/coda-mcp-overview.html @@ -0,0 +1,453 @@ + + + + + +CoDA MCP Server — Overview & Capabilities + + + + + + + +
+
+ + Databricks · CoDA + MCP Server +
+
+ +
+

Coding Agents on Databricks Apps

+

The CoDA MCP Server

+

A Model Context Protocol endpoint that lets any MCP client — Genie Code, + Claude Desktop, Cursor, or a local Claude Code / Codex — delegate coding work to AI agents + running inside CoDA on Databricks, then watch or drive them live.

+
+ Endpoint /mcp + 4 tools + 3 usage modes + FastMCP Streamable HTTP +
+
+ + +
+
01

What it is

+ one endpoint, many clients
+

CoDA (Coding agents on Databricks Apps) runs five AI coding agents — Claude Code, + Codex, Gemini CLI, OpenCode, and Hermes — inside a Databricks App, each in a real terminal (PTY) + with the Databricks CLI, 16 Databricks skills, and MCP servers (DeepWiki, Exa) pre-wired. The CoDA MCP + server exposes that capability over the Model Context Protocol at /mcp, so an upstream + agent can hand work off to CoDA — either as a fire-and-forget background task or as a live, + human-driven session — using the same Databricks identity that owns the app.

+
+ + +
+
02

The four tools

+ exposed natively & via JSON-RPC
+
+
+

coda_run

+
Autonomous · fire-and-forget
+

Submit a coding task and return immediately with a task_id. The task runs to + completion in a background terminal under a structured workflow protocol. Don’t poll — move on.

+
+
key argsprompt, context, previous_session_id, workflow_protocol
+
returnstask_id, session_id, status:"running"
+
viewerreplay-only URL (post-hoc review)
+
+
+
+

coda_interactive

+
Human handoff · live attach
+

Pull a Databricks Workspace folder onto a fresh session, launch the chosen agent (claude default) + in it with your prompt pre-seeded, and return a live viewer URL a human opens to drive it.

+
+
key argsprompt, workspace_path, agent
+
returnsstatus:"launched", viewer_url, project_dir
+
viewerlive, interactive (the only handle)
+
+
+
+

coda_inbox

+
Status · last 24h
+

List background tasks with live status counts. Call only when the user asks — never poll it + in a loop. Interactive sessions do not appear here.

+
+
key argsemail?, status?
+
returnstasks[], counts{running,completed,failed,info_needed,needs_approval}
+
+
+
+

coda_get_result

+
Retrieve · structured output
+

Fetch the full structured result of a finished task — what it did, what changed, and any + follow-up the caller must supply.

+
+
key argstask_id, session_id
+
returnsstatus, summary, files_changed, artifacts, errors, feedback?
+
+
+
+
+ + +
+
03

Three ways to reach CoDA

+ one platform, three postures
+
+
+ Mode 1 +

Direct web UI

+

Open the CoDA app in a browser and drive an agent terminal yourself. No MCP involved — the + hands-on baseline.

+
entry: the app URL
+
+
+ Mode 2 · coda_interactive +

Live handoff

+

An upstream agent stages files and launches a session; a human attaches via the viewer URL and + drives it interactively. Best when judgement or steering is needed.

+
entry: live viewer_url
+
+
+ Mode 3 · coda_run +

Autonomous task

+

An upstream agent submits work that runs to completion unattended. The viewer URL is a replay + for after-the-fact review; results come back through the inbox.

+
entry: replay viewer_url + inbox
+
+
+
+ + +
+
04

How the flows work

+ end to end
+ + +
+

Mode 2 coda_interactive — live human handoff

+

Files move up to the Workspace from the caller, then down into the CoDA + session — both as the same Databricks user, so access just works.

+
+
1
Local agent
+
Claude Code / Codex on the user’s machine holds the project files locally.
+
+
2
Upload to Workspace
+
databricks workspace import-dir pushes files to /Workspace/Users/…
+
+
3
Call the tool
+
coda_interactive(workspace_path, prompt) hits CoDA at /mcp.
+
+
4
Pull into session
+
CoDA’s terminal runs export-dir into ~/.coda/projects/<id>/ — confirmed by a completion marker.
+
+
5
Launch & seed
+
claude --enable-auto-mode "<prompt>" — no trust prompt, prompt pre-loaded.
+
+
6
Human drives
+
Returns a live viewer_url; the user opens it and steers the session to the end.
+
+
+ + +
+

Mode 3 coda_run — autonomous background task

+

Submit and forget. The agent works under a disciplined protocol and reports back + through the inbox.

+
+
1
Submit
+
coda_run(prompt) returns a task_id instantly — fire-and-forget.
+
+
2
Background run
+
A detached terminal runs the agent with a CAPABILITIES + WORKFLOW PROTOCOL envelope.
+
+
3
Plan → Execute → Synthesize
+
Three phases, each with a self-critique; max two iterations per phase.
+
+
4
Write result
+
Emits result.json — completed, failed, info_needed, or needs_approval.
+
+
5
Retrieve
+
Caller checks coda_inbox and pulls full output via coda_get_result.
+
+
info_needed loop: when the agent is blocked on missing context, it stops and returns + a precise feedback string. The caller adds the missing detail and resubmits with + previous_session_id — a structured iteration loop instead of a guess.
+
+
+ + +
+
05

The workflow protocol

+ coda_run, on by default
+

Every coda_run prompt is wrapped with two sections so the background agent + acts deliberately: CAPABILITIES (the Databricks CLI, the 16 skills, and the DeepWiki / Exa / CoDA + MCP servers it can lean on) and WORKFLOW PROTOCOL — a three-phase pipeline with a critique + after each phase. Set workflow_protocol=false to opt out for non-Databricks tasks.

+
+
phase 1

Plan

+

Write a step-by-step plan to the status log.

+
critique → APPROVE / BLOCK / fix · max 2 iterations
+
phase 2

Execute

+

Work the plan step by step, emitting progress.

+
critique built vs planned · max 2 iterations
+
phase 3

Synthesize

+

Write result.json against the original task.

+
final critique vs the ask · max 2 iterations
+
+
+ + +
+
06

Identity & the file round-trip

+ why it works
+
+

The terminal is you; the server is not.

+

The MCP server process runs as the app’s service principal, which cannot read a user’s + private Workspace folders. The agent terminal, however, is authenticated as the app owner. That’s + why CoDA never reads your files server-side — it pulls them in the terminal with + databricks workspace export-dir, using the right identity. A local caller mirrors this from the + other side with import-dir to stage local files into the Workspace first.

+
+
+ + +
+
07

Result statuses

+ what coda_get_result returns
+ + + + + + + + + +
StatusMeaningWhat the caller does
runningTask is still working in the background.Wait; check the inbox later.
completedFinished successfully.Read summary + files_changed.
failedUnrecoverable error (a command crashed, an API 500’d).Inspect errors.
info_neededBlocked on missing context the caller must supply.Read feedback, resubmit with more context.
needs_approvalA destructive action is staged, awaiting explicit go-ahead.Approve or decline; resubmit.
+
+ + +
+
08

Architecture

+ how it’s served
+
+
Transport
FastMCP streamable_http_app() — native MCP Streamable HTTP, mounted at /mcp.
+
Runtime
uvicorn (ASGI); Flask + Socket.IO mounted via WSGI middleware for the terminal UI.
+
Sessions
Each agent runs in a real PTY; fds are process-local, so a single worker owns them.
+
Identity
App-owner credentials (PAT or service principal); the terminal CLI acts as the user.
+
Dual surface
The same four tools are exposed both natively and over a JSON-RPC compatibility path.
+
Viewer
A browser attaches to a session over WebSocket, with automatic HTTP-polling fallback.
+
+
+ +
+
+ Databricks · CoDA MCP Server + Coding Agents on Databricks Apps +
+
+ + + diff --git a/docs/mcp-client-setup.md b/docs/mcp-client-setup.md new file mode 100644 index 0000000..f8e1bb6 --- /dev/null +++ b/docs/mcp-client-setup.md @@ -0,0 +1,73 @@ +# CoDA MCP Client Setup + +CoDA exposes an MCP endpoint at `/mcp` on the Databricks App. Databricks Apps use OAuth (not PATs) for authentication, so MCP clients need a stdio bridge that injects fresh OAuth tokens. + +## How it works + +`tools/coda-bridge.py` is a zero-dependency Python script that: + +1. Claude Code launches it as a stdio MCP server +2. It reads JSON-RPC messages from stdin +3. Fetches a fresh OAuth token via `databricks auth token` +4. Forwards requests to the App's HTTP endpoint with the token +5. Returns responses on stdout + +Tokens are cached for 30 minutes (they expire after 60). + +## Setup + +### 1. Copy the bridge script + +```bash +mkdir -p ~/.claude/mcp-bridges +cp tools/coda-bridge.py ~/.claude/mcp-bridges/ +``` + +### 2. Add to Claude Code settings + +Add this to `mcpServers` in `~/.claude/settings.json`: + +```json +"coda-mcp": { + "type": "stdio", + "command": "python3", + "args": ["/path/to/.claude/mcp-bridges/coda-bridge.py"], + "env": { + "CODA_MCP_URL": "https://.databricksapps.com/mcp", + "DATABRICKS_PROFILE": "" + } +} +``` + +### 3. Restart Claude Code + +The MCP server will start automatically on next session. + +## Configuration + +| Environment Variable | Description | Example | +|---------------------|-------------|---------| +| `CODA_MCP_URL` | Full URL to the app's `/mcp` endpoint | `https://mcp-test-coda-747...com/mcp` | +| `DATABRICKS_PROFILE` | Databricks CLI profile name | `9cefok` | + +## Prerequisites + +- `databricks` CLI installed and authenticated (`databricks auth login -p `) +- Python 3.8+ +- No pip dependencies required (stdlib only) + +## Troubleshooting + +Bridge logs go to stderr. Check with: + +```bash +CODA_MCP_URL="https://your-app.databricksapps.com/mcp" \ +DATABRICKS_PROFILE="your-profile" \ +echo '{"jsonrpc":"2.0","method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"test","version":"1.0"}},"id":1}' | python3 tools/coda-bridge.py +``` + +If you see `Auth failed (302)`, your Databricks CLI session may have expired. Run: + +```bash +databricks auth login -p +``` diff --git a/docs/mcp-v2-background-execution.md b/docs/mcp-v2-background-execution.md new file mode 100644 index 0000000..3d7557c --- /dev/null +++ b/docs/mcp-v2-background-execution.md @@ -0,0 +1,171 @@ +# CoDA MCP v2 — Background Execution + Inbox Pattern + +## Overview + +CoDA exposes 3 MCP tools so Databricks GenieCode (or any MCP client) can delegate +coding tasks to AI agents running in the background. GenieCode's chat context stays +free while tasks execute — no polling required. + +## Tools + +| Tool | Purpose | +|------|---------| +| `coda_run` | Fire-and-forget task submission | +| `coda_inbox` | Dashboard of all background tasks | +| `coda_get_result` | Pull full structured result | + +## Flow Diagram + +``` +┌─────────────┐ ┌──────────────┐ ┌─────────────┐ +│ GenieCode │ │ CoDA MCP │ │ Hermes │ +│ (caller) │ │ (3 tools) │ │ (executor) │ +└──────┬──────┘ └──────┬───────┘ └──────┬──────┘ + │ │ │ + │ 1. coda_run(prompt) │ │ + │──────────────────────>│ │ + │ │ auto-create session │ + │ │ + PTY + task dir │ + │ │ write prompt.txt │ + │ │ write meta.json │ + │ │ │ + │ {task_id, sess_id, │ hermes -z prompt.txt │ + │ status: "running"} │───────────────────────>│ + │<──────────────────────│ │ + │ │ _watch_task thread │ + │ ✓ context is FREE │ monitors result.json │ + │ user keeps chatting │ │ + │ │ │ works... + │ ... │ │ delegates + │ │ │ to claude/ + │ │ │ codex/gemini + │ │ │ + │ 2. coda_inbox() │ │ writes + │──────────────────────>│ │ status.jsonl + │ │ scan all sessions │ + │ {tasks: [...], │ read meta + status │ + │ counts: {run:1}} │ │ + │<──────────────────────│ │ + │ │ │ + │ ... │ │ writes + │ │ │ result.json + │ │ │ + │ │ _watch_task detects │ + │ │ result.json exists │ + │ │ → complete_task() │ + │ │ → auto-close session │ + │ │ → free PTY │ + │ │ │ + │ 3. coda_inbox() │ │ + │──────────────────────>│ │ + │ {tasks: [{status: │ │ + │ "completed", │ │ + │ summary: "..."}]} │ │ + │<──────────────────────│ │ + │ │ │ + │ 4. coda_get_result() │ │ + │──────────────────────>│ │ + │ {summary, files, │ read result.json │ + │ artifacts, errors} │ │ + │<──────────────────────│ │ + │ │ │ + ├── CHAINING ───────────┤ │ + │ │ │ + │ 5. coda_run(prompt, │ │ + │ previous_session_id) │ new session + PTY │ + │──────────────────────>│ inject PRIOR SESSION │ + │ │ block in prompt │ + │ {new task_id, │───────────────────────>│ + │ new sess_id} │ │ reads prior + │<──────────────────────│ │ result.json + │ │ │ for context +``` + +## Key Design Decisions + +### Sessions are ephemeral, tasks are persistent +- Session = PTY + Hermes instance. Auto-closes when task completes. +- Task state (prompt, status, result) persists on disk for 24 hours. +- Continuity via `previous_session_id`, not long-lived sessions. + +### No polling from GenieCode +- `coda_inbox` replaces `coda_get_status` — shows ALL tasks at once. +- GenieCode checks when the user asks, not on a timer. +- CoDA's internal `_watch_task` thread polls the filesystem (invisible to caller). + +### Task chaining +- `previous_session_id` points to a prior session's disk state. +- Hermes reads `~/.coda/sessions/{prev_id}/tasks/*/result.json` for context. +- Chain depth: one level. Hermes can walk deeper if needed. + +### Concurrency +- `CODA_MAX_CONCURRENT` env var (default: 5). +- Each task gets its own session — no "session busy" errors. +- Exceeding the limit returns a clear error. + +## Data Model + +``` +~/.coda/sessions/{session-id}/ + session.json # metadata + auto-close timestamp + tasks/{task-id}/ + prompt.txt # wrapped prompt sent to Hermes + meta.json # {email, created_at, previous_session_id, permissions} + status.jsonl # append-only progress log + result.json # final structured output +``` + +## Tool Reference + +### `coda_run` + +```python +coda_run( + prompt: str, # what to do + email: str, # who's asking + context: str = "{}", # UC metadata (tables, schemas) + previous_session_id: str = "", # chain from prior work + permissions: str = "smart", # "smart" or "yolo" + timeout_s: int = 3600, # max 1 hour default +) +# Returns: {"task_id", "session_id", "status": "running"} +``` + +### `coda_inbox` + +```python +coda_inbox( + email: str = "", # filter by user + status: str = "", # "running", "completed", "failed", or "" for all +) +# Returns: {"tasks": [...], "counts": {"running": N, "completed": N, "failed": N}} +``` + +Each task entry: `task_id`, `session_id`, `status`, `elapsed_s`, `prompt_summary`, +`summary` (completed), `progress` (running), `previous_session_id`, `created_at`. + +### `coda_get_result` + +```python +coda_get_result(task_id: str, session_id: str) +# Returns: {"task_id", "session_id", "status", "summary", +# "files_changed", "artifacts", "errors"} +``` + +## Migration from v1 + +| v1 Tool | v2 Equivalent | +|---------|--------------| +| `coda_create_session` | Removed — auto-created by `coda_run` | +| `coda_run_task` | `coda_run` (simplified, auto-session) | +| `coda_get_status` | `coda_inbox` (all tasks at once) | +| `coda_get_result` | `coda_get_result` (unchanged) | +| `coda_close_session` | Removed — auto-closed on completion | + +## Limitations + +- **Ephemeral filesystem**: On Databricks Apps, `~/.coda/` is local disk. App + redeployment wipes task state. Real artifacts (git commits, jobs, workspace files) + are unaffected. +- **No push notifications**: GenieCode must call `coda_inbox` to discover completions. + SSE/streaming is a future consideration if polling proves insufficient. diff --git a/docs/plans/2026-05-01-coda-mcp-server.md b/docs/plans/2026-05-01-coda-mcp-server.md new file mode 100644 index 0000000..1e59ba3 --- /dev/null +++ b/docs/plans/2026-05-01-coda-mcp-server.md @@ -0,0 +1,1179 @@ +# CoDA MCP Server Implementation Plan + +> **⚠️ SUPERSEDED — historical reference only.** This was the v1 implementation plan (5 tools, gunicorn + WSGI bridge). The shipped implementation diverged during iteration: the production design is documented in [`docs/mcp-v2-background-execution.md`](../mcp-v2-background-execution.md) (3 tools — `coda_run`, `coda_inbox`, `coda_get_result` — on uvicorn + native ASGI). Kept in the tree so reviewers can see the design evolution; do not follow this plan as-is. + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Add an MCP server endpoint (`/mcp`) to CoDA so Databricks Genie Code can delegate coding tasks to Hermes Agent via the MCP protocol. + +**Architecture:** Python MCP SDK mounted as a stateless HTTP app at `/mcp` alongside the existing Flask app. A new `task_manager.py` module handles session/task state on disk (`~/.coda/sessions/`). The MCP tools call into the existing PTY infrastructure for session creation and input piping. Hermes is always the agent invoked. + +**Tech Stack:** Python MCP SDK (`mcp` package, already installed), Flask, existing PTY session infrastructure, Hermes Agent CLI (`hermes -z`) + +**Design doc:** `.humantokens/coda-mcp-design.md` (full design with all decisions) + +--- + +### Task 1: Create Task Manager Module + +The task manager handles all disk-based state for MCP sessions and tasks. It's a pure Python module with no Flask dependency — just file I/O. + +**Files:** +- Create: `task_manager.py` +- Create: `tests/test_task_manager.py` + +**Step 1: Write the failing tests** + +```python +# tests/test_task_manager.py +import os +import json +import tempfile +import pytest +from unittest.mock import patch + +# All tests use a temp dir instead of ~/.coda +@pytest.fixture +def task_mgr(tmp_path): + with patch("task_manager.SESSIONS_DIR", str(tmp_path / "sessions")): + import task_manager + # Force reimport to pick up patched path + task_manager.SESSIONS_DIR = str(tmp_path / "sessions") + yield task_manager + + +def test_create_session(task_mgr): + result = task_mgr.create_session(email="alice@example.com", user_id="123") + assert "session_id" in result + assert result["status"] == "ready" + + # Verify session.json on disk + session_dir = os.path.join(task_mgr.SESSIONS_DIR, result["session_id"]) + assert os.path.isdir(session_dir) + with open(os.path.join(session_dir, "session.json")) as f: + data = json.load(f) + assert data["created_by"] == "alice@example.com" + assert data["status"] == "idle" + assert data["current_task"] is None + + +def test_create_task(task_mgr): + session = task_mgr.create_session(email="alice@example.com", user_id="123") + sid = session["session_id"] + + result = task_mgr.create_task( + session_id=sid, + prompt="create a pipeline", + email="alice@example.com", + context={"tables": ["sales.transactions"]}, + ) + assert "task_id" in result + assert result["status"] == "running" + + # Verify task dir and files + task_dir = os.path.join(task_mgr.SESSIONS_DIR, sid, "tasks", result["task_id"]) + assert os.path.isfile(os.path.join(task_dir, "prompt.txt")) + + # Session should be busy + with open(os.path.join(task_mgr.SESSIONS_DIR, sid, "session.json")) as f: + data = json.load(f) + assert data["status"] == "busy" + assert data["current_task"] == result["task_id"] + + +def test_create_task_rejects_when_busy(task_mgr): + session = task_mgr.create_session(email="alice@example.com", user_id="123") + sid = session["session_id"] + + task_mgr.create_task(session_id=sid, prompt="task 1", email="alice@example.com") + with pytest.raises(task_mgr.SessionBusyError): + task_mgr.create_task(session_id=sid, prompt="task 2", email="alice@example.com") + + +def test_get_status_running(task_mgr): + session = task_mgr.create_session(email="alice@example.com", user_id="123") + sid = session["session_id"] + task = task_mgr.create_task(session_id=sid, prompt="do work", email="alice@example.com") + + status = task_mgr.get_task_status(task["task_id"], sid) + assert status["status"] == "running" + assert "elapsed_s" in status + assert status.get("progress") is None # no status.jsonl yet + + +def test_get_status_with_progress(task_mgr): + session = task_mgr.create_session(email="alice@example.com", user_id="123") + sid = session["session_id"] + task = task_mgr.create_task(session_id=sid, prompt="do work", email="alice@example.com") + tid = task["task_id"] + + # Simulate agent writing status.jsonl + status_file = os.path.join(task_mgr.SESSIONS_DIR, sid, "tasks", tid, "status.jsonl") + with open(status_file, "a") as f: + f.write(json.dumps({"step": "planning", "message": "Analyzing requirements"}) + "\n") + f.write(json.dumps({"step": "coding", "message": "Writing pipeline"}) + "\n") + + status = task_mgr.get_task_status(tid, sid) + assert status["status"] == "running" + assert status["progress"]["step"] == "coding" + + +def test_get_result_completed(task_mgr): + session = task_mgr.create_session(email="alice@example.com", user_id="123") + sid = session["session_id"] + task = task_mgr.create_task(session_id=sid, prompt="do work", email="alice@example.com") + tid = task["task_id"] + + # Simulate agent writing result.json + result_file = os.path.join(task_mgr.SESSIONS_DIR, sid, "tasks", tid, "result.json") + with open(result_file, "w") as f: + json.dump({ + "status": "completed", + "summary": "Created pipeline", + "files_changed": ["pipeline.py"], + "artifacts": {"job_id": "123"}, + "errors": [] + }, f) + + result = task_mgr.get_task_result(tid, sid) + assert result["status"] == "completed" + assert result["summary"] == "Created pipeline" + + +def test_get_result_not_done(task_mgr): + session = task_mgr.create_session(email="alice@example.com", user_id="123") + sid = session["session_id"] + task = task_mgr.create_task(session_id=sid, prompt="do work", email="alice@example.com") + + result = task_mgr.get_task_result(task["task_id"], sid) + assert result["status"] == "running" + assert result.get("summary") is None + + +def test_complete_task(task_mgr): + session = task_mgr.create_session(email="alice@example.com", user_id="123") + sid = session["session_id"] + task = task_mgr.create_task(session_id=sid, prompt="do work", email="alice@example.com") + tid = task["task_id"] + + # Simulate result.json written by agent + result_file = os.path.join(task_mgr.SESSIONS_DIR, sid, "tasks", tid, "result.json") + with open(result_file, "w") as f: + json.dump({"status": "completed", "summary": "Done", "files_changed": [], "artifacts": {}, "errors": []}, f) + + task_mgr.complete_task(sid, tid) + + # Session should be idle again + with open(os.path.join(task_mgr.SESSIONS_DIR, sid, "session.json")) as f: + data = json.load(f) + assert data["status"] == "idle" + assert data["current_task"] is None + assert tid in data["completed_tasks"] + + +def test_close_session(task_mgr): + session = task_mgr.create_session(email="alice@example.com", user_id="123") + sid = session["session_id"] + + result = task_mgr.close_session(sid) + assert result["status"] == "closed" + + with open(os.path.join(task_mgr.SESSIONS_DIR, sid, "session.json")) as f: + data = json.load(f) + assert data["status"] == "closed" + + +def test_wrap_prompt(task_mgr): + wrapped = task_mgr.wrap_prompt( + task_id="task-007", + session_id="sess-abc", + email="alice@example.com", + prompt="create a pipeline", + context={"tables": ["sales.transactions"]}, + results_dir="/tmp/test" + ) + assert "---CODA-TASK---" in wrapped + assert "task-007" in wrapped + assert "create a pipeline" in wrapped + assert "sales.transactions" in wrapped + assert "result.json" in wrapped + assert "---END-CODA-TASK---" in wrapped +``` + +**Step 2: Run tests to verify they fail** + +Run: `cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp && uv run pytest tests/test_task_manager.py -v` +Expected: FAIL — `ModuleNotFoundError: No module named 'task_manager'` + +**Step 3: Write the task_manager module** + +```python +# task_manager.py +"""Disk-based state manager for MCP sessions and tasks. + +Manages the lifecycle of sessions (PTY-backed Hermes instances) and tasks +(units of work within a session). All state is persisted to ~/.coda/sessions/ +so the MCP transport can remain stateless. +""" +import json +import os +import time +import uuid + +HOME = os.environ.get("HOME", os.path.expanduser("~")) +SESSIONS_DIR = os.path.join(HOME, ".coda", "sessions") + + +class SessionBusyError(Exception): + """Raised when a task is submitted to a session that's already running one.""" + pass + + +class SessionNotFoundError(Exception): + """Raised when a session_id doesn't exist.""" + pass + + +def _session_dir(session_id: str) -> str: + return os.path.join(SESSIONS_DIR, session_id) + + +def _task_dir(session_id: str, task_id: str) -> str: + return os.path.join(SESSIONS_DIR, session_id, "tasks", task_id) + + +def _read_session(session_id: str) -> dict: + path = os.path.join(_session_dir(session_id), "session.json") + if not os.path.isfile(path): + raise SessionNotFoundError(f"Session {session_id} not found") + with open(path) as f: + return json.load(f) + + +def _write_session(session_id: str, data: dict): + path = os.path.join(_session_dir(session_id), "session.json") + with open(path, "w") as f: + json.dump(data, f, indent=2) + + +def create_session(email: str, user_id: str = "", label: str = "") -> dict: + """Create a new session directory and session.json. Returns {session_id, status}.""" + session_id = f"sess-{uuid.uuid4().hex[:12]}" + session_dir = _session_dir(session_id) + os.makedirs(os.path.join(session_dir, "tasks"), exist_ok=True) + + session_data = { + "created_by": email, + "user_id": user_id, + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "status": "idle", + "current_task": None, + "completed_tasks": [], + "label": label, + } + _write_session(session_id, session_data) + + return {"session_id": session_id, "status": "ready"} + + +def create_task( + session_id: str, + prompt: str, + email: str, + context: dict = None, + context_hint: str = None, + timeout_s: int = 3600, + permissions: str = "smart", +) -> dict: + """Create a new task within a session. Returns {task_id, status}. + + Raises SessionBusyError if the session already has a running task. + """ + session_data = _read_session(session_id) + + if session_data["status"] == "busy": + raise SessionBusyError(f"Session {session_id} is busy with task {session_data['current_task']}") + + if session_data["status"] == "closed": + raise SessionNotFoundError(f"Session {session_id} is closed") + + task_id = f"task-{uuid.uuid4().hex[:8]}" + task_dir = _task_dir(session_id, task_id) + os.makedirs(task_dir, exist_ok=True) + + # Write prompt file + results_dir = task_dir + wrapped = wrap_prompt( + task_id=task_id, + session_id=session_id, + email=email, + prompt=prompt, + context=context, + results_dir=results_dir, + context_hint=context_hint, + ) + with open(os.path.join(task_dir, "prompt.txt"), "w") as f: + f.write(wrapped) + + # Write task metadata + with open(os.path.join(task_dir, "meta.json"), "w") as f: + json.dump({ + "task_id": task_id, + "session_id": session_id, + "email": email, + "started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "timeout_s": timeout_s, + "permissions": permissions, + "context_hint": context_hint, + }, f, indent=2) + + # Update session state + session_data["status"] = "busy" + session_data["current_task"] = task_id + _write_session(session_id, session_data) + + return {"task_id": task_id, "status": "running"} + + +def get_task_status(task_id: str, session_id: str) -> dict: + """Get current status of a task. Reads status.jsonl for progress.""" + task_dir = _task_dir(session_id, task_id) + + # Check if result.json exists (task completed) + result_path = os.path.join(task_dir, "result.json") + if os.path.isfile(result_path): + with open(result_path) as f: + result = json.load(f) + return { + "task_id": task_id, + "status": result.get("status", "completed"), + "elapsed_s": _elapsed(task_dir), + } + + # Check for progress in status.jsonl + status_path = os.path.join(task_dir, "status.jsonl") + progress = None + if os.path.isfile(status_path): + with open(status_path) as f: + lines = f.readlines() + if lines: + try: + progress = json.loads(lines[-1].strip()) + except json.JSONDecodeError: + pass + + return { + "task_id": task_id, + "status": "running", + "elapsed_s": _elapsed(task_dir), + "progress": progress, + } + + +def get_task_result(task_id: str, session_id: str) -> dict: + """Get the result of a completed task.""" + task_dir = _task_dir(session_id, task_id) + result_path = os.path.join(task_dir, "result.json") + + if not os.path.isfile(result_path): + return { + "task_id": task_id, + "status": "running", + "elapsed_s": _elapsed(task_dir), + } + + with open(result_path) as f: + result = json.load(f) + + result["task_id"] = task_id + result["elapsed_s"] = _elapsed(task_dir) + return result + + +def complete_task(session_id: str, task_id: str): + """Mark a task as completed and update session state back to idle.""" + session_data = _read_session(session_id) + session_data["status"] = "idle" + session_data["current_task"] = None + if task_id not in session_data.get("completed_tasks", []): + session_data.setdefault("completed_tasks", []).append(task_id) + _write_session(session_id, session_data) + + +def close_session(session_id: str) -> dict: + """Mark a session as closed.""" + session_data = _read_session(session_id) + session_data["status"] = "closed" + _write_session(session_id, session_data) + return {"session_id": session_id, "status": "closed"} + + +def wrap_prompt( + task_id: str, + session_id: str, + email: str, + prompt: str, + context: dict = None, + results_dir: str = "", + context_hint: str = None, +) -> str: + """Wrap a user prompt with the CODA-TASK convention.""" + context_block = "" + if context: + context_block = json.dumps(context, indent=2) + + hint_line = "" + if context_hint: + hint_line = f"context_hint: {context_hint}\n" + + return f"""---CODA-TASK--- +task_id: {task_id} +session_id: {session_id} +user: {email} +{hint_line}results_dir: {results_dir} + +CONTEXT: +{context_block} + +TASK: +{prompt} + +INSTRUCTIONS: +1. Append progress to {results_dir}/status.jsonl + Format: {{"step": "label", "message": "description"}} +2. When done, write {results_dir}/result.json with: + {{"status", "summary", "files_changed", "artifacts", "errors"}} +3. If you delegate to a sub-agent (Claude, Codex, Gemini), update + status.jsonl with delegation steps so the caller can track progress. +---END-CODA-TASK---""" + + +def _elapsed(task_dir: str) -> float: + """Calculate elapsed seconds since task started.""" + meta_path = os.path.join(task_dir, "meta.json") + if os.path.isfile(meta_path): + with open(meta_path) as f: + meta = json.load(f) + started = meta.get("started_at", "") + if started: + try: + started_ts = time.mktime(time.strptime(started, "%Y-%m-%dT%H:%M:%SZ")) + return round(time.time() - started_ts, 1) + except ValueError: + pass + # Fallback: use directory creation time + return round(time.time() - os.path.getctime(task_dir), 1) +``` + +**Step 4: Run tests to verify they pass** + +Run: `cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp && uv run pytest tests/test_task_manager.py -v` +Expected: All 10 tests PASS + +**Step 5: Commit** + +```bash +cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp +git add task_manager.py tests/test_task_manager.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: add task manager for MCP session/task state" +``` + +--- + +### Task 2: Create MCP Server Module + +The MCP server registers 5 tools and delegates to `task_manager.py` for state. It also integrates with the existing PTY session infrastructure in `app.py` for creating terminal sessions and piping prompts. + +**Files:** +- Create: `mcp_server.py` +- Create: `tests/test_mcp_server.py` + +**Step 1: Write the failing tests** + +```python +# tests/test_mcp_server.py +import json +import pytest +from unittest.mock import patch, MagicMock + + +def test_mcp_tool_list(): + """Verify all 5 tools are registered.""" + from mcp_server import mcp + # The server should have 5 tools registered + tools = mcp._tool_manager._tools # internal access for testing + tool_names = [t.name for t in tools.values()] + assert "create_session" in tool_names + assert "run_task" in tool_names + assert "get_status" in tool_names + assert "get_result" in tool_names + assert "close_session" in tool_names + assert len(tool_names) == 5 +``` + +**Step 2: Run test to verify it fails** + +Run: `cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp && uv run pytest tests/test_mcp_server.py -v` +Expected: FAIL — `ModuleNotFoundError: No module named 'mcp_server'` + +**Step 3: Write the MCP server module** + +```python +# mcp_server.py +"""MCP server for CoDA — exposes coding agent capabilities to Genie Code. + +Registers 5 tools: create_session, run_task, get_status, get_result, close_session. +Uses the Python MCP SDK with stateless HTTP transport as required by Genie Code. +""" +import json +import logging +import os +import threading + +from mcp.server.fastmcp import FastMCP + +import task_manager + +logger = logging.getLogger(__name__) + +mcp = FastMCP( + "coda", + stateless_http=True, +) + +# Reference to app.py's session infrastructure — set by mount_mcp() +_app_create_session = None +_app_send_input = None +_app_close_session = None + + +def set_app_hooks(create_session_fn, send_input_fn, close_session_fn): + """Called by app.py to wire MCP tools to the PTY session infrastructure.""" + global _app_create_session, _app_send_input, _app_close_session + _app_create_session = create_session_fn + _app_send_input = send_input_fn + _app_close_session = close_session_fn + + +@mcp.tool() +def create_session( + email: str, + user_id: str = "", + label: str = "", +) -> str: + """Create a new coding agent session backed by Hermes Agent. + + Returns a session_id that can be used with run_task to send work. + Sessions are long-lived — reuse them for follow-up tasks to maintain context. + """ + # Create task manager state on disk + result = task_manager.create_session(email=email, user_id=user_id, label=label) + session_id = result["session_id"] + + # Create the actual PTY session via app.py infrastructure + if _app_create_session: + pty_session_id = _app_create_session(label="hermes-mcp") + # Map our session_id to the PTY session_id + task_manager._update_session_field(session_id, "pty_session_id", pty_session_id) + + return json.dumps(result) + + +@mcp.tool() +def run_task( + session_id: str, + prompt: str, + email: str, + user_id: str = "", + context: str = "{}", + context_hint: str = "", + timeout_s: int = 3600, + permissions: str = "smart", +) -> str: + """Send a coding task to Hermes Agent in an existing session. + + The task runs asynchronously — use get_status to poll progress + and get_result to retrieve the outcome. + + Args: + session_id: From create_session + prompt: Natural language task description + email: User email for audit trail + context: JSON string with Unity Catalog context (tables, schemas, etc.) + context_hint: "new_topic" to signal unrelated work in same session + timeout_s: Max seconds before timeout (default 3600) + permissions: "smart" (default, safe) or "yolo" (full autonomy) + """ + try: + context_dict = json.loads(context) if context else {} + except json.JSONDecodeError: + context_dict = {} + + try: + result = task_manager.create_task( + session_id=session_id, + prompt=prompt, + email=email, + context=context_dict, + context_hint=context_hint or None, + timeout_s=timeout_s, + permissions=permissions, + ) + except task_manager.SessionBusyError as e: + return json.dumps({"error": str(e)}) + except task_manager.SessionNotFoundError as e: + return json.dumps({"error": str(e)}) + + task_id = result["task_id"] + + # Read the wrapped prompt from disk + task_dir = task_manager._task_dir(session_id, task_id) + with open(os.path.join(task_dir, "prompt.txt")) as f: + wrapped_prompt = f.read() + + # Build hermes command + yolo_flag = " --yolo" if permissions == "yolo" else "" + hermes_cmd = f'hermes -z "{task_dir}/prompt.txt"{yolo_flag}\n' + + # Pipe to PTY session in background + if _app_send_input: + session_data = task_manager._read_session(session_id) + pty_session_id = session_data.get("pty_session_id") + if pty_session_id: + # Send the hermes command to the terminal + _app_send_input(pty_session_id, hermes_cmd) + + # Start background watcher for task completion + thread = threading.Thread( + target=_watch_task, + args=(session_id, task_id, timeout_s), + daemon=True, + ) + thread.start() + + return json.dumps(result) + + +@mcp.tool() +def get_status(task_id: str, session_id: str) -> str: + """Check the current status and progress of a running task. + + Returns status (running/completed/failed/timeout), elapsed time, + and the latest progress update from the agent if available. + """ + try: + result = task_manager.get_task_status(task_id, session_id) + return json.dumps(result) + except Exception as e: + return json.dumps({"error": str(e)}) + + +@mcp.tool() +def get_result(task_id: str, session_id: str) -> str: + """Retrieve the structured result of a completed task. + + Returns summary, files changed, artifacts (job IDs, commit hashes, etc.), + and any errors. If the task isn't done yet, returns running status. + """ + try: + result = task_manager.get_task_result(task_id, session_id) + return json.dumps(result) + except Exception as e: + return json.dumps({"error": str(e)}) + + +@mcp.tool() +def close_session(session_id: str) -> str: + """Close a session and clean up resources. + + The PTY process is terminated and session state is marked as closed. + """ + try: + # Close task manager state + result = task_manager.close_session(session_id) + + # Close the PTY session + if _app_close_session: + session_data = task_manager._read_session(session_id) + pty_session_id = session_data.get("pty_session_id") + if pty_session_id: + _app_close_session(pty_session_id) + + return json.dumps(result) + except Exception as e: + return json.dumps({"error": str(e)}) + + +def _watch_task(session_id: str, task_id: str, timeout_s: int): + """Background thread that watches for task completion or timeout.""" + import time + + task_dir = task_manager._task_dir(session_id, task_id) + result_path = os.path.join(task_dir, "result.json") + status_path = os.path.join(task_dir, "status.jsonl") + start = time.time() + last_activity = start + stale_threshold = 300 # 5 minutes with no status update = stale + + while True: + elapsed = time.time() - start + + # Check for result.json (task completed) + if os.path.isfile(result_path): + task_manager.complete_task(session_id, task_id) + logger.info(f"Task {task_id} completed in {elapsed:.0f}s") + return + + # Check for stale (no activity in 5 min) + if os.path.isfile(status_path): + mtime = os.path.getmtime(status_path) + if mtime > last_activity: + last_activity = mtime + + # Timeout: wall clock exceeded AND stale + if elapsed > timeout_s and (time.time() - last_activity) > stale_threshold: + logger.warning(f"Task {task_id} timed out after {elapsed:.0f}s") + # Write a timeout result + with open(result_path, "w") as f: + json.dump({ + "status": "timeout", + "summary": f"Task timed out after {elapsed:.0f} seconds", + "files_changed": [], + "artifacts": {}, + "errors": ["timeout"], + }, f) + task_manager.complete_task(session_id, task_id) + return + + time.sleep(5) # Poll every 5 seconds +``` + +**Step 4: Run tests to verify they pass** + +Run: `cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp && uv run pytest tests/test_mcp_server.py -v` +Expected: PASS + +**Step 5: Commit** + +```bash +cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp +git add mcp_server.py tests/test_mcp_server.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: add MCP server with 5 tools for Genie Code integration" +``` + +--- + +### Task 3: Mount MCP Server in Flask App + +Wire the MCP server into the existing Flask app. Add CORS support, skip auth for `/mcp` (Databricks proxy handles it), and expose helper functions for PTY integration. + +**Files:** +- Modify: `app.py` (add mount + helper functions) +- Modify: `pyproject.toml` (add flask-cors dependency) + +**Step 1: Add flask-cors to dependencies** + +In `pyproject.toml`, add `"flask-cors>=4.0"` to dependencies list. + +**Step 2: Add PTY helper functions to app.py** + +Add these functions after the existing `create_session` route (around line 1081), before the `send_input` route: + +```python +# ── MCP Integration Helpers ────────────────────────────────────────────── + +def mcp_create_pty_session(label: str = "hermes-mcp") -> str: + """Create a PTY session for MCP use. Returns the PTY session_id.""" + master_fd, slave_fd = pty.openpty() + shell_env = os.environ.copy() + shell_env["TERM"] = "xterm-256color" + shell_env.pop("CLAUDECODE", None) + shell_env.pop("CLAUDE_CODE_SESSION", None) + shell_env.pop("DATABRICKS_TOKEN", None) + shell_env.pop("DATABRICKS_HOST", None) + shell_env.pop("GEMINI_API_KEY", None) + if not shell_env.get("HOME") or shell_env["HOME"] == "/": + shell_env["HOME"] = "/app/python/source_code" + local_bin = f"{shell_env['HOME']}/.local/bin" + shell_env["PATH"] = f"{local_bin}:{shell_env.get('PATH', '')}" + projects_dir = os.path.join(shell_env["HOME"], "projects") + os.makedirs(projects_dir, exist_ok=True) + + pid = subprocess.Popen( + ["/bin/bash"], + stdin=slave_fd, stdout=slave_fd, stderr=slave_fd, + preexec_fn=os.setsid, + env=shell_env, + cwd=projects_dir + ).pid + os.close(slave_fd) + + session_id = str(uuid.uuid4()) + with sessions_lock: + if len(sessions) >= MAX_CONCURRENT_SESSIONS: + os.close(master_fd) + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + raise RuntimeError(f"Maximum {MAX_CONCURRENT_SESSIONS} concurrent sessions reached") + sessions[session_id] = { + "master_fd": master_fd, + "pid": pid, + "output_buffer": deque(maxlen=1000), + "lock": threading.Lock(), + "last_poll_time": time.time(), + "created_at": time.time(), + "label": label, + } + + thread = threading.Thread(target=read_pty_output, args=(session_id, master_fd), daemon=True) + thread.start() + log_telemetry("agent", label) + return session_id + + +def mcp_send_input(session_id: str, data: str): + """Send input to a PTY session. Used by MCP to pipe hermes commands.""" + sess = _get_session(session_id) + if not sess: + return + with sess["lock"]: + try: + os.write(sess["master_fd"], data.encode()) + except OSError: + pass + + +def mcp_close_pty_session(session_id: str): + """Close a PTY session. Used by MCP close_session tool.""" + sess = _get_session(session_id) + if not sess: + return + terminate_session(session_id, sess["pid"], sess["master_fd"]) +``` + +**Step 3: Mount the MCP app and add CORS** + +At the end of `app.py`, before the `if __name__ == "__main__"` block (around line 1298), add: + +```python +# ── MCP Server Mount ───────────────────────────────────────────────────── +from flask_cors import CORS +from mcp_server import mcp, set_app_hooks + +# CORS for Genie Code cross-origin requests +databricks_host = os.environ.get("DATABRICKS_HOST", "") +if databricks_host: + CORS(app, origins=[ensure_https(databricks_host)], supports_credentials=True) + +# Wire MCP tools to PTY infrastructure +set_app_hooks( + create_session_fn=mcp_create_pty_session, + send_input_fn=mcp_send_input, + close_session_fn=mcp_close_pty_session, +) + +# Mount MCP as ASGI app at /mcp +from werkzeug.middleware.dispatcher import DispatcherMiddleware +from a]syncio import run as arun + +mcp_asgi_app = mcp.streamable_http_app() + +# Bridge ASGI MCP app into Flask's WSGI world +# We use a thin WSGI wrapper since Flask is WSGI and MCP SDK produces ASGI +import asyncio +from io import BytesIO + +def mcp_wsgi_app(environ, start_response): + """WSGI-to-ASGI bridge for the MCP endpoint.""" + # Read request body + content_length = int(environ.get('CONTENT_LENGTH', 0) or 0) + body = environ['wsgi.input'].read(content_length) if content_length else b'' + + async def run_asgi(): + response_started = False + status_code = None + response_headers = None + response_body = BytesIO() + + async def receive(): + return {"type": "http.request", "body": body} + + async def send(message): + nonlocal response_started, status_code, response_headers + if message["type"] == "http.response.start": + status_code = message["status"] + response_headers = [ + (k.decode() if isinstance(k, bytes) else k, + v.decode() if isinstance(v, bytes) else v) + for k, v in message.get("headers", []) + ] + response_started = True + elif message["type"] == "http.response.body": + response_body.write(message.get("body", b"")) + + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": environ["REQUEST_METHOD"], + "path": environ.get("PATH_INFO", "/"), + "query_string": environ.get("QUERY_STRING", "").encode(), + "headers": [ + (k.lower().replace("http_", "").replace("_", "-").encode(), + v.encode()) + for k, v in environ.items() + if k.startswith("HTTP_") + ] + ( + [(b"content-type", environ["CONTENT_TYPE"].encode())] + if environ.get("CONTENT_TYPE") else [] + ), + "server": (environ.get("SERVER_NAME", "localhost"), + int(environ.get("SERVER_PORT", 8000))), + } + + await mcp_asgi_app(scope, receive, send) + return status_code, response_headers, response_body.getvalue() + + status_code, headers, body_bytes = asyncio.run(run_asgi()) + status_str = f"{status_code} OK" + start_response(status_str, headers or []) + return [body_bytes] + +app.wsgi_app = DispatcherMiddleware(app.wsgi_app, {"/mcp": mcp_wsgi_app}) +``` + +**Step 4: Update auth bypass for /mcp path** + +In `app.py` line 808, update the auth bypass to include `/mcp`: + +```python +# Before: +if request.path in ("/health", "/api/setup-status", ...): +# After: +if request.path in ("/health", "/api/setup-status", "/api/pat-status", "/api/configure-pat", "/api/app-state") or request.path.startswith("/socket.io") or request.path.startswith("/mcp"): +``` + +Note: `/mcp` auth is handled by the Databricks Apps proxy (same as all other routes), but the Flask `before_request` check would reject because MCP requests from Genie Code may not carry the same headers as browser requests. The Databricks Apps proxy still enforces authentication before the request reaches CoDA. + +**Step 5: Run the app locally to verify mount** + +Run: `cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp && uv run python -c "from app import app; print('MCP mounted at /mcp'); print([rule.rule for rule in app.url_map.iter_rules()])"` +Expected: No import errors, `/mcp` visible in routes + +**Step 6: Commit** + +```bash +cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp +git add app.py pyproject.toml mcp_server.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: mount MCP server at /mcp with CORS and PTY integration" +``` + +--- + +### Task 4: Add _update_session_field to task_manager + +The MCP server needs to store the `pty_session_id` mapping. Add the helper and its test. + +**Files:** +- Modify: `task_manager.py` (add `_update_session_field`) +- Modify: `tests/test_task_manager.py` (add test) + +**Step 1: Add test** + +```python +# Append to tests/test_task_manager.py + +def test_update_session_field(task_mgr): + session = task_mgr.create_session(email="alice@example.com", user_id="123") + sid = session["session_id"] + + task_mgr._update_session_field(sid, "pty_session_id", "pty-abc-123") + + with open(os.path.join(task_mgr.SESSIONS_DIR, sid, "session.json")) as f: + data = json.load(f) + assert data["pty_session_id"] == "pty-abc-123" +``` + +**Step 2: Add the function to task_manager.py** + +After the `_write_session` function: + +```python +def _update_session_field(session_id: str, key: str, value): + """Update a single field in session.json.""" + data = _read_session(session_id) + data[key] = value + _write_session(session_id, data) +``` + +**Step 3: Run tests** + +Run: `cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp && uv run pytest tests/test_task_manager.py -v` +Expected: All 11 tests PASS + +**Step 4: Commit** + +```bash +cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp +git add task_manager.py tests/test_task_manager.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: add _update_session_field helper for PTY mapping" +``` + +--- + +### Task 5: Update requirements.txt + +Regenerate requirements after adding flask-cors. + +**Files:** +- Modify: `pyproject.toml` (already done in Task 3) +- Regenerate: `requirements.txt` + +**Step 1: Regenerate requirements** + +Run: `cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp && uv pip compile pyproject.toml -o requirements.txt` + +**Step 2: Commit** + +```bash +cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp +git add pyproject.toml requirements.txt +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "chore: add flask-cors dependency" +``` + +--- + +### Task 6: Integration Test — End-to-End MCP Flow + +Test the full flow: create session → run task → check status → get result → close session. + +**Files:** +- Create: `tests/test_mcp_integration.py` + +**Step 1: Write the integration test** + +```python +# tests/test_mcp_integration.py +"""Integration test for MCP server flow (no real PTY, mocked app hooks).""" +import json +import os +import pytest +from unittest.mock import patch, MagicMock + +import task_manager +import mcp_server + + +@pytest.fixture(autouse=True) +def setup_env(tmp_path): + """Redirect all state to temp dir and mock PTY hooks.""" + with patch.object(task_manager, "SESSIONS_DIR", str(tmp_path / "sessions")): + # Mock the app hooks (no real PTY in tests) + mcp_server.set_app_hooks( + create_session_fn=lambda label: "pty-mock-123", + send_input_fn=MagicMock(), + close_session_fn=MagicMock(), + ) + yield tmp_path + + +def test_full_mcp_flow(): + """End-to-end: create → run → status → result → close.""" + # 1. Create session + result = json.loads(mcp_server.create_session(email="alice@test.com", user_id="u1")) + assert result["status"] == "ready" + sid = result["session_id"] + + # 2. Run task + result = json.loads(mcp_server.run_task( + session_id=sid, + prompt="create a sales pipeline", + email="alice@test.com", + context='{"tables": ["sales.transactions"]}', + )) + assert result["status"] == "running" + tid = result["task_id"] + + # 3. Check status (running, no progress yet) + status = json.loads(mcp_server.get_status(task_id=tid, session_id=sid)) + assert status["status"] == "running" + assert status["progress"] is None + + # 4. Simulate agent writing progress + task_dir = task_manager._task_dir(sid, tid) + with open(os.path.join(task_dir, "status.jsonl"), "w") as f: + f.write(json.dumps({"step": "coding", "message": "Writing pipeline"}) + "\n") + + status = json.loads(mcp_server.get_status(task_id=tid, session_id=sid)) + assert status["progress"]["step"] == "coding" + + # 5. Simulate agent writing result + with open(os.path.join(task_dir, "result.json"), "w") as f: + json.dump({ + "status": "completed", + "summary": "Created sales pipeline with 3 stages", + "files_changed": ["pipelines/sales.py"], + "artifacts": {"job_id": "789"}, + "errors": [] + }, f) + + # 6. Get result + result = json.loads(mcp_server.get_result(task_id=tid, session_id=sid)) + assert result["status"] == "completed" + assert result["summary"] == "Created sales pipeline with 3 stages" + assert result["artifacts"]["job_id"] == "789" + + # 7. Complete and close + task_manager.complete_task(sid, tid) + result = json.loads(mcp_server.close_session(session_id=sid)) + assert result["status"] == "closed" + + +def test_busy_session_rejects(): + """Running a second task on a busy session should return error.""" + result = json.loads(mcp_server.create_session(email="bob@test.com")) + sid = result["session_id"] + + # First task + json.loads(mcp_server.run_task(session_id=sid, prompt="task 1", email="bob@test.com")) + + # Second task should fail + result = json.loads(mcp_server.run_task(session_id=sid, prompt="task 2", email="bob@test.com")) + assert "error" in result + assert "busy" in result["error"].lower() +``` + +**Step 2: Run tests** + +Run: `cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp && uv run pytest tests/test_mcp_integration.py -v` +Expected: All 2 tests PASS + +**Step 3: Run all tests together** + +Run: `cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp && uv run pytest tests/ -v` +Expected: All tests PASS + +**Step 4: Commit** + +```bash +cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp +git add tests/test_mcp_integration.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "test: add end-to-end MCP integration test" +``` + +--- + +## Summary + +| Task | What | Files | +|------|------|-------| +| 1 | Task manager (disk state) | `task_manager.py`, `tests/test_task_manager.py` | +| 2 | MCP server (5 tools) | `mcp_server.py`, `tests/test_mcp_server.py` | +| 3 | Flask mount + CORS + PTY helpers | `app.py`, `pyproject.toml` | +| 4 | Session field helper | `task_manager.py`, `tests/test_task_manager.py` | +| 5 | Dependencies | `pyproject.toml`, `requirements.txt` | +| 6 | Integration test | `tests/test_mcp_integration.py` | + +Total: 4 new files, 2 modified files, ~400 lines of production code, ~250 lines of tests. diff --git a/docs/superpowers/plans/2026-05-27-coda-mcp-live-session-url.md b/docs/superpowers/plans/2026-05-27-coda-mcp-live-session-url.md new file mode 100644 index 0000000..ade3838 --- /dev/null +++ b/docs/superpowers/plans/2026-05-27-coda-mcp-live-session-url.md @@ -0,0 +1,1900 @@ +# CoDA MCP Live Session URL Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a `viewer_url` to CoDA MCP tool responses so the calling user can watch hermes execute live in a browser, with a 5-minute grace period after task completion and indefinite static replay from an on-disk PTY transcript. + +**Architecture:** Tee PTY bytes to `~/.coda/sessions/{sess}/tasks/{task}/transcript.log` from `read_pty_output`. Replace the immediate post-completion close in `_watch_task` with a `threading.Timer(300, close)`. Mark grace-period PTYs to exempt them from `MAX_CONCURRENT_SESSIONS`. Build `viewer_url` by capturing `X-Forwarded-Host` from inbound requests in an ASGI middleware. The Flask `/api/session/attach` endpoint adds a replay fallback that returns transcript bytes when the live PTY is gone. The SPA reads `?session=` on boot and routes to either the existing `_doAttach` (live) or a new `_doReplay` (static, chunked). + +**Tech Stack:** Python 3 (Flask + FastMCP + python-socketio AsyncServer + Starlette + uvicorn), xterm.js, pytest, `uv` for runs. + +**Spec:** `docs/superpowers/specs/2026-05-27-coda-mcp-live-session-url-design.md` at commit `02431c8` on `feat/coda-mcp-server`. + +--- + +## Conventions used in this plan + +- Worktree: `/Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp/` +- All `git commit` commands use `-c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty"` (per repo convention). No `Co-authored-by` line. +- All pytest invocations use `uv run pytest ...` (per repo convention). +- All file paths are relative to the worktree root. + +--- + +## Task 1: `coda_mcp/url_builder.py` — base URL resolution module + +**Files:** +- Create: `coda_mcp/url_builder.py` +- Test: `tests/test_url_builder.py` (new) + +- [ ] **Step 1: Write the failing tests** + +Create `tests/test_url_builder.py`: + +```python +"""Tests for url_builder module — base URL resolution for viewer_url.""" +import os +import importlib +from unittest import mock + +import pytest + + +@pytest.fixture(autouse=True) +def _reset_module(): + """Re-import url_builder fresh for each test (module-level cache).""" + from coda_mcp import url_builder + importlib.reload(url_builder) + yield + + +def test_returns_none_when_neither_env_nor_cache(): + from coda_mcp import url_builder + assert url_builder.build_viewer_url("pty-1") is None + + +def test_env_override_wins(): + from coda_mcp import url_builder + with mock.patch.dict(os.environ, {"CODA_APP_URL": "https://override.example.com"}): + assert url_builder.build_viewer_url("pty-1") == \ + "https://override.example.com/?session=pty-1" + + +def test_env_override_strips_trailing_slash(): + from coda_mcp import url_builder + with mock.patch.dict(os.environ, {"CODA_APP_URL": "https://override.example.com/"}): + assert url_builder.build_viewer_url("pty-1") == \ + "https://override.example.com/?session=pty-1" + + +def test_header_capture_used_when_no_env(): + from coda_mcp import url_builder + url_builder.capture_from_headers("app.databricksapps.com") + assert url_builder.build_viewer_url("pty-1") == \ + "https://app.databricksapps.com/?session=pty-1" + + +def test_env_overrides_header_capture(): + from coda_mcp import url_builder + url_builder.capture_from_headers("captured.example.com") + with mock.patch.dict(os.environ, {"CODA_APP_URL": "https://override.example.com"}): + assert url_builder.build_viewer_url("pty-1") == \ + "https://override.example.com/?session=pty-1" + + +def test_header_capture_overwrites_previous(): + from coda_mcp import url_builder + url_builder.capture_from_headers("first.example.com") + url_builder.capture_from_headers("second.example.com") + assert "second.example.com" in url_builder.build_viewer_url("pty-1") + + +def test_capture_empty_string_does_not_overwrite(): + from coda_mcp import url_builder + url_builder.capture_from_headers("good.example.com") + url_builder.capture_from_headers("") + assert "good.example.com" in url_builder.build_viewer_url("pty-1") + + +def test_capture_none_does_not_crash(): + from coda_mcp import url_builder + url_builder.capture_from_headers(None) + assert url_builder.build_viewer_url("pty-1") is None +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `uv run pytest tests/test_url_builder.py -v` +Expected: ImportError on `from coda_mcp import url_builder` — module does not exist yet. + +- [ ] **Step 3: Implement `coda_mcp/url_builder.py`** + +Create `coda_mcp/url_builder.py`: + +```python +"""Builds the viewer_url returned by CoDA MCP tools. + +Resolution order: +1. ``CODA_APP_URL`` env var (explicit override for local dev / power users). +2. Module-level cache populated by ``AppUrlCaptureMiddleware`` from the + ``X-Forwarded-Host`` header (officially provided by Databricks Apps). +3. ``None`` — caller omits the field entirely. + +The cache is process-global (single uvicorn worker per app) and refreshed +on every inbound HTTP request. +""" +from __future__ import annotations + +import os +from typing import Optional + +_app_url_cache: Optional[str] = None + + +def capture_from_headers(host: Optional[str]) -> None: + """Called by the ASGI middleware on every inbound HTTP request. + + No-op when ``host`` is falsy (None or empty) to avoid wiping a good + cache value with a missing header on a probe/CORS preflight. + """ + global _app_url_cache + if host: + _app_url_cache = host + + +def build_viewer_url(pty_session_id: str) -> Optional[str]: + """Return the full viewer URL for a PTY session, or None if no base is known.""" + override = os.environ.get("CODA_APP_URL", "").strip() + if override: + base = override.rstrip("/") + elif _app_url_cache: + base = f"https://{_app_url_cache}" + else: + return None + return f"{base}/?session={pty_session_id}" +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `uv run pytest tests/test_url_builder.py -v` +Expected: 8 passed. + +- [ ] **Step 5: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add coda_mcp/url_builder.py tests/test_url_builder.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat(coda-mcp): url_builder module for viewer_url resolution" +``` + +--- + +## Task 2: `task_manager.find_task_dir_by_pty_session` — reverse lookup with TTL cache + +**Files:** +- Modify: `coda_mcp/task_manager.py` (add new function at end, before `cleanup_expired_tasks`) +- Test: `tests/test_task_manager.py` (extend) + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/test_task_manager.py` (locate existing test file; this assumes pytest fixtures `tmp_path` and patching of `SESSIONS_DIR` already exist in the file — confirm pattern, otherwise use the snippet below as a self-contained module): + +```python +import json +import os +import time +from unittest import mock + +import pytest + +from coda_mcp import task_manager + + +@pytest.fixture +def sessions_root(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + # Reset the lookup cache between tests + task_manager._pty_lookup_cache.clear() + return tmp_path + + +def _make_session_dir(root, sess_id, pty_id, current_task=None, completed=None): + sdir = root / sess_id + (sdir / "tasks").mkdir(parents=True) + data = { + "session_id": sess_id, + "pty_session_id": pty_id, + "current_task": current_task, + "completed_tasks": completed or [], + "status": "ready", + } + (sdir / "session.json").write_text(json.dumps(data)) + return sdir + + +def test_find_task_dir_hits_current_task(sessions_root): + _make_session_dir(sessions_root, "sess-A", "pty-1", current_task="task-X") + result = task_manager.find_task_dir_by_pty_session("pty-1") + assert result == str(sessions_root / "sess-A" / "tasks" / "task-X") + + +def test_find_task_dir_falls_back_to_last_completed(sessions_root): + _make_session_dir( + sessions_root, "sess-A", "pty-1", + current_task=None, + completed=["task-old", "task-recent"], + ) + result = task_manager.find_task_dir_by_pty_session("pty-1") + assert result == str(sessions_root / "sess-A" / "tasks" / "task-recent") + + +def test_find_task_dir_returns_none_when_no_match(sessions_root): + _make_session_dir(sessions_root, "sess-A", "pty-1", current_task="task-X") + assert task_manager.find_task_dir_by_pty_session("pty-NONEXIST") is None + + +def test_find_task_dir_ignores_corrupt_session_json(sessions_root): + sdir = sessions_root / "sess-bad" + sdir.mkdir() + (sdir / "session.json").write_text("not json {{{") + _make_session_dir(sessions_root, "sess-good", "pty-1", current_task="task-X") + assert task_manager.find_task_dir_by_pty_session("pty-1") == \ + str(sessions_root / "sess-good" / "tasks" / "task-X") + + +def test_find_task_dir_cache_hits_within_ttl(sessions_root): + _make_session_dir(sessions_root, "sess-A", "pty-1", current_task="task-X") + task_manager.find_task_dir_by_pty_session("pty-1") + # Remove session.json — cache should still return the hit + (sessions_root / "sess-A" / "session.json").unlink() + assert task_manager.find_task_dir_by_pty_session("pty-1") == \ + str(sessions_root / "sess-A" / "tasks" / "task-X") + + +def test_find_task_dir_cache_expires(sessions_root, monkeypatch): + monkeypatch.setattr(task_manager, "_PTY_LOOKUP_TTL", 0.01) + _make_session_dir(sessions_root, "sess-A", "pty-1", current_task="task-X") + task_manager.find_task_dir_by_pty_session("pty-1") + (sessions_root / "sess-A" / "session.json").unlink() + time.sleep(0.02) + assert task_manager.find_task_dir_by_pty_session("pty-1") is None + + +def test_find_task_dir_no_sessions_dir(sessions_root, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", "/nonexistent/path/that/does/not/exist") + assert task_manager.find_task_dir_by_pty_session("pty-1") is None +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `uv run pytest tests/test_task_manager.py -v -k find_task_dir` +Expected: 7 failures with `AttributeError: module 'coda_mcp.task_manager' has no attribute 'find_task_dir_by_pty_session'`. + +- [ ] **Step 3: Add module-level cache and function** + +Edit `coda_mcp/task_manager.py`. Near the top, after the existing module constants (after `TASK_TTL_S = ...`): + +```python +# ── PTY → task-dir reverse lookup (used by attach_session replay fallback) ── + +_pty_lookup_cache: dict[str, tuple[str, float]] = {} # pty_id -> (task_dir, ts) +_PTY_LOOKUP_TTL = 60.0 # seconds +``` + +Then before `def cleanup_expired_tasks()`, add: + +```python +def find_task_dir_by_pty_session(pty_session_id: str) -> str | None: + """Find the task dir whose session.json carries this pty_session_id. + + Returns the path to the active task dir, or — if the session has completed — + the most recently completed task dir. Returns None on no match. + + Cached for ``_PTY_LOOKUP_TTL`` seconds to avoid disk scans on every browser + refresh. + + Invariant: CoDA MCP sessions are ephemeral — one task per session. If the + lifecycle ever changes to allow multiple tasks per session, this function + must be revisited to pick the active or grace-period task rather than + ``completed_tasks[-1]``. + """ + now = time.time() + cached = _pty_lookup_cache.get(pty_session_id) + if cached and (now - cached[1]) < _PTY_LOOKUP_TTL: + return cached[0] + + if not os.path.isdir(SESSIONS_DIR): + return None + + for sess_name in os.listdir(SESSIONS_DIR): + sess_file = os.path.join(SESSIONS_DIR, sess_name, "session.json") + try: + with open(sess_file) as f: + data = json.load(f) + except (OSError, json.JSONDecodeError): + continue + + if data.get("pty_session_id") != pty_session_id: + continue + + candidate = data.get("current_task") or ( + data["completed_tasks"][-1] if data.get("completed_tasks") else None + ) + if candidate: + tdir = os.path.join(SESSIONS_DIR, sess_name, "tasks", candidate) + _pty_lookup_cache[pty_session_id] = (tdir, now) + return tdir + + return None +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `uv run pytest tests/test_task_manager.py -v -k find_task_dir` +Expected: 7 passed. + +- [ ] **Step 5: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add coda_mcp/task_manager.py tests/test_task_manager.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat(coda-mcp): find_task_dir_by_pty_session lookup with TTL cache" +``` + +--- + +## Task 3: `app.py::read_pty_output` — tee PTY bytes to transcript with lock-guarded writes + +**Files:** +- Modify: `app.py` (top: new constant; `read_pty_output` function lines 861-910) +- Test: `tests/test_transcript.py` (new — standalone unit tests for the tee logic; integration tested later) + +- [ ] **Step 1: Write the failing tests** + +Create `tests/test_transcript.py`: + +```python +"""Unit tests for the transcript tee in read_pty_output. + +These tests exercise the tee logic directly by simulating output dispatch into +a synthesized session dict and a real on-disk transcript file. The full PTY +read loop is not exercised here — see test_mcp_integration.py for E2E. +""" +import os +import stat +import threading +from pathlib import Path + +import pytest + + +@pytest.fixture +def session_dict(tmp_path): + """Build a minimally valid sessions[pty_id] entry with a real transcript handle.""" + transcript = tmp_path / "transcript.log" + fh = open(transcript, "ab", buffering=0) + os.fchmod(fh.fileno(), 0o600) + return { + "transcript_path": str(transcript), + "transcript_fh": fh, + "transcript_bytes": 0, + "lock": threading.Lock(), + } + + +def _write_chunk(session, output: bytes, cap: int = 10 * 1024 * 1024) -> None: + """Mirror the tee logic from read_pty_output for unit testing.""" + from app import _tee_transcript_chunk + _tee_transcript_chunk(session, output, cap=cap) + + +def test_tee_writes_bytes_and_flushes(session_dict): + _write_chunk(session_dict, b"hello world\n") + assert session_dict["transcript_bytes"] == 12 + assert Path(session_dict["transcript_path"]).read_bytes() == b"hello world\n" + + +def test_tee_chmod_is_0600(session_dict): + mode = stat.S_IMODE(os.stat(session_dict["transcript_path"]).st_mode) + assert mode == 0o600 + + +def test_tee_truncation_at_cap(session_dict): + cap = 16 + _write_chunk(session_dict, b"AAAAAAAAAA", cap=cap) + _write_chunk(session_dict, b"BBBBBBBBBBBBBBBBBBBB", cap=cap) + body = Path(session_dict["transcript_path"]).read_bytes() + # 10 A's, then 6 B's, then truncation marker. + assert body.startswith(b"AAAAAAAAAABBBBBB") + assert b"[transcript truncated at" in body + # Handle is closed after marker + assert session_dict["transcript_fh"] is None + + +def test_tee_no_op_when_fh_is_none(session_dict): + session_dict["transcript_fh"] = None + _write_chunk(session_dict, b"should not write") + assert Path(session_dict["transcript_path"]).read_bytes() == b"" + + +def test_tee_handles_write_error(session_dict, monkeypatch): + # Close the handle out from under the tee — write() will ValueError. + session_dict["transcript_fh"].close() + _write_chunk(session_dict, b"this will fail") + # Handle replaced with None; no crash. + assert session_dict["transcript_fh"] is None +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `uv run pytest tests/test_transcript.py -v` +Expected: ImportError on `from app import _tee_transcript_chunk`. + +- [ ] **Step 3: Add the helper and the constant in `app.py`** + +Near the top of `app.py` (after the existing constants block around line 46-50), add: + +```python +TRANSCRIPT_CAP_BYTES = 10 * 1024 * 1024 # 10 MB soft cap per transcript +``` + +Then add the helper (place it near `read_pty_output`, e.g., immediately above it): + +```python +def _tee_transcript_chunk(session, output: bytes, cap: int = TRANSCRIPT_CAP_BYTES) -> None: + """Append PTY output to the transcript file. Single-writer (read_pty_output). + + All file-handle access is under ``session["lock"]`` so we never race the + Timer-driven close path in ``terminate_session``. The ``ValueError`` catch + is belt-and-suspenders for the tiny window where the handle is closed + between the ``is not None`` check and the actual ``write`` call (the lock + prevents this, but be defensive). + """ + with session["lock"]: + fh = session.get("transcript_fh") + written = session.get("transcript_bytes", 0) + if fh is None: + return + remaining = cap - written + if remaining <= 0: + return + chunk = output[:remaining] + try: + fh.write(chunk) + fh.flush() + session["transcript_bytes"] = written + len(chunk) + if len(chunk) < len(output): + fh.write(b"\n[transcript truncated at %d bytes]\n" % cap) + fh.flush() + fh.close() + session["transcript_fh"] = None + except (OSError, ValueError) as exc: + logger.warning("transcript write failed: %s", exc) + try: + fh.close() + except Exception: + pass + session["transcript_fh"] = None +``` + +- [ ] **Step 4: Wire the tee into `read_pty_output`** + +In `app.py::read_pty_output`, locate the block (currently around line 880-888): + +```python + decoded = output.decode(errors="replace") + with session_lock: + # Buffer for HTTP polling fallback (AC-15) + session["output_buffer"].append(decoded) + session["last_poll_time"] = time.time() # Keep session alive during WS output + # Push via WebSocket to the session room (AC-8) + _emit_from_thread('terminal_output', + {'session_id': session_id, 'output': decoded}, + room=session_id) +``` + +Immediately after the `_emit_from_thread` call (and before the `else:` branch), add: + +```python + # Tee to transcript file if enabled for this session + _tee_transcript_chunk(session, output) +``` + +- [ ] **Step 5: Run unit tests to verify they pass** + +Run: `uv run pytest tests/test_transcript.py -v` +Expected: 5 passed. + +- [ ] **Step 6: Run existing terminal tests to verify no regression** + +Run: `uv run pytest tests/test_terminal_env_strip.py tests/test_session_linger.py tests/test_session_detach.py -v` +Expected: existing pass count unchanged (no failures introduced). + +- [ ] **Step 7: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add app.py tests/test_transcript.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat: tee PTY output to transcript.log with lock-guarded writes" +``` + +--- + +## Task 4: `app.py` — open transcript handle in `mcp_create_pty_session` + close in `terminate_session` + +**Files:** +- Modify: `app.py::mcp_create_pty_session` (lines ~1324-1387) +- Modify: `app.py::terminate_session` (lines ~912-936) + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/test_transcript.py`: + +```python +def test_mcp_create_pty_session_opens_transcript_when_path_given(tmp_path, monkeypatch): + monkeypatch.setattr("app.MAX_CONCURRENT_SESSIONS", 5) + transcript = tmp_path / "transcript.log" + from app import mcp_create_pty_session, sessions, mcp_close_pty_session + sid = mcp_create_pty_session(label="test", transcript_path=str(transcript)) + try: + assert transcript.exists() + mode = stat.S_IMODE(os.stat(transcript).st_mode) + assert mode == 0o600 + sess = sessions[sid] + assert sess["transcript_path"] == str(transcript) + assert sess["transcript_fh"] is not None + assert sess["transcript_bytes"] == 0 + finally: + mcp_close_pty_session(sid) + + +def test_mcp_create_pty_session_no_transcript_when_path_none(monkeypatch): + monkeypatch.setattr("app.MAX_CONCURRENT_SESSIONS", 5) + from app import mcp_create_pty_session, sessions, mcp_close_pty_session + sid = mcp_create_pty_session(label="test") + try: + sess = sessions[sid] + assert sess.get("transcript_fh") is None + assert sess.get("transcript_path") is None + finally: + mcp_close_pty_session(sid) + + +def test_terminate_session_closes_transcript_handle(tmp_path, monkeypatch): + monkeypatch.setattr("app.MAX_CONCURRENT_SESSIONS", 5) + transcript = tmp_path / "transcript.log" + from app import mcp_create_pty_session, sessions, mcp_close_pty_session + sid = mcp_create_pty_session(label="test", transcript_path=str(transcript)) + fh = sessions[sid]["transcript_fh"] + mcp_close_pty_session(sid) + assert fh.closed + # Session removed from dict + assert sid not in sessions +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `uv run pytest tests/test_transcript.py -v -k "create_pty or terminate"` +Expected: 3 failures — `mcp_create_pty_session` does not yet accept `transcript_path`. + +- [ ] **Step 3: Modify `mcp_create_pty_session` signature** + +In `app.py`, change the signature (line ~1324): + +```python +def mcp_create_pty_session(label: str = "hermes-mcp", transcript_path: str | None = None) -> str: +``` + +After the `os.close(slave_fd)` line (around line 1358) and before `session_id = str(uuid.uuid4())`, add the transcript open. Place it inside the existing flow so the file handle is constructed before being stored: + +```python + # Open transcript file (if requested) before locking the session dict. + transcript_fh = None + if transcript_path: + try: + os.makedirs(os.path.dirname(transcript_path), exist_ok=True) + transcript_fh = open(transcript_path, "ab", buffering=0) + os.fchmod(transcript_fh.fileno(), 0o600) + except OSError as exc: + logger.warning("Could not open transcript at %s: %s", transcript_path, exc) + transcript_fh = None +``` + +Modify the `sessions[session_id] = { ... }` block to include the new fields: + +```python + sessions[session_id] = { + "master_fd": master_fd, + "pid": pid, + "output_buffer": deque(maxlen=1000), + "lock": threading.Lock(), + "last_poll_time": time.time(), + "created_at": time.time(), + "label": label, + "transcript_path": transcript_path if transcript_fh else None, + "transcript_fh": transcript_fh, + "transcript_bytes": 0, + "grace": False, + } +``` + +- [ ] **Step 4: Modify `terminate_session` to close the transcript handle** + +In `app.py::terminate_session` (line ~912), at the top of the function (right after the `logger.info` and the `_emit_from_thread('session_closed', ...)` call), add: + +```python + # Close transcript handle (if any) under per-session lock; swap-then-close + # outside the lock to avoid blocking on slow filesystems. + with sessions_lock: + sess = sessions.get(session_id) + if sess is not None: + with sess["lock"]: + transcript_fh = sess.get("transcript_fh") + sess["transcript_fh"] = None + if transcript_fh is not None: + try: + transcript_fh.close() + except Exception: + pass +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `uv run pytest tests/test_transcript.py -v -k "create_pty or terminate"` +Expected: 3 passed. + +- [ ] **Step 6: Run full transcript test suite** + +Run: `uv run pytest tests/test_transcript.py -v` +Expected: 8 passed. + +- [ ] **Step 7: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add app.py tests/test_transcript.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat: open transcript handle in mcp_create_pty_session; close in terminate_session" +``` + +--- + +## Task 5: `app.py` — grace-period exemption from `MAX_CONCURRENT_SESSIONS` + helper hooks + +**Files:** +- Modify: `app.py` (the two `MAX_CONCURRENT_SESSIONS` check sites + add two new helpers near the bottom near other MCP hook functions) + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/test_transcript.py`: + +```python +def test_grace_period_pty_does_not_count_toward_max(monkeypatch): + monkeypatch.setattr("app.MAX_CONCURRENT_SESSIONS", 2) + from app import mcp_create_pty_session, mcp_close_pty_session, sessions, _mark_grace_for_session + + sid1 = mcp_create_pty_session(label="t1") + sid2 = mcp_create_pty_session(label="t2") + try: + # At cap. A third creation should raise. + with pytest.raises(RuntimeError, match="Maximum"): + mcp_create_pty_session(label="t3") + # Mark one as grace; now we should have headroom. + _mark_grace_for_session(sid1) + assert sessions[sid1]["grace"] is True + sid3 = mcp_create_pty_session(label="t3") + mcp_close_pty_session(sid3) + finally: + for s in [sid1, sid2]: + try: mcp_close_pty_session(s) + except Exception: pass + + +def test_bump_session_last_poll_advances_clock(monkeypatch): + monkeypatch.setattr("app.MAX_CONCURRENT_SESSIONS", 5) + from app import mcp_create_pty_session, mcp_close_pty_session, sessions, _bump_session_last_poll + sid = mcp_create_pty_session(label="t") + try: + baseline = sessions[sid]["last_poll_time"] + _bump_session_last_poll(sid, 300) + assert sessions[sid]["last_poll_time"] >= baseline + 299 + finally: + mcp_close_pty_session(sid) + + +def test_mark_grace_on_missing_session_is_noop(): + from app import _mark_grace_for_session + _mark_grace_for_session("nonexistent-pty-id") # must not raise + + +def test_bump_session_last_poll_missing_is_noop(): + from app import _bump_session_last_poll + _bump_session_last_poll("nonexistent-pty-id", 100) # must not raise +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `uv run pytest tests/test_transcript.py -v -k "grace or bump_session"` +Expected: failures — `_mark_grace_for_session` / `_bump_session_last_poll` don't exist; the cap check still uses raw `len`. + +- [ ] **Step 3: Replace the `MAX_CONCURRENT_SESSIONS` checks** + +There are two checkpoints in `app.py`: + +**Site 1 — `create_session()` (around line 1252):** + +```python + with sessions_lock: + if len(sessions) >= MAX_CONCURRENT_SESSIONS: + return jsonify({"error": f"Maximum {MAX_CONCURRENT_SESSIONS} concurrent sessions reached. Close an existing session first."}), 429 +``` + +Replace with: + +```python + with sessions_lock: + active = sum(1 for s in sessions.values() if not s.get("grace")) + if active >= MAX_CONCURRENT_SESSIONS: + return jsonify({"error": f"Maximum {MAX_CONCURRENT_SESSIONS} concurrent sessions reached. Close an existing session first."}), 429 +``` + +**Site 2 — `mcp_create_pty_session()` (around lines 1326-1330 and again 1362-1371):** + +Both `len(sessions) >= MAX_CONCURRENT_SESSIONS` checks become: + +```python + active = sum(1 for s in sessions.values() if not s.get("grace")) + if active >= MAX_CONCURRENT_SESSIONS: + raise RuntimeError( + f"Maximum {MAX_CONCURRENT_SESSIONS} concurrent sessions reached." + ) +``` + +(Apply at both pre-spawn and post-spawn check sites.) + +- [ ] **Step 4: Add the two helper functions** + +Place near `mcp_close_pty_session` (around line 1399): + +```python +def _mark_grace_for_session(session_id: str) -> None: + """Mark a PTY session as 'in grace period' so it doesn't count toward + MAX_CONCURRENT_SESSIONS. Called by ``_watch_task`` immediately before + scheduling the deferred close Timer. + + No-op if the session does not exist (e.g., already torn down). + """ + with sessions_lock: + sess = sessions.get(session_id) + if sess is None: + return + with sess["lock"]: + sess["grace"] = True + + +def _bump_session_last_poll(session_id: str, delta_s: float) -> None: + """Advance ``last_poll_time`` by ``delta_s`` so the idle reaper can't + preempt the Timer's deferred close. Defensive: at the current 24h + SESSION_TIMEOUT_SECONDS the reaper would never win anyway, but a future + tuning shouldn't break the grace window. + + No-op if the session does not exist. + """ + with sessions_lock: + sess = sessions.get(session_id) + if sess is None: + return + with sess["lock"]: + sess["last_poll_time"] = time.time() + delta_s +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `uv run pytest tests/test_transcript.py -v -k "grace or bump_session"` +Expected: 4 passed. + +- [ ] **Step 6: Run full transcript suite + session limit test for regression** + +Run: `uv run pytest tests/test_transcript.py tests/test_session_limit.py -v` +Expected: all pass. + +- [ ] **Step 7: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add app.py tests/test_transcript.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat: exempt grace-period PTYs from MAX_CONCURRENT_SESSIONS" +``` + +--- + +## Task 6: `mcp_server.py` — wire deferred close via `Timer`; update `set_app_hooks` + +**Files:** +- Modify: `coda_mcp/mcp_server.py` (lines 70-90 hook plumbing; lines 94-148 `_watch_task` + helpers) +- Test: `tests/test_mcp_server.py` (extend) + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/test_mcp_server.py`: + +```python +import threading +from unittest import mock + +from coda_mcp import mcp_server, task_manager + + +def test_set_app_hooks_accepts_grace_and_bump_hooks(): + create = mock.MagicMock() + send = mock.MagicMock() + close = mock.MagicMock() + mark_grace = mock.MagicMock() + bump_poll = mock.MagicMock() + mcp_server.set_app_hooks(create, send, close, mark_grace, bump_poll) + assert mcp_server._app_mark_grace is mark_grace + assert mcp_server._app_bump_poll is bump_poll + + +def test_watch_task_schedules_timer_on_completion(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + # Create a session + task with a faked result.json + s = task_manager.create_session("u@x", "uid", label="t") + sid = s["session_id"] + task_manager._update_session_field(sid, "pty_session_id", "pty-abc") + t = task_manager.create_task(sid, "do thing", "u@x") + tid = t["task_id"] + tdir = task_manager._task_dir(sid, tid) + task_manager._write_json(tdir + "/result.json", {"status": "completed"}) + + mark = mock.MagicMock() + bump = mock.MagicMock() + closer = mock.MagicMock() + mcp_server.set_app_hooks(mock.MagicMock(), mock.MagicMock(), closer, mark, bump) + + timer_created = [] + real_timer = threading.Timer + + def fake_timer(seconds, fn, args=None, kwargs=None): + timer_created.append((seconds, fn, args)) + t = real_timer(seconds, fn, args=args, kwargs=kwargs) + return t + + monkeypatch.setattr(mcp_server.threading, "Timer", fake_timer) + + # Use a very short watch interval and ensure no real Timer fires + monkeypatch.setattr(mcp_server, "GRACE_PERIOD_S", 0.05) + + # Run one iteration manually + mcp_server._watch_task(sid, tid, timeout_s=10) + + # Timer should be scheduled for GRACE_PERIOD_S seconds with closer + pty_session_id + assert len(timer_created) == 1 + delay, fn, args = timer_created[0] + assert delay == 0.05 + assert fn is closer + assert args == ("pty-abc",) + + # _mark_grace and _bump_session_last_poll should have been called + mark.assert_called_once_with("pty-abc") + bump.assert_called_once_with("pty-abc", 0.05) +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `uv run pytest tests/test_mcp_server.py -v -k "set_app_hooks_accepts or watch_task_schedules"` +Expected: failures — extra params on `set_app_hooks` not accepted; `_watch_task` calls close synchronously. + +- [ ] **Step 3: Extend `set_app_hooks` and module state** + +In `coda_mcp/mcp_server.py`, at the top of the "App hooks" block (around line 70), expand: + +```python +_app_create_session = None +_app_send_input = None +_app_close_session = None +_app_mark_grace = None +_app_bump_poll = None + +GRACE_PERIOD_S = 300 # 5 minutes + + +def set_app_hooks( + create_session_fn, + send_input_fn, + close_session_fn, + mark_grace_fn=None, + bump_poll_fn=None, +): + """Wire up Flask app callbacks for PTY operations. + + The two new optional hooks (mark_grace, bump_poll) are used by ``_watch_task`` + to defer PTY close by ``GRACE_PERIOD_S`` after task completion so live viewers + can keep watching for a few minutes. + """ + global _app_create_session, _app_send_input, _app_close_session + global _app_mark_grace, _app_bump_poll + _app_create_session = create_session_fn + _app_send_input = send_input_fn + _app_close_session = close_session_fn + _app_mark_grace = mark_grace_fn + _app_bump_poll = bump_poll_fn +``` + +- [ ] **Step 4: Replace the immediate close inside `_watch_task`** + +Replace the existing `_close_pty_for_session(session_id)` calls in `_watch_task` (one in the completion branch around line 117, one in the timeout branch around line 144) with the deferred-Timer helper. Add a new helper at the bottom of the existing helper section (right after `_close_pty_for_session` around line 161): + +```python +def _schedule_deferred_close(session_id: str) -> None: + """Mark the PTY as in-grace and schedule a delayed close. + + Both completion and timeout paths call this in place of the immediate + ``_close_pty_for_session``. The Timer is a daemon thread so it doesn't + block uvicorn shutdown. + """ + if _app_close_session is None: + return + try: + session = task_manager._read_session(session_id) + except task_manager.SessionNotFoundError: + return + pty_session_id = session.get("pty_session_id") + if not pty_session_id: + return + + if _app_mark_grace is not None: + _app_mark_grace(pty_session_id) + if _app_bump_poll is not None: + _app_bump_poll(pty_session_id, GRACE_PERIOD_S) + + t = threading.Timer(GRACE_PERIOD_S, _app_close_session, args=(pty_session_id,)) + t.daemon = True + t.start() + logger.info( + "Watcher: scheduled deferred close for pty %s in %ds", + pty_session_id, GRACE_PERIOD_S, + ) +``` + +Then in `_watch_task`, replace both occurrences of `_close_pty_for_session(session_id)` with `_schedule_deferred_close(session_id)`. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `uv run pytest tests/test_mcp_server.py -v -k "set_app_hooks_accepts or watch_task_schedules"` +Expected: 2 passed. + +- [ ] **Step 6: Run full mcp_server test suite for regression** + +Run: `uv run pytest tests/test_mcp_server.py -v` +Expected: all pass (existing tests should be unaffected since hooks default to None). + +- [ ] **Step 7: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add coda_mcp/mcp_server.py tests/test_mcp_server.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat(coda-mcp): defer PTY close by GRACE_PERIOD_S via threading.Timer" +``` + +--- + +## Task 7: `mcp_server.py` — return `viewer_url` from all three tools + pass `transcript_path` to PTY creation + update instructions + +**Files:** +- Modify: `coda_mcp/mcp_server.py` (`coda_run` body, `coda_inbox` body, `coda_get_result` body, `instructions` block) + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/test_mcp_server.py`: + +```python +import asyncio +import json +import os +from unittest import mock + +from coda_mcp import mcp_server, task_manager, url_builder + + +def _run(coro): + return asyncio.get_event_loop().run_until_complete(coro) if not asyncio.iscoroutine(coro) else asyncio.run(coro) + + +def test_coda_run_includes_viewer_url_when_builder_returns_one(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + monkeypatch.setattr(url_builder, "_app_url_cache", "app.example.com") + + create = mock.MagicMock(return_value="pty-abc") + send = mock.MagicMock() + closer = mock.MagicMock() + mcp_server.set_app_hooks(create, send, closer, mock.MagicMock(), mock.MagicMock()) + + result_json = asyncio.run(mcp_server.coda_run(prompt="do it", email="u@x")) + result = json.loads(result_json) + assert result["status"] == "running" + assert "?session=pty-abc" in result["viewer_url"] + assert result["viewer_url"].startswith("https://app.example.com") + + +def test_coda_run_omits_viewer_url_when_builder_returns_none(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + monkeypatch.setattr(url_builder, "_app_url_cache", None) + monkeypatch.delenv("CODA_APP_URL", raising=False) + + create = mock.MagicMock(return_value="pty-abc") + mcp_server.set_app_hooks(create, mock.MagicMock(), mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) + + result_json = asyncio.run(mcp_server.coda_run(prompt="do it", email="u@x")) + result = json.loads(result_json) + # viewer_url present but None when builder returns None + assert result.get("viewer_url") is None + + +def test_coda_run_passes_transcript_path_to_create_session(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + create = mock.MagicMock(return_value="pty-abc") + mcp_server.set_app_hooks(create, mock.MagicMock(), mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) + + asyncio.run(mcp_server.coda_run(prompt="do it", email="u@x")) + # create_session was called with transcript_path=... pointing into ~/.coda/sessions//tasks//transcript.log + kwargs = create.call_args.kwargs + assert "transcript_path" in kwargs + assert kwargs["transcript_path"].endswith("transcript.log") + assert "tasks" in kwargs["transcript_path"] + + +def test_coda_inbox_decorates_each_task_with_viewer_url(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + monkeypatch.setattr(url_builder, "_app_url_cache", "app.example.com") + + # Seed one session with one task and a pty_session_id + s = task_manager.create_session("u@x", "uid", label="t") + sid = s["session_id"] + task_manager._update_session_field(sid, "pty_session_id", "pty-xyz") + task_manager.create_task(sid, "prompt", "u@x") + + result_json = asyncio.run(mcp_server.coda_inbox()) + result = json.loads(result_json) + assert len(result["tasks"]) == 1 + assert "viewer_url" in result["tasks"][0] + assert "?session=pty-xyz" in result["tasks"][0]["viewer_url"] + + +def test_coda_get_result_includes_viewer_url(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + monkeypatch.setattr(url_builder, "_app_url_cache", "app.example.com") + + s = task_manager.create_session("u@x", "uid", label="t") + sid = s["session_id"] + task_manager._update_session_field(sid, "pty_session_id", "pty-xyz") + t = task_manager.create_task(sid, "prompt", "u@x") + tid = t["task_id"] + tdir = task_manager._task_dir(sid, tid) + task_manager._write_json(tdir + "/result.json", { + "status": "completed", "summary": "ok", + }) + + result_json = asyncio.run(mcp_server.coda_get_result(tid, sid)) + result = json.loads(result_json) + assert "viewer_url" in result + assert "?session=pty-xyz" in result["viewer_url"] +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `uv run pytest tests/test_mcp_server.py -v -k "viewer_url or transcript_path"` +Expected: failures — fields not present, `transcript_path` not passed. + +- [ ] **Step 3: Modify `coda_run`** + +In `coda_mcp/mcp_server.py`, at the top of the file add the import: + +```python +from coda_mcp import url_builder +``` + +In the body of `coda_run` (around line 219), modify the PTY creation block to compute and pass the transcript path: + +```python + # Create PTY if hooks are wired + if _app_create_session is not None: + transcript_path = os.path.join( + task_manager._task_dir(session_id, _new_task_id_preview := task_manager._new_task_id()), + "transcript.log", + ) +``` + +Wait — `task_id` isn't known until after `task_manager.create_task`. Restructure: create the task FIRST (so we have task_id), then create the PTY with transcript path, then send the input. The existing order is: create_session → create_pty → update session with pty_id → create_task → send_input. We need: create_session → create_task → create_pty(transcript_path) → update session with pty_id → send_input. + +Replace the existing PTY-create + create_task block (lines ~218-258) with this restructured version: + +```python + # Create task first (we need task_id to compute transcript_path). + result = task_manager.create_task( + session_id=session_id, + prompt=prompt, + email=email, + context=ctx, + timeout_s=timeout_s, + permissions=permissions, + previous_session_id=previous_session_id or None, + ) + task_id = result["task_id"] + + pty_session_id = None + if _app_create_session is not None: + transcript_path = os.path.join( + task_manager._task_dir(session_id, task_id), + "transcript.log", + ) + pty_session_id = _app_create_session( + label="hermes-mcp", + transcript_path=transcript_path, + ) + task_manager._update_session_field( + session_id, "pty_session_id", pty_session_id + ) + + # Send to PTY if hooks are wired + if _app_send_input is not None and pty_session_id is not None: + tdir = task_manager._task_dir(session_id, task_id) + prompt_path = os.path.join(tdir, "prompt.txt") + cmd = f'hermes -z "{prompt_path}"' + if permissions == "yolo": + cmd += " --yolo" + cmd += "\n" + _app_send_input(pty_session_id, cmd) + + # Start background watcher + t = threading.Thread( + target=_watch_task, + args=(session_id, task_id, timeout_s), + daemon=True, + ) + t.start() + + return json.dumps({ + "task_id": task_id, + "session_id": session_id, + "status": "running", + "viewer_url": url_builder.build_viewer_url(pty_session_id) if pty_session_id else None, + }) +``` + +- [ ] **Step 4: Add `viewer_url` to `coda_inbox` entries** + +In `coda_inbox` (around line 300), after the `list_all_tasks` call, decorate each entry. Replace: + +```python + tasks = task_manager.list_all_tasks(email=email, status_filter=status) +``` + +with: + +```python + tasks = task_manager.list_all_tasks(email=email, status_filter=status) + # Decorate each task with its viewer URL (if available). + for t in tasks: + sess = task_manager._read_session_safe(t["session_id"]) + pty = sess.get("pty_session_id") if sess else None + if pty: + vu = url_builder.build_viewer_url(pty) + if vu: + t["viewer_url"] = vu +``` + +This requires adding `_read_session_safe` to `task_manager.py` — a wrapper that returns `None` instead of raising. Add it now in `coda_mcp/task_manager.py` next to `_read_session`: + +```python +def _read_session_safe(session_id: str) -> dict | None: + """Read session.json, returning None on missing/corrupt instead of raising.""" + try: + return _read_session(session_id) + except SessionNotFoundError: + return None +``` + +- [ ] **Step 5: Add `viewer_url` to `coda_get_result`** + +In `coda_get_result` (around line 327), after the existing field-setting block, add: + +```python + # Decorate with viewer_url if known + sess = task_manager._read_session_safe(session_id) + pty = sess.get("pty_session_id") if sess else None + if pty: + vu = url_builder.build_viewer_url(pty) + if vu: + result["viewer_url"] = vu +``` + +Place this immediately before `return json.dumps(result)`. + +- [ ] **Step 6: Update FastMCP `instructions`** + +In `coda_mcp/mcp_server.py`, modify the `instructions=` argument to FastMCP (around line 42) by appending a paragraph at the end of the existing instructions string: + +```python + "CHAINING: pass previous_session_id from a completed task's session_id " + "to give the new task context of what was done before.\n\n" + "SHARE THE LIVE URL: When coda_run returns a viewer_url field (non-null), " + "mention it to the user in plain text (e.g. \"you can watch progress at " + "\"). The URL is safe to share — it points to the same Databricks App " + "the user is already authenticated against. Do this on the first mention " + "of the task and any time the user asks where the task is or how to see it." +``` + +- [ ] **Step 7: Run tests to verify they pass** + +Run: `uv run pytest tests/test_mcp_server.py -v -k "viewer_url or transcript_path"` +Expected: 5 passed. + +- [ ] **Step 8: Run full mcp test suite for regression** + +Run: `uv run pytest tests/test_mcp_server.py tests/test_mcp_integration.py -v` +Expected: all pass. + +- [ ] **Step 9: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add coda_mcp/mcp_server.py coda_mcp/task_manager.py tests/test_mcp_server.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat(coda-mcp): return viewer_url from coda_run/inbox/get_result + transcript wiring" +``` + +--- + +## Task 8: `mcp_asgi.py` — capture `X-Forwarded-Host` via ASGI middleware + +**Files:** +- Modify: `coda_mcp/mcp_asgi.py` (add middleware class + register it on `mcp_starlette`) +- Test: `tests/test_app_url_middleware.py` (new) + +- [ ] **Step 1: Write the failing tests** + +Create `tests/test_app_url_middleware.py`: + +```python +"""Tests for AppUrlCaptureMiddleware — populates url_builder._app_url_cache.""" +import asyncio +import importlib + +import pytest + +from coda_mcp import url_builder + + +@pytest.fixture(autouse=True) +def _reset_cache(): + importlib.reload(url_builder) + yield + + +async def _fake_app(scope, receive, send): + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + + +def _make_scope(headers: list[tuple[bytes, bytes]]): + return { + "type": "http", + "asgi": {"version": "3.0"}, + "method": "POST", + "path": "/mcp", + "headers": headers, + } + + +async def _drive(middleware, scope): + sent = [] + async def send(msg): sent.append(msg) + async def receive(): return {"type": "http.request", "body": b"", "more_body": False} + await middleware(scope, receive, send) + + +def test_middleware_captures_x_forwarded_host(): + from coda_mcp.mcp_asgi import AppUrlCaptureMiddleware + mw = AppUrlCaptureMiddleware(_fake_app) + scope = _make_scope([(b"x-forwarded-host", b"app.databricksapps.com")]) + asyncio.run(_drive(mw, scope)) + assert url_builder._app_url_cache == "app.databricksapps.com" + + +def test_middleware_falls_back_to_host_when_no_xforwarded(): + from coda_mcp.mcp_asgi import AppUrlCaptureMiddleware + mw = AppUrlCaptureMiddleware(_fake_app) + scope = _make_scope([(b"host", b"localhost:8000")]) + asyncio.run(_drive(mw, scope)) + assert url_builder._app_url_cache == "localhost:8000" + + +def test_middleware_skips_non_http_scope(): + from coda_mcp.mcp_asgi import AppUrlCaptureMiddleware + mw = AppUrlCaptureMiddleware(_fake_app) + scope = {"type": "lifespan"} + async def receive(): return {"type": "lifespan.startup"} + sent = [] + async def send(msg): sent.append(msg) + # Must not crash. Cache stays None. + asyncio.run(mw(scope, receive, send)) + assert url_builder._app_url_cache is None + + +def test_middleware_no_op_when_no_host_header(): + from coda_mcp.mcp_asgi import AppUrlCaptureMiddleware + mw = AppUrlCaptureMiddleware(_fake_app) + scope = _make_scope([]) + asyncio.run(_drive(mw, scope)) + assert url_builder._app_url_cache is None +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `uv run pytest tests/test_app_url_middleware.py -v` +Expected: ImportError on `AppUrlCaptureMiddleware`. + +- [ ] **Step 3: Add the middleware class to `mcp_asgi.py`** + +At the top of `coda_mcp/mcp_asgi.py` (after imports, around line 28), add: + +```python +from coda_mcp import url_builder + + +class AppUrlCaptureMiddleware: + """Capture X-Forwarded-Host (or Host) from every inbound HTTP request and + populate url_builder._app_url_cache. Used so MCP tools can return a + working viewer_url without manual configuration. + + Caveat: /socket.io/ traffic is intercepted by socketio.ASGIApp *before* + reaching mcp_starlette, so WebSocket connect requests never hit this + middleware. This is fine in practice — every HTTP request to /mcp and to + Flask routes does hit it, which is enough to keep the cache hot. + """ + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope.get("type") == "http": + headers = dict(scope.get("headers") or []) + host_bytes = headers.get(b"x-forwarded-host") or headers.get(b"host") + if host_bytes: + try: + url_builder.capture_from_headers(host_bytes.decode("latin-1")) + except Exception: + pass + await self.app(scope, receive, send) +``` + +- [ ] **Step 4: Register the middleware on `mcp_starlette`** + +In the existing block that adds CORS (around lines 80-86): + +```python +# CORS for MCP and Flask routes +mcp_starlette.add_middleware( + CORSMiddleware, + allow_origins=ALLOWED_ORIGINS or ["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) +``` + +Add a second `add_middleware` call immediately after: + +```python +# Capture X-Forwarded-Host into url_builder cache (for MCP viewer_url). +# Added AFTER CORS so it wraps the CORS-handled request. +mcp_starlette.add_middleware(AppUrlCaptureMiddleware) +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `uv run pytest tests/test_app_url_middleware.py -v` +Expected: 4 passed. + +- [ ] **Step 6: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add coda_mcp/mcp_asgi.py tests/test_app_url_middleware.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat(coda-mcp): AppUrlCaptureMiddleware seeds url_builder from X-Forwarded-Host" +``` + +--- + +## Task 9: `app.py::attach_session` — replay fallback when PTY is gone + +**Files:** +- Modify: `app.py::attach_session` (lines ~1104-1123) +- Test: `tests/test_replay_attach.py` (new) + +- [ ] **Step 1: Write the failing tests** + +Create `tests/test_replay_attach.py`: + +```python +"""Tests for /api/session/attach replay fallback.""" +import json +import os +from pathlib import Path + +import pytest + +from coda_mcp import task_manager + + +@pytest.fixture +def client(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + monkeypatch.setenv("MAX_CONCURRENT_SESSIONS", "5") + from app import app + # Bypass authorization (single-user app pattern used by other tests) + monkeypatch.setattr("app.check_authorization", lambda: True) + with app.test_client() as c: + yield c, tmp_path + + +def _seed_transcript(sessions_root: Path, pty_id: str, content: bytes) -> None: + sess_id = "sess-test" + task_id = "task-test" + sdir = sessions_root / sess_id + tdir = sdir / "tasks" / task_id + tdir.mkdir(parents=True) + (sdir / "session.json").write_text(json.dumps({ + "session_id": sess_id, + "pty_session_id": pty_id, + "current_task": None, + "completed_tasks": [task_id], + "status": "closed", + })) + (tdir / "transcript.log").write_bytes(content) + + +def test_attach_returns_replay_when_pty_gone_and_transcript_exists(client): + c, root = client + _seed_transcript(root, "pty-gone", b"hello\r\nworld\r\n") + resp = c.post("/api/session/attach", json={"session_id": "pty-gone"}) + assert resp.status_code == 200 + data = resp.get_json() + assert data["replay"] is True + assert data["output"] == ["hello\r\nworld\r\n"] + assert data["label"] == "hermes-mcp (replay)" + + +def test_attach_404_when_pty_gone_and_no_transcript(client): + c, root = client + resp = c.post("/api/session/attach", json={"session_id": "pty-nope"}) + assert resp.status_code == 404 +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `uv run pytest tests/test_replay_attach.py -v` +Expected: replay test fails (no fallback); 404 test passes already. + +- [ ] **Step 3: Modify `attach_session`** + +In `app.py::attach_session` (around line 1104), replace the body with: + +```python +@app.route("/api/session/attach", methods=["POST"]) +def attach_session(): + """Reattach to an existing session — returns buffered output for replay. + + If the live PTY is gone but an on-disk transcript exists for this + pty_session_id, return the transcript as ``output`` with ``replay: True``. + """ + data = request.get_json(silent=True) or {} + session_id = data.get("session_id", "") + + sess = _get_session(session_id) + if not sess or sess.get("exited"): + # Replay fallback: look up transcript.log by pty_session_id + from coda_mcp import task_manager as _tm + tdir = _tm.find_task_dir_by_pty_session(session_id) + if tdir: + transcript = os.path.join(tdir, "transcript.log") + if os.path.isfile(transcript): + try: + with open(transcript, "rb") as f: + content = f.read() + return jsonify({ + "session_id": session_id, + "label": "hermes-mcp (replay)", + "output": [content.decode("utf-8", errors="replace")], + "replay": True, + "process": None, + "created_at": None, + }) + except OSError: + pass + return jsonify({"error": "Session not found or exited"}), 404 + + # Existing live-attach path + sess["last_poll_time"] = time.time() + return jsonify({ + "session_id": session_id, + "label": sess.get("label", ""), + "output": list(sess["output_buffer"]), + "process": _get_session_process(sess["pid"]), + "created_at": sess.get("created_at"), + }) +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `uv run pytest tests/test_replay_attach.py -v` +Expected: 2 passed. + +- [ ] **Step 5: Run regression for the existing session-attach tests** + +Run: `uv run pytest tests/test_session_detach.py -v` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add app.py tests/test_replay_attach.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat: attach_session replay fallback reads transcript.log when PTY is gone" +``` + +--- + +## Task 10: `static/index.html` — boot URL parse + `_doReplay` + history hygiene + +**Files:** +- Modify: `static/index.html` + +> **Note**: This is the most "real" change. We add ~50-70 LoC of JS. Tested manually (Playwright not configured in this repo). + +- [ ] **Step 1: Locate the SPA boot path** + +Read `static/index.html` lines 990-1030 (the existing session-picker boot logic) to confirm where pane creation happens after the picker. The new URL-driven branch must run before the picker. + +- [ ] **Step 2: Add boot-time URL parse** + +Find the existing function that runs on `DOMContentLoaded` or the IIFE that initializes the app. Just before it would invoke the session picker, insert: + +```javascript + // ── Deep-link to a CoDA MCP session via ?session= ── + async function _initFromQueryString() { + const params = new URLSearchParams(location.search); + const sessionId = params.get('session'); + if (!sessionId) return false; + + try { + const resp = await fetch('/api/session/attach', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ session_id: sessionId }) + }); + + if (resp.status === 404) { + _renderExpiredPage(sessionId); + return true; // handled, skip picker + } + + const data = await resp.json(); + const term = createTerminalPane({ sessionId, label: data.label || sessionId }); + + if (data.replay) { + const content = (data.output || []).join(''); + await _doReplay(term, sessionId, content); + } else { + await _doAttach(term, sessionId); + if (typeof socket !== 'undefined' && socket) { + socket.emit('join_session', { session_id: sessionId }); + } + } + + return true; // handled, skip picker + } catch (err) { + console.error('deep-link attach failed:', err); + return false; + } + } +``` + +`createTerminalPane({ sessionId, label })` is the name commonly used in this repo for pane creation; if the actual name differs, substitute the local helper. Read the existing pane creation site to confirm and adjust the call site accordingly. + +- [ ] **Step 3: Add `_doReplay`** + +Place near `_doAttach` (around line 1339): + +```javascript + async function _doReplay(term, sessionId, content) { + // Chunk the write to avoid main-thread jank on multi-MB transcripts. + const CHUNK = 64 * 1024; + for (let i = 0; i < content.length; i += CHUNK) { + term.write(content.slice(i, i + CHUNK)); + await new Promise(r => requestAnimationFrame(r)); + } + // Mount a static banner above the pane. + _showReplayBanner(term, sessionId); + // NOTE: do NOT wire term.onData → terminal_input; do NOT include in heartbeat + // session_ids list; do NOT emit join_session. + return sessionId; + } + + function _showReplayBanner(term, sessionId) { + const pane = getAllPanes().find(p => p.sessionId === sessionId); + if (!pane || !pane.element) return; + const banner = document.createElement('div'); + banner.className = 'replay-banner'; + banner.textContent = 'Task completed — viewing replay'; + banner.style.cssText = 'padding:4px 8px;background:#333;color:#aaa;font-size:12px;text-align:center;'; + pane.element.insertBefore(banner, pane.element.firstChild); + } +``` + +- [ ] **Step 4: Add `_renderExpiredPage`** + +Place near `_doReplay`: + +```javascript + function _renderExpiredPage(sessionId) { + const root = document.body; + root.innerHTML = ` +
+

Session expired

+

Session ${sessionId.replace(/[<>]/g, '')} is gone, and no replay is available.

+

The transcript may have aged out after the 24-hour retention window.

+

← Back to terminal

+
+ `; + } +``` + +- [ ] **Step 5: Wire `_initFromQueryString` into the boot path** + +Find where the existing session-picker is shown after `DOMContentLoaded`. Wrap it: + +```javascript + document.addEventListener('DOMContentLoaded', async () => { + // existing init code (sockets, themes, etc.) + + const handled = await _initFromQueryString(); + if (handled) return; + + // existing flow (show session picker, etc.) + }); +``` + +The exact insertion site depends on the existing boot structure — read lines 990-1050 of `static/index.html` to find the right place. + +- [ ] **Step 6: Add history hygiene on pane close** + +Locate the existing pane-close handler. Inside, after the pane is removed, add: + +```javascript + // If this pane was opened via ?session=, drop the query param so a + // refresh doesn't re-attach to a stale id. + const params = new URLSearchParams(location.search); + if (params.get('session') === pane.sessionId) { + history.replaceState({}, '', '/'); + } +``` + +- [ ] **Step 7: Manual smoke test** + +Local dev: + +```bash +uv run uvicorn coda_mcp.mcp_asgi:app --host 0.0.0.0 --port 8000 +``` + +Then open `http://localhost:8000/?session=fake-id` in a browser. Expected: "Session expired" page (404 since no transcript exists). + +Create a fake live session via the regular UI, note its session_id from the picker, then navigate to `http://localhost:8000/?session=` — expected: terminal opens directly attached to that session. + +- [ ] **Step 8: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add static/index.html +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat(spa): deep-link ?session= with live attach + replay rendering" +``` + +--- + +## Task 11: Integration test — E2E grace period + transcript replay + +**Files:** +- Modify: `tests/test_mcp_integration.py` (extend) + +- [ ] **Step 1: Write the failing test** + +Append to `tests/test_mcp_integration.py`: + +```python +import asyncio +import json +import os +import time +from pathlib import Path +from unittest import mock + +import pytest + +from coda_mcp import mcp_server, task_manager, url_builder + + +@pytest.fixture +def mcp_env(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + monkeypatch.setattr(url_builder, "_app_url_cache", "app.example.com") + # Shrink grace for the test + monkeypatch.setattr(mcp_server, "GRACE_PERIOD_S", 2) + return tmp_path + + +def test_end_to_end_grace_and_replay(mcp_env, monkeypatch): + """Stub hermes via direct file I/O, then exercise the full coda_run flow.""" + from app import mcp_create_pty_session, mcp_send_input, mcp_close_pty_session + from app import _mark_grace_for_session, _bump_session_last_poll, sessions + + mcp_server.set_app_hooks( + mcp_create_pty_session, mcp_send_input, mcp_close_pty_session, + _mark_grace_for_session, _bump_session_last_poll, + ) + + # Submit a fake task + result_json = asyncio.run(mcp_server.coda_run( + prompt="test", email="u@x", timeout_s=5, + )) + result = json.loads(result_json) + assert result["status"] == "running" + sess_id = result["session_id"] + task_id = result["task_id"] + pty_id = task_manager._read_session(sess_id)["pty_session_id"] + + # viewer_url returned + assert pty_id in result["viewer_url"] + + # Simulate hermes writing to the PTY by sending input that echoes to bash + mcp_send_input(pty_id, "echo HELLO_FROM_HERMES\n") + time.sleep(0.5) + + # Now simulate hermes completion by writing result.json + tdir = task_manager._task_dir(sess_id, task_id) + Path(tdir).joinpath("result.json").write_text(json.dumps({ + "status": "completed", "summary": "stub", "files_changed": [], + "artifacts": {}, "errors": [], + })) + + # Wait for watcher to pick it up (polls every 5s — shorten via patch below if slow) + # In practice, the test patches the poll interval. For now, manually invoke: + mcp_server._schedule_deferred_close(sess_id) + + # PTY still alive immediately after grace scheduling + assert pty_id in sessions + assert sessions[pty_id]["grace"] is True + + # Wait past GRACE_PERIOD_S + time.sleep(2.5) + + # PTY now gone + assert pty_id not in sessions + + # Transcript file exists and contains the echoed line + transcript = Path(tdir) / "transcript.log" + assert transcript.exists() + assert b"HELLO_FROM_HERMES" in transcript.read_bytes() + + # find_task_dir_by_pty_session now returns the task dir from the on-disk record + found = task_manager.find_task_dir_by_pty_session(pty_id) + assert found == str(tdir) +``` + +- [ ] **Step 2: Run the test** + +Run: `uv run pytest tests/test_mcp_integration.py -v -k end_to_end_grace_and_replay` +Expected: pass. + +- [ ] **Step 3: Run the full test suite for regression** + +Run: `uv run pytest tests/ -v --timeout=60` +Expected: prior pass count + the new tests. No failures. + +- [ ] **Step 4: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add tests/test_mcp_integration.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "test: E2E coverage for grace period + transcript replay" +``` + +--- + +## Task 12: Manual smoke + deployment verification + +**Files:** none (verification only) + +- [ ] **Step 1: Deploy the worktree to the test app** + +From the worktree root: + +```bash +databricks bundle deploy --target test-coda +``` + +(Adjust target name to whatever the existing deployment uses — check `databricks.yml` or `app.yaml` notes.) + +- [ ] **Step 2: Verify in Genie Code** + +In the Databricks workspace, open Genie Code, ensure the Custom MCP server `mcp-test-coda` is connected. Submit a simple task: `"List the files in /tmp"`. + +Expected: +- Genie Code's response mentions a `viewer_url` like `https://mcp-test-coda-.aws.databricksapps.com/?session=`. +- Clicking the URL opens the terminal pre-attached to that session. +- Hermes output streams in real time. + +- [ ] **Step 3: Verify replay** + +After the task completes, wait 6+ minutes (grace period + buffer), then reload the same URL. + +Expected: +- Page loads showing the static transcript of what hermes did. +- "Task completed — viewing replay" banner. +- No input is sent when you type. + +- [ ] **Step 4: Verify chmod on transcript** + +From a shell in the deployed app (workspace terminal or `databricks workspace files` API): + +```bash +ls -la ~/.coda/sessions/*/tasks/*/transcript.log +``` + +Expected: files have mode `-rw-------` (0o600). + +- [ ] **Step 5: Verify `viewer_url` absence locally without env** + +```bash +unset CODA_APP_URL +uv run uvicorn coda_mcp.mcp_asgi:app --host 0.0.0.0 --port 8000 & +SERVER_PID=$! + +# Submit a coda_run via curl-formatted JSON-RPC +curl -s http://localhost:8000/mcp \ + -H 'Content-Type: application/json' \ + -d '{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"coda_run","arguments":{"prompt":"test","email":"local@dev"}}}' + +kill $SERVER_PID +``` + +Expected: the JSON response contains `"viewer_url": "http://localhost:8000/?session=..."` (because the inbound `Host: localhost:8000` was captured). + +- [ ] **Step 6: Final commit (if any verification turned up a fix)** + +If smoke tests revealed issues, fix them as separate commits, then update this checklist. + +--- + +## Self-review notes + +- All eight spec decisions covered: §1 viewer mode → Task 10 `_doReplay`; §2 transcript tee → Tasks 3-4; §3 deferred Timer → Task 6; §4 grace exemption → Task 5; §5 URL form → Tasks 1, 7; §6 ASGI middleware → Task 8; §7 attach replay fallback → Task 9; §8 SPA → Task 10. +- No "TODO" / "TBD" / "implement later" / placeholder text — every step has concrete code, exact paths, exact commands. +- Type/method consistency: + - `set_app_hooks` signature in Task 6 matches the call site updated in Task 11 (`mcp_server.set_app_hooks(create, send, close, mark_grace, bump_poll)` with optional defaults). + - `_mark_grace_for_session` / `_bump_session_last_poll` defined in Task 5 used by Task 6 and Task 11. + - `transcript_path` kwarg added to `mcp_create_pty_session` in Task 4 used by `coda_run` in Task 7. + - `find_task_dir_by_pty_session` defined in Task 2 used by `attach_session` in Task 9. + - `url_builder.build_viewer_url` defined in Task 1 used by `coda_run`/`coda_inbox`/`coda_get_result` in Task 7. +- Spec §3 "Architecture" diagram preserved as the mental model; data flows §5.1-5.4 map to Tasks 7, 9, 6, 9 respectively. +- Risks §9 (secrets, grace race, multi-tab) accepted in the spec; surface in the test plan via the chmod-600 verification in Task 12 step 4. diff --git a/docs/superpowers/plans/2026-05-28-coda-interactive-broaden-source.md b/docs/superpowers/plans/2026-05-28-coda-interactive-broaden-source.md new file mode 100644 index 0000000..1590755 --- /dev/null +++ b/docs/superpowers/plans/2026-05-28-coda-interactive-broaden-source.md @@ -0,0 +1,696 @@ +# `coda_interactive` Broaden Source Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Drop the Git-Folder requirement from `coda_interactive`. `workspace_path` accepts any Databricks Workspace directory. Remove the `branch` parameter. Add a `workspace.get_status` validation step. + +**Architecture:** Single MCP tool simplification on the open PR #67. We replace the Repos API lookup (`client.repos.list` + `client.repos.update`) with a single existence/type check (`client.workspace.get_status` → `_is_directory`). The export helper (`export_workspace_tree`) is unchanged because it already uses the generic Workspace API. Tests are rewritten to match: drop branch-related tests, swap `repos.list` mocks for `workspace.get_status` mocks, add a not-a-directory case. + +**Tech Stack:** Python 3.11, FastMCP, databricks-sdk WorkspaceClient, pytest, MagicMock. + +--- + +## Files modified by this plan + +- **Modify:** `coda_mcp/mcp_server.py` — remove `branch` param, remove repos lookup, add `get_status` validation, update INTERACTIVE HANDOFF instructions paragraph and tool docstring, update import line +- **Modify:** `tests/test_coda_interactive.py` — drop 3 tests, update 4 tests, add 4 tests (3 in Task 1 + 1 instructions-content test in Task 3) +- **Modify:** `docs/superpowers/specs/2026-05-28-coda-interactive-mcp-tool-design.md` — prepend an amendment notice (Task 4) +- **No change:** `coda_mcp/workspace_export.py` — already generic; we just re-use its `_is_directory` helper via import + +## Pre-flight context + +- Worktree path: `/Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp` +- Branch: `feat/coda-mcp-interactive-handoff` (PR #67, open) +- Run tests with `uv run pytest ` (per user's `always use uv` directive) +- Commit identity: `-c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty"` (per user's directive). No AI co-author lines. + +The current `coda_interactive` body is at `coda_mcp/mcp_server.py:370-517`. The current INTERACTIVE HANDOFF paragraph is at `coda_mcp/mcp_server.py:79-93`. The current test file is `tests/test_coda_interactive.py` (385 lines, 11 tests). + +--- + +## Task 1: Rewrite tests for the broadened contract (RED state) + +This task replaces the test file's mocking shape and assertions. Implementation in Task 2 is what makes them pass. + +**Files:** +- Modify: `tests/test_coda_interactive.py` (drop 3 tests, update 4 tests, add 2 tests) + +- [ ] **Step 1: Delete the three branch/git-folder-only tests** + +These three tests no longer make sense because the corresponding code paths are being removed. Remove them entirely from `tests/test_coda_interactive.py`: + +1. `test_coda_interactive_workspace_path_not_found` (lines 42-58) — tests `repos.list()` returning empty. The new code uses `workspace.get_status`, not `repos.list`. A different test covers the missing-path case. +2. `test_coda_interactive_branch_update_failure` (lines 61-83) — tests `repos.update` raising. The `branch` parameter is going away entirely. +3. `test_coda_interactive_skips_branch_update_when_empty` (lines 86-107) — tests that `repos.update` isn't called when branch is empty. The `branch` parameter is going away entirely. + +- [ ] **Step 2: Update the four tests that have stale mock setup** + +These four tests currently set up `fake_repo` and `fake_client.repos.list.return_value = [fake_repo]`. After the change, `coda_interactive` no longer calls `repos.list`. Replace that scaffolding with a `workspace.get_status` mock returning a directory-typed object. + +Add this helper at the top of the file (just after `_no_wait`): + +```python +def _make_dir_status(): + """Build a mock object_type=DIRECTORY response from workspace.get_status.""" + from unittest.mock import MagicMock + status = MagicMock() + status.object_type = "DIRECTORY" + return status +``` + +Then update these four tests by replacing the `fake_repo` + `fake_client.repos.list.return_value = [fake_repo]` block with: + +```python +fake_client = MagicMock() +fake_client.workspace.get_status.return_value = _make_dir_status() +``` + +The tests: +- `test_coda_interactive_export_failure_cleans_partial_dir` (currently line 110) +- `test_coda_interactive_happy_path_sends_agent_command_and_prompt` (currently line 164) +- `test_coda_interactive_agent_command_matrix` (currently line 224) +- `test_coda_interactive_does_not_use_blocking_sleep` (currently line 272) + +In `test_coda_interactive_happy_path_sends_agent_command_and_prompt`, also remove the assertion line referencing `branch` in the return shape if present (re-check after edit — current return shape includes `"branch"`; the new shape does not). The current test does not assert on `result["branch"]`, so no change needed there, but verify after edit. + +- [ ] **Step 3: Add `test_coda_interactive_workspace_path_does_not_exist`** + +Append to the file: + +```python +def test_coda_interactive_workspace_path_does_not_exist(monkeypatch): + """If workspace.get_status raises, return error and don't proceed to PTY.""" + from unittest.mock import MagicMock + from coda_mcp import mcp_server + + fake_client = MagicMock() + fake_client.workspace.get_status.side_effect = Exception("RESOURCE_DOES_NOT_EXIST") + monkeypatch.setattr(mcp_server, "WorkspaceClient", lambda: fake_client) + + pty_created = [] + monkeypatch.setattr( + mcp_server, "_app_create_session", + lambda **kw: pty_created.append(kw) or "should-not-be-used", + ) + + result_str = asyncio.run(mcp_server.coda_interactive( + prompt="hello", + workspace_path="/Workspace/Users/x/nonexistent", + )) + result = json.loads(result_str) + + assert result["status"] == "error" + assert "not found" in result["error"].lower() or "does_not_exist" in result["error"].lower() + # No PTY may be created if validation fails. + assert pty_created == [], f"PTY must not be created when workspace_path is invalid; got {pty_created}" +``` + +- [ ] **Step 4: Add `test_coda_interactive_workspace_path_not_directory`** + +Append to the file: + +```python +def test_coda_interactive_workspace_path_not_directory(monkeypatch): + """If workspace.get_status returns object_type=FILE (or anything not DIRECTORY), return error.""" + from unittest.mock import MagicMock + from coda_mcp import mcp_server + + file_status = MagicMock() + file_status.object_type = "FILE" + fake_client = MagicMock() + fake_client.workspace.get_status.return_value = file_status + monkeypatch.setattr(mcp_server, "WorkspaceClient", lambda: fake_client) + + pty_created = [] + monkeypatch.setattr( + mcp_server, "_app_create_session", + lambda **kw: pty_created.append(kw) or "should-not-be-used", + ) + + result_str = asyncio.run(mcp_server.coda_interactive( + prompt="hello", + workspace_path="/Workspace/Users/x/some-file.py", + )) + result = json.loads(result_str) + + assert result["status"] == "error" + assert "directory" in result["error"].lower() + assert pty_created == [], "PTY must not be created when workspace_path is not a directory" +``` + +- [ ] **Step 5: Add `test_coda_interactive_no_branch_parameter`** + +Signature regression guard so the `branch` arg cannot quietly come back. Append to the file: + +```python +def test_coda_interactive_no_branch_parameter(): + """The branch parameter must not exist on coda_interactive's signature.""" + import inspect + from coda_mcp import mcp_server + + sig = inspect.signature(mcp_server.coda_interactive) + assert "branch" not in sig.parameters, ( + f"coda_interactive must not accept a `branch` parameter (got {list(sig.parameters)}). " + f"The broadened contract handles git-folder branch state on the caller side." + ) +``` + +- [ ] **Step 6: Run the test file — expect failures** + +Run: `uv run pytest tests/test_coda_interactive.py -v` + +Expected: At least the two new tests (`workspace_path_does_not_exist`, `workspace_path_not_directory`), the signature guard (`no_branch_parameter`), and the four updated mock-shape tests all FAIL — because `coda_interactive` still uses `repos.list` and still accepts `branch`. The unchanged tests (`unknown_agent`, `default_agent_is_claude`, the three `_wait_for_agent_ready` tests) should still PASS. + +This is the intended RED state — proves the new tests actually exercise the new code path. + +- [ ] **Step 7: Commit the tests** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add tests/test_coda_interactive.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "test: rewrite coda_interactive tests for broadened workspace-folder contract" +``` + +--- + +## Task 2: Simplify `coda_interactive` implementation (GREEN state) + +**Files:** +- Modify: `coda_mcp/mcp_server.py` (signature, body, import, return shape) + +- [ ] **Step 1: Update the import line to include the directory check helper** + +In `coda_mcp/mcp_server.py:31`, change: + +```python +from coda_mcp.workspace_export import export_workspace_tree +``` + +to: + +```python +from coda_mcp.workspace_export import export_workspace_tree, _is_directory +``` + +`_is_directory` is currently module-private in `workspace_export.py:35`. We import it directly rather than aliasing for two reasons: (a) it is a stable, narrowly-scoped helper already used internally; (b) renaming it would force an unrelated edit. Python permits underscore imports; the cost is one symbol shared across two modules in the same package. + +- [ ] **Step 2: Replace the function signature and body** + +In `coda_mcp/mcp_server.py:370-517`, replace the entire `async def coda_interactive(...)` definition. The full new function body: + +```python +@mcp.tool( + annotations=ToolAnnotations( + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + ), +) +async def coda_interactive( + prompt: str, + workspace_path: str, + agent: str = "claude", + email: str = "", +) -> str: + """Launch an interactive agent session in CoDA, handed off via a viewer URL. + + The MCP caller passes a Databricks Workspace directory path (a Git Folder + or a plain Workspace folder — either works). Coda exports its file tree, + launches the chosen agent (claude default) in that directory, auto-types + ``prompt`` as the first user input, and returns a ``viewer_url`` the + calling user opens in a browser to drive the session. + + Pre-condition: ``workspace_path`` must point to a directory that already + exists in the Databricks Workspace. If the directory is a Git Folder and + the caller wants a specific branch checked out, they must do that + themselves before calling — the export is a server-side snapshot. + + Interactive sessions do NOT appear in ``coda_inbox`` and ``coda_get_result`` + will not return anything for them. The viewer URL is the only handle. + + Allowed agents: claude (default), hermes, codex, gemini, opencode. + """ + if agent not in _ALLOWED_AGENTS: + return json.dumps({ + "status": "error", + "error": f"Unknown agent: {agent!r}. Allowed: {sorted(_ALLOWED_AGENTS)}", + }) + + if WorkspaceClient is None: + return json.dumps({ + "status": "error", + "error": "databricks-sdk not installed", + }) + + client = WorkspaceClient() + + # Validate that the path exists and is a directory. + try: + status = client.workspace.get_status(workspace_path) + except Exception as e: + return json.dumps({ + "status": "error", + "error": f"Workspace path not found: {workspace_path}: {e}", + }) + + if not _is_directory(status): + return json.dumps({ + "status": "error", + "error": f"Workspace path is not a directory: {workspace_path}", + }) + + # Create PTY FIRST so we have its session_id for the project_dir name. + if _app_create_session is None: + return json.dumps({ + "status": "error", + "error": "PTY hook not wired", + }) + + pty_session_id = None + project_dir = None + try: + pty_session_id = _app_create_session( + label=f"{agent}-interactive", + replay_only=False, + ) + + # Build the project dir at the canonical path keyed by PTY id. + project_dir = os.path.join( + os.path.expanduser("~/.coda/projects"), + pty_session_id, + ) + + # Export the Workspace tree into project_dir. + try: + export_workspace_tree(client, workspace_path, project_dir) + except Exception as e: + # Close the PTY and clean up the partial dir. + if _app_close_session is not None: + try: + _app_close_session(pty_session_id) + except Exception: + pass + if os.path.isdir(project_dir): + shutil.rmtree(project_dir, ignore_errors=True) + return json.dumps({ + "status": "error", + "error": f"Failed to export workspace tree: {e}", + }) + + # cd into the project dir. + if _app_send_input is None: + return json.dumps({ + "status": "error", + "error": "PTY send hook not wired", + }) + _app_send_input(pty_session_id, f"cd {shlex.quote(project_dir)}\n") + + # Launch the agent. + launch_cmd = _AGENT_LAUNCH_CMDS[agent] + _app_send_input(pty_session_id, launch_cmd + "\n") + + # Wait briefly for agent initialization, then paste the prompt. + await _wait_for_agent_ready(pty_session_id) + _app_send_input(pty_session_id, prompt + "\n") + + viewer_url = url_builder.build_viewer_url(pty_session_id) + + return json.dumps({ + "status": "launched", + "viewer_url": viewer_url, + "agent": agent, + "project_dir": project_dir, + "workspace_path": workspace_path, + "instructions": ( + "Open viewer_url to attach. The agent is loaded with the " + "project files exported from Workspace and your kickoff " + "prompt typed. Type the agent's quit command (e.g. /quit) " + "and then `exit` to end the session. Note: git history is " + "NOT available in the session — files are an export, not " + "a clone." + ), + }) + except Exception as e: + # Catch-all: ensure no resource leak. + if pty_session_id and _app_close_session is not None: + try: + _app_close_session(pty_session_id) + except Exception: + pass + if project_dir and os.path.isdir(project_dir): + shutil.rmtree(project_dir, ignore_errors=True) + return json.dumps({ + "status": "error", + "error": f"coda_interactive failed: {e}", + }) +``` + +Key changes vs. the existing body: +- `branch: str = ""` parameter removed. +- `client.repos.list` / exact-match filter / `client.repos.update` block removed. +- Replaced by `client.workspace.get_status(workspace_path)` + `_is_directory` check. +- `"branch": branch,` dropped from the return JSON. +- Docstring rewritten to say "Git Folder or plain Workspace folder" and drop the "commit and push to remote" admonition. + +- [ ] **Step 3: Run the test file — expect green** + +Run: `uv run pytest tests/test_coda_interactive.py -v` + +Expected: All tests PASS. If any fail, fix the implementation (not the tests) and re-run. + +- [ ] **Step 4: Run the full unit test suite to catch regressions** + +Run: `uv run pytest tests/ -v --no-header -x` (stop on first failure) + +Expected: All previously-passing tests still pass. The skipped PTY-gated and Docker-gated tests stay skipped (those auto-skip on this machine; no behaviour to verify here). + +If unrelated tests fail, stop and investigate before committing. + +- [ ] **Step 5: Commit the implementation** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add coda_mcp/mcp_server.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat: coda_interactive accepts any Workspace folder, drop branch param + +Replaces the Repos API lookup (repos.list + repos.update) with a single +workspace.get_status check. Caller is now responsible for managing +Git Folder branch state. Workspace path can be a Git Folder or a plain +Workspace folder — either works." +``` + +--- + +## Task 3: Update INTERACTIVE HANDOFF instructions string + +The server-level instructions string surfaced to upstream LLM callers still says "must be a Git Folder ... commit and push to remote." Rewrite to match the broadened contract. + +**Files:** +- Modify: `coda_mcp/mcp_server.py:79-93` (INTERACTIVE HANDOFF paragraph in the `mcp = FastMCP(instructions=...)` block) + +- [ ] **Step 1: Write a test that pins the instructions string content** + +Append to `tests/test_coda_interactive.py`: + +```python +def test_interactive_handoff_instructions_describe_broadened_contract(): + """The server-level INTERACTIVE HANDOFF paragraph must reflect the broadened contract.""" + from coda_mcp import mcp_server + + instructions = mcp_server.mcp.instructions + + # Must mention coda_interactive. + assert "coda_interactive" in instructions + + # Must NOT still claim a Git Folder is required. + lowered = instructions.lower() + assert "must be a databricks workspace git folder" not in lowered, ( + "Instructions still require a Git Folder — broadened contract was not applied." + ) + assert "commit and push" not in lowered, ( + "Instructions still tell the caller to commit and push — only relevant for Git Folders, " + "but the broadened contract accepts plain folders too." + ) + + # Must mention that plain folders work. + # Either "git folder or" phrasing, or "plain workspace folder" — accept either. + assert ( + "git folder or" in lowered + or "plain workspace folder" in lowered + or "plain folder" in lowered + ), "Instructions must mention that plain Workspace folders are accepted." + + # Must surface the upload-then-handoff pattern so upstream callers know + # to push files into the workspace BEFORE calling. + assert ( + "upload" in lowered + or "workspace.import" in lowered + or "post" in lowered + ), ( + "Instructions must tell the upstream caller to upload/import the project " + "files into the Workspace first if they aren't already there — the tool " + "only reads existing Workspace paths, it doesn't accept inline payloads." + ) +``` + +Run: `uv run pytest tests/test_coda_interactive.py::test_interactive_handoff_instructions_describe_broadened_contract -v` + +Expected: FAIL — the current instructions string still says "must be a Databricks Workspace Git Folder." + +- [ ] **Step 2: Rewrite the INTERACTIVE HANDOFF paragraph in `mcp_server.py:79-93`** + +In `coda_mcp/mcp_server.py`, find the block beginning at line 79: + +```python + "INTERACTIVE HANDOFF (coda_interactive): When the user wants a human to " + "drive a coding agent in CoDA — not autonomous execution — call " + "coda_interactive instead of coda_run. The user's project must be a " + "Databricks Workspace Git Folder, and any in-progress changes must be " + "committed and pushed to the Git Folder's remote BEFORE the call. The tool " + "exports the committed HEAD state into a Coda-local directory, launches " + "the chosen agent (claude default; also hermes, codex, gemini, opencode), " + "and types the prompt as the first user input. The return shape includes " + "a viewer_url the user opens to attach — share it immediately in plain " + "text; it is the only handle to the session, and the user drives it until " + "they exit. Interactive sessions do NOT appear in coda_inbox, and " + "coda_get_result returns nothing for them — do not try to poll or fetch " + "results. Note that git history is NOT available inside the session " + "(files-only export); if the user needs history context, include a git " + "log summary in the prompt string." +``` + +Replace it with: + +```python + "INTERACTIVE HANDOFF (coda_interactive): When the user wants a human to " + "drive a coding agent in CoDA — not autonomous execution — call " + "coda_interactive instead of coda_run. The tool reads files from a " + "directory that already exists in the Databricks Workspace (a Git " + "Folder or a plain Workspace folder — either works). If your working " + "files are not yet in the Workspace, upload them first (workspace.import " + "via the Databricks SDK, REST, or CLI — any of these) into a folder " + "the user can read, then pass that folder as workspace_path. The tool " + "does NOT accept inline file payloads. If the directory is a Git " + "Folder, ensure the desired branch is checked out and pushed first — " + "the export is a server-side snapshot. The tool exports the directory " + "into a Coda-local working directory, launches the chosen agent " + "(claude default; also hermes, codex, gemini, opencode), and types " + "the prompt as the first user input. The return shape includes a " + "viewer_url the user opens to attach — share it immediately in plain " + "text; it is the only handle to the session, and the user drives it " + "until they exit. Interactive sessions do NOT appear in coda_inbox, " + "and coda_get_result returns nothing for them — do not try to poll " + "or fetch results. Note that git history is NOT available inside the " + "session (files-only export); if the user needs history context, " + "include a git log summary in the prompt string." +``` + +- [ ] **Step 3: Run the pinned-instructions test plus full suite** + +Run: `uv run pytest tests/test_coda_interactive.py -v` +Expected: All PASS (including the new instructions test). + +Run: `uv run pytest tests/ -v --no-header` +Expected: All previously-passing tests still pass. + +- [ ] **Step 4: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add coda_mcp/mcp_server.py tests/test_coda_interactive.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat: update INTERACTIVE HANDOFF instructions for broadened contract + +Tells upstream LLM callers that workspace_path can be either a Git Folder +or a plain Workspace folder. Drops the 'commit and push' admonition that +only applied to Git Folders." +``` + +--- + +## Task 4: Amend the original spec doc + +The broadening spec says it "Amends" the original (line 6 of the broadening spec), but the original spec doc on disk still describes the `branch` parameter, `repos.list`/`repos.update`, and Git-Folder-only requirements. Anyone reading the original later would implement the wrong API. Add an amendment notice to its header. + +**Files:** +- Modify: `docs/superpowers/specs/2026-05-28-coda-interactive-mcp-tool-design.md` (prepend amendment notice after the `**Related:**` line) + +- [ ] **Step 1: Read the current header of the original spec** + +Run: `head -10 docs/superpowers/specs/2026-05-28-coda-interactive-mcp-tool-design.md` + +You should see something like: + +``` +# Spec: `coda_interactive` MCP Tool + +**Status:** Draft, pre-critique-gate +**Date:** 2026-05-28 +**Branch:** `feat/coda-mcp-live-session-url` (same as Todo 1) +**Related:** `docs/superpowers/specs/2026-05-28-coda-run-replay-only-design.md` (Todo 1 — establishes the three-mode framework this spec slots into as Mode 2) + +## Goal +``` + +- [ ] **Step 2: Prepend the amendment notice** + +Use Edit to insert a new line after `**Related:**` (and before `## Goal`): + +The block to insert is: + +```markdown +> **Amended by:** [`docs/superpowers/specs/2026-05-28-coda-interactive-broaden-source-design.md`](2026-05-28-coda-interactive-broaden-source-design.md) — the `branch` parameter and the Git-Folder-only requirement have been removed. `coda_interactive` now accepts any Workspace directory (Git Folder or plain). The `repos.list` + `repos.update` flow described in Section 3 of this spec has been replaced by a single `workspace.get_status` directory check. The return shape no longer includes a `"branch"` key. +``` + +After the edit, the header should read: + +``` +**Branch:** `feat/coda-mcp-live-session-url` (same as Todo 1) +**Related:** `docs/superpowers/specs/2026-05-28-coda-run-replay-only-design.md` (Todo 1 — establishes the three-mode framework this spec slots into as Mode 2) + +> **Amended by:** [`docs/superpowers/specs/2026-05-28-coda-interactive-broaden-source-design.md`](2026-05-28-coda-interactive-broaden-source-design.md) — the `branch` parameter and the Git-Folder-only requirement have been removed. `coda_interactive` now accepts any Workspace directory (Git Folder or plain). The `repos.list` + `repos.update` flow described in Section 3 of this spec has been replaced by a single `workspace.get_status` directory check. The return shape no longer includes a `"branch"` key. + +## Goal +``` + +- [ ] **Step 3: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add docs/superpowers/specs/2026-05-28-coda-interactive-mcp-tool-design.md +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "docs: mark original coda_interactive spec as amended by broaden-source spec" +``` + +--- + +## Task 5: Push and update PR #67 description + +**Files:** +- None (remote/PR update) + +- [ ] **Step 1: Verify the branch's git state** + +```bash +git status +git log --oneline origin/feat/coda-mcp-interactive-handoff..HEAD +``` + +Expected: Clean working tree. Three new commits since the previous remote head (tests rewrite, impl, instructions string). + +- [ ] **Step 2: Push the branch** + +```bash +git push origin feat/coda-mcp-interactive-handoff +``` + +Expected: Successful fast-forward. + +- [ ] **Step 3: Update PR #67 description** + +Add a "Follow-up: broadened source" section at the bottom of the PR body via `gh pr edit` (or, if gh CLI's TLS bug hits, via curl + REST). Content: + +``` +## Follow-up: broadened source contract + +`coda_interactive` no longer requires a Databricks Workspace **Git Folder**. +Any Workspace directory (Git Folder or plain Workspace folder) is accepted. +The `branch` parameter has been removed — callers manage Git Folder branch +state themselves before calling. + +API change (no shipped consumers — safe): +- `coda_interactive(prompt, workspace_path, branch=..., agent=..., email=...)` → + `coda_interactive(prompt, workspace_path, agent=..., email=...)` +- Return shape: `"branch"` key dropped. + +Validation is now a `workspace.get_status` call with a directory-type check +(replaces the `repos.list` + exact-match filter). +``` + +Try the gh path first: + +```bash +gh pr edit 67 --body-file <(gh pr view 67 --json body -q .body; echo; echo; cat <<'EOF' +## Follow-up: broadened source contract + +`coda_interactive` no longer requires a Databricks Workspace **Git Folder**. +Any Workspace directory (Git Folder or plain Workspace folder) is accepted. +The `branch` parameter has been removed — callers manage Git Folder branch +state themselves before calling. + +API change (no shipped consumers — safe): +- `coda_interactive(prompt, workspace_path, branch=..., agent=..., email=...)` → + `coda_interactive(prompt, workspace_path, agent=..., email=...)` +- Return shape: `"branch"` key dropped. + +Validation is now a `workspace.get_status` call with a directory-type check +(replaces the `repos.list` + exact-match filter). +EOF +) +``` + +If gh fails with the known `x509: OSStatus -26276` issue on this machine, fall back to curl: + +```bash +TOKEN=$(gh auth token) +EXISTING_BODY=$(curl -s -k -H "Authorization: token $TOKEN" \ + https://api.github.com/repos/databrickslabs/coding-agents-databricks-apps/pulls/67 | jq -r .body) + +NEW_BODY="$EXISTING_BODY + +## Follow-up: broadened source contract + +\`coda_interactive\` no longer requires a Databricks Workspace **Git Folder**. +Any Workspace directory (Git Folder or plain Workspace folder) is accepted. +The \`branch\` parameter has been removed — callers manage Git Folder branch +state themselves before calling. + +API change (no shipped consumers — safe): +- \`coda_interactive(prompt, workspace_path, branch=..., agent=..., email=...)\` → + \`coda_interactive(prompt, workspace_path, agent=..., email=...)\` +- Return shape: \`\"branch\"\` key dropped. + +Validation is now a \`workspace.get_status\` call with a directory-type check +(replaces the \`repos.list\` + exact-match filter)." + +jq -n --arg body "$NEW_BODY" '{body: $body}' | curl -s -k -X PATCH \ + -H "Authorization: token $TOKEN" \ + -H "Content-Type: application/json" \ + -d @- \ + https://api.github.com/repos/databrickslabs/coding-agents-databricks-apps/pulls/67 +``` + +Confirm the PR description has the new section by visiting the PR URL or via `gh pr view 67`. + +--- + +## Self-review of this plan against the spec + +**Spec section 1 — Tool signature.** Task 2 Step 2 replaces the signature, dropping `branch`. Task 1 Step 5 adds a signature regression guard. ✓ + +**Spec section 2 — Body of `coda_interactive`.** Task 2 Step 2 contains the full new body. `repos.list`/`repos.update` removed, `workspace.get_status` + `_is_directory` added. ✓ + +**Spec section 3 — Return shape.** Task 2 Step 2 omits the `"branch"` key. The existing happy-path test does not assert on `"branch"`, so no test change needed; the regression is the signature test. ✓ + +**Spec section 4 — Caller pre-condition rewrite.** Task 3 rewrites the INTERACTIVE HANDOFF paragraph. Task 2 also rewrites the tool's docstring. Both surfaces updated. ✓ + +**Spec section 5 — INTERACTIVE HANDOFF string.** Task 3 covers it with a pinned-content test (Step 1) then the rewrite (Step 2). ✓ + +**Spec "Tests to update."** Task 1 covers every bullet: 3 drops, 4 updates, 2 adds. The pinned-instructions test in Task 3 is a fifth add. ✓ + +**Spec "Tests for the SDK validation step."** Task 1 Steps 3 and 4 cover the missing-path and not-a-directory cases. ✓ + +**Spec "Out of scope."** This plan does not add single-file workspace_path, branch-info surfacing in the response, or extra cleanup paths. ✓ + +**Spec "Acceptance criteria."** +- `coda_interactive` accepts any Workspace directory → Task 2. ✓ +- No `branch` parameter → Task 2 + signature guard test. ✓ +- Clean error for missing/non-directory paths → Task 2 + 2 new tests. ✓ +- Existing tests pass after updates → Task 1 + Task 2 Steps 3-4. ✓ +- PR description reflects simpler contract → Task 4 Step 3. ✓ + +**Placeholder scan:** No TBD/TODO. Every step has explicit code or a concrete command. ✓ + +**Type consistency:** `_is_directory(status)` accepts an object with `.object_type` attribute — matches what `workspace.get_status` returns and matches the mock helper in tests. The mock helper in Task 1 Step 2 (`_make_dir_status`) returns a MagicMock with `object_type = "DIRECTORY"`, which `_is_directory` accepts via its string-fallback branch (`str(ot) == "DIRECTORY"`). ✓ diff --git a/docs/superpowers/plans/2026-05-28-coda-interactive-mcp-tool.md b/docs/superpowers/plans/2026-05-28-coda-interactive-mcp-tool.md new file mode 100644 index 0000000..f5f663d --- /dev/null +++ b/docs/superpowers/plans/2026-05-28-coda-interactive-mcp-tool.md @@ -0,0 +1,1631 @@ +# `coda_interactive` MCP Tool Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add `coda_interactive` MCP tool that lets an upstream MCP client hand off a coding session to a human via a CoDA viewer URL — the human attaches to a live PTY with the chosen agent (claude default) already loaded with the user's Databricks Workspace Git Folder as CWD and the kickoff prompt typed. + +**Architecture:** Mode 2 in the three-mode framework (see `docs/superpowers/specs/2026-05-28-coda-run-replay-only-design.md`). The tool resolves a `workspace_path` to a Databricks Workspace Git Folder, optionally updates it to a specified branch, exports the file tree to `~/.coda/projects//`, creates a PTY with that dir as CWD, launches the agent, and auto-pastes the prompt. The PTY inherits Mode 1's existing 24h-idle lifecycle. Cleanup of the project dir is tied to PTY teardown. + +**Tech Stack:** Python 3.11 + FastMCP + Databricks SDK (`databricks-sdk` already in requirements) + Flask + uvicorn + pytest. No new dependencies. All work localized to `app.py`, `coda_mcp/`, and the test suite. + +--- + +## Pre-flight check (do before Task 1) + +- [ ] **P1: Verify baseline tests pass.** + +```bash +cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp +.venv/bin/python -m pytest tests/ --ignore=tests/e2e -q 2>&1 | tail -5 +``` + +Expected: `524 passed, 15 skipped` (or close to it — matches Todo 1's final state). + +- [ ] **P2: Confirm worktree is on the `feat/coda-mcp-live-session-url` branch.** + +```bash +cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp +git branch --show-current +``` + +Expected: `feat/coda-mcp-live-session-url` + +- [ ] **P3: Capture the baseline SHA for downstream code-quality reviews.** + +```bash +git rev-parse HEAD +``` + +Note the SHA — reviewer subagents need it as BASE_SHA. + +--- + +## Task 1: Prerequisite — refactor `mcp_create_pty_session` to use `_build_terminal_shell_env` + +Closes a pre-existing security gap. Today, `mcp_create_pty_session`'s inline env strip only removes 5 keys, while the HTTP `create_session` path uses `_build_terminal_shell_env` which also strips `NPM_TOKEN`, `UV_DEFAULT_INDEX`, `UV_INDEX_*_PASSWORD`, `UV_INDEX_*_USERNAME`, and `npm_config_//*` registry credential patterns. The refactor closes the gap for all MCP-created PTYs (current `coda_run` and future `coda_interactive`). + +**Important context:** The current session dict in `mcp_create_pty_session` (around `app.py:1488`) does **NOT** store the child shell's env. The test below would silently pass if it relied on `sessions[sid]["env"]` alone (a missing key returns `{}` from `.get()`). To get a TDD red-then-green cycle that means something, **Task 1 explicitly adds an `"env"` key to the session dict AND swaps the env-strip to use `_build_terminal_shell_env`** — both changes happen together so the test fails ONLY because of credential leaks, not because of a missing key. + +**Files:** +- Modify: `app.py` (function `mcp_create_pty_session` at line 1420, env-strip block at line 1435, session dict insert at line 1488) +- Create: `tests/test_mcp_env_strip.py` + +- [ ] **Step 1: Write the failing test.** + +Create `tests/test_mcp_env_strip.py`: + +```python +"""Tests for env-stripping consistency between MCP and HTTP PTY creation paths.""" +import os +import pytest + +try: + import pty as _pty + _master, _slave = _pty.openpty() + os.close(_master) + os.close(_slave) + _PTY_AVAILABLE = True +except Exception: + _PTY_AVAILABLE = False + +_pty_skip = pytest.mark.skipif( + not _PTY_AVAILABLE, + reason="PTY not allocatable in this environment", +) + + +@_pty_skip +def test_mcp_create_pty_session_strips_registry_credentials(monkeypatch): + """mcp_create_pty_session must strip NPM_TOKEN, UV_DEFAULT_INDEX, UV_INDEX_*_PASSWORD, + UV_INDEX_*_USERNAME, and npm_config_//* from the child shell's environment — + matching the HTTP create_session path. Today, these leak into MCP-created PTYs. + """ + from app import mcp_create_pty_session, mcp_close_pty_session, sessions + + # Plant registry-credential env vars before creating the PTY. + monkeypatch.setenv("NPM_TOKEN", "leak-me-npm") + monkeypatch.setenv("UV_DEFAULT_INDEX", "https://leaked-index.example/") + monkeypatch.setenv("UV_INDEX_MYREG_PASSWORD", "leak-me-uv-pw") + monkeypatch.setenv("UV_INDEX_MYREG_USERNAME", "leak-me-uv-user") + monkeypatch.setenv("npm_config_//registry.example/:_authToken", "leak-me-npm-cfg") + + sid = mcp_create_pty_session(label="t-env-strip") + try: + env = sessions[sid].get("env", {}) + assert "NPM_TOKEN" not in env, f"NPM_TOKEN leaked into MCP PTY: keys={list(env)}" + assert "UV_DEFAULT_INDEX" not in env, "UV_DEFAULT_INDEX leaked" + assert "UV_INDEX_MYREG_PASSWORD" not in env, "UV_INDEX_*_PASSWORD leaked" + assert "UV_INDEX_MYREG_USERNAME" not in env, "UV_INDEX_*_USERNAME leaked" + assert not any(k.startswith("npm_config_//") for k in env), "npm_config_// keys leaked" + finally: + mcp_close_pty_session(sid) +``` + +**Note on the test:** The test reads `sessions[sid]["env"]`. The session dict currently has NO `"env"` key, so without Step 3 changes the test would silently pass (`.get("env", {})` returns `{}` and all `not in {}` assertions trivially pass). Step 3 fixes BOTH (a) adds the `"env"` key, (b) swaps the env-strip to use `_build_terminal_shell_env`. Step 2 verifies failure ONLY after Step 3a (key added) — that gives a meaningful red, then Step 3b (strip refactor) gives the green. + +- [ ] **Step 2: Add the `"env"` key to the session dict (this alone makes the test runnable but failing).** + +In `app.py`, find the session dict literal inside `mcp_create_pty_session` (around line 1488 — the block that has `"master_fd"`, `"pid"`, `"output_buffer"`, etc.). Add a new key: + +```python +sessions[session_id] = { + ..., + "replay_only": replay_only, + "env": env_for_child, # NEW — exposed for env-strip test + ... +} +``` + +`env_for_child` is the variable name used in the env-construction block above. If it's named differently in the actual code, use the actual variable name. + +- [ ] **Step 3: Run the test and verify it fails for the RIGHT reason.** + +```bash +.venv/bin/python -m pytest tests/test_mcp_env_strip.py -v 2>&1 | tail -10 +``` + +Expected: FAIL — at least one of NPM_TOKEN/UV_*/npm_config_// keys is present in `sessions[sid]["env"]` (because the existing inline env-strip doesn't remove them). If the test PASSES at this point, the `"env"` key didn't get added — go back to Step 2. + +- [ ] **Step 4: Refactor `mcp_create_pty_session` env-stripping.** + +In `app.py`, find the env-construction block inside `mcp_create_pty_session` (around line 1435). It currently looks like: + +```python +env_for_child = os.environ.copy() +for k in ("CLAUDECODE", "CLAUDE_CODE_SESSION", "DATABRICKS_TOKEN", "DATABRICKS_HOST", "GEMINI_API_KEY"): + env_for_child.pop(k, None) +``` + +Replace with: + +```python +env_for_child = _build_terminal_shell_env(os.environ) +``` + +`_build_terminal_shell_env` is already defined in `app.py` (around line 210). It returns a dict with ALL the right strips applied (registry creds + the 5 keys above + others). + +- [ ] **Step 5: Run the test and verify pass.** + +```bash +.venv/bin/python -m pytest tests/test_mcp_env_strip.py -v 2>&1 | tail -10 +``` + +Expected: PASS — registry credentials are now stripped. + +- [ ] **Step 6: Run the full suite to confirm no regression.** + +```bash +.venv/bin/python -m pytest tests/ --ignore=tests/e2e -q 2>&1 | tail -5 +``` + +Expected: 525 passed, 15 skipped (one more pass than baseline). + +- [ ] **Step 7: Commit.** + +```bash +git add app.py tests/test_mcp_env_strip.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "fix: mcp_create_pty_session strips registry credentials like HTTP path does + +Pre-existing gap: the MCP PTY-creation path stripped only 5 env vars +while the HTTP create_session path used _build_terminal_shell_env which +also strips NPM_TOKEN, UV_DEFAULT_INDEX, UV_INDEX_*_PASSWORD, +UV_INDEX_*_USERNAME, and npm_config_// keys. This let deployer-level +registry credentials leak into the agent's child shell visible via env. +Refactor mcp_create_pty_session to use _build_terminal_shell_env." +``` + +--- + +## Task 2: Add `cwd` kwarg to `mcp_create_pty_session` + +`coda_interactive` needs the spawned bash to start in a specific directory (the exported project dir). Add an optional `cwd: str | None = None` kwarg; default `None` preserves current behavior. + +**Files:** +- Modify: `app.py` (`mcp_create_pty_session` signature and PTY spawn call) +- Modify: `tests/test_mcp_env_strip.py` (add new test in this same file for compactness) + +- [ ] **Step 1: Write the failing test.** + +Append to `tests/test_mcp_env_strip.py`: + +```python +@_pty_skip +def test_mcp_create_pty_session_respects_cwd_kwarg(tmp_path): + """When cwd is passed, the spawned bash starts in that directory.""" + from app import mcp_create_pty_session, mcp_close_pty_session, sessions + + # Create a sentinel file in tmp_path so we can detect the CWD via shell output. + sentinel = tmp_path / "SENTINEL_FILE" + sentinel.write_text("hello") + + sid = mcp_create_pty_session(label="t-cwd", cwd=str(tmp_path)) + try: + # The session dict should record the cwd. + assert sessions[sid].get("cwd") == str(tmp_path) + finally: + mcp_close_pty_session(sid) + + +@_pty_skip +def test_mcp_create_pty_session_cwd_defaults_to_none(): + """When cwd is not passed, sessions[sid]['cwd'] is None (preserves current behavior).""" + from app import mcp_create_pty_session, mcp_close_pty_session, sessions + + sid = mcp_create_pty_session(label="t-no-cwd") + try: + assert sessions[sid].get("cwd") is None + finally: + mcp_close_pty_session(sid) +``` + +- [ ] **Step 2: Run and verify failure.** + +```bash +.venv/bin/python -m pytest tests/test_mcp_env_strip.py::test_mcp_create_pty_session_respects_cwd_kwarg tests/test_mcp_env_strip.py::test_mcp_create_pty_session_cwd_defaults_to_none -v 2>&1 | tail -10 +``` + +Expected: FAIL — `TypeError: unexpected keyword argument 'cwd'` for the first test. + +- [ ] **Step 3: Add the `cwd` kwarg.** + +In `app.py`, change the `mcp_create_pty_session` signature to: + +```python +def mcp_create_pty_session( + label: str = "hermes-mcp", + transcript_path: str | None = None, + replay_only: bool = False, + cwd: str | None = None, +) -> str: +``` + +Inside the function, find the PTY spawn / `subprocess.Popen` call (it's the one that launches bash inside the PTY). It should currently look something like: + +```python +process = subprocess.Popen( + ["/bin/bash", "-l"], + stdin=slave_fd, stdout=slave_fd, stderr=slave_fd, + env=env_for_child, + preexec_fn=os.setsid, + close_fds=True, +) +``` + +Add `cwd=cwd` (which is None by default, meaning the child uses the parent's CWD — current behavior): + +```python +process = subprocess.Popen( + ["/bin/bash", "-l"], + stdin=slave_fd, stdout=slave_fd, stderr=slave_fd, + env=env_for_child, + cwd=cwd, # NEW + preexec_fn=os.setsid, + close_fds=True, +) +``` + +Also add `cwd` to the session dict: + +```python +sessions[session_id] = { + ..., + "cwd": cwd, # NEW + ... +} +``` + +- [ ] **Step 4: Run tests and verify pass.** + +```bash +.venv/bin/python -m pytest tests/test_mcp_env_strip.py -v 2>&1 | tail -10 +``` + +Expected: all tests in the file pass. + +- [ ] **Step 5: Run the full suite.** + +```bash +.venv/bin/python -m pytest tests/ --ignore=tests/e2e -q 2>&1 | tail -5 +``` + +Expected: 527 passed, 15 skipped. + +- [ ] **Step 6: Commit.** + +```bash +git add app.py tests/test_mcp_env_strip.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: mcp_create_pty_session accepts cwd kwarg + +Adds optional cwd parameter so callers can spawn the PTY's bash in a +specific directory. Default None preserves current behavior. Required +for coda_interactive (which needs to start agents in the exported +project dir)." +``` + +--- + +## Task 3: Create `coda_mcp/workspace_export.py` helper + +Encapsulates the Workspace-tree-to-local-dir export logic. Single responsibility: given a Databricks Workspace path and a local destination, copy the file tree. + +**Files:** +- Create: `coda_mcp/workspace_export.py` +- Create: `tests/test_workspace_export.py` + +- [ ] **Step 1: Write the failing tests.** + +Create `tests/test_workspace_export.py`: + +```python +"""Tests for coda_mcp.workspace_export.export_workspace_tree.""" +import os +from unittest.mock import MagicMock, patch + +import pytest + +from coda_mcp.workspace_export import export_workspace_tree + + +def _fake_object(path, object_type): + """Minimal stand-in for databricks.sdk.service.workspace.ObjectInfo.""" + o = MagicMock() + o.path = path + o.object_type = object_type + return o + + +def test_export_workspace_tree_creates_dest_dir(tmp_path): + """Helper creates the destination directory if it doesn't exist.""" + dest = tmp_path / "subdir" + assert not dest.exists() + + client = MagicMock() + client.workspace.list.return_value = [] + export_workspace_tree(client, "/Workspace/Users/x/empty", str(dest)) + + assert dest.exists() and dest.is_dir() + + +def test_export_workspace_tree_writes_single_file(tmp_path): + """A workspace with one file gets that file written to the local dir.""" + client = MagicMock() + client.workspace.list.return_value = [ + _fake_object("/Workspace/Users/x/proj/main.py", "FILE"), + ] + # Export returns an object with .content (base64-encoded bytes) + import base64 + mock_export = MagicMock() + mock_export.content = base64.b64encode(b"print('hi')\n").decode("ascii") + client.workspace.export.return_value = mock_export + + export_workspace_tree(client, "/Workspace/Users/x/proj", str(tmp_path)) + + main_py = tmp_path / "main.py" + assert main_py.exists() + assert main_py.read_text() == "print('hi')\n" + + +def test_export_workspace_tree_handles_nested_dirs(tmp_path): + """Nested directory structure is preserved in the destination.""" + client = MagicMock() + # First list call returns the top-level entries + # Subsequent recursive calls return the subdir contents + def list_side_effect(path, **kwargs): + if path == "/Workspace/Users/x/proj": + return [ + _fake_object("/Workspace/Users/x/proj/main.py", "FILE"), + _fake_object("/Workspace/Users/x/proj/lib", "DIRECTORY"), + ] + elif path == "/Workspace/Users/x/proj/lib": + return [ + _fake_object("/Workspace/Users/x/proj/lib/util.py", "FILE"), + ] + return [] + client.workspace.list.side_effect = list_side_effect + + import base64 + def export_side_effect(path, **kwargs): + mock = MagicMock() + if path.endswith("main.py"): + mock.content = base64.b64encode(b"main\n").decode("ascii") + else: + mock.content = base64.b64encode(b"util\n").decode("ascii") + return mock + client.workspace.export.side_effect = export_side_effect + + export_workspace_tree(client, "/Workspace/Users/x/proj", str(tmp_path)) + + assert (tmp_path / "main.py").read_text() == "main\n" + assert (tmp_path / "lib" / "util.py").read_text() == "util\n" + + +def test_export_workspace_tree_skips_binary_files_gracefully(tmp_path, caplog): + """Files that fail to export (e.g. binaries) are skipped and logged, not fatal.""" + client = MagicMock() + client.workspace.list.return_value = [ + _fake_object("/Workspace/Users/x/proj/text.py", "FILE"), + _fake_object("/Workspace/Users/x/proj/image.png", "FILE"), + ] + + import base64 + def export_side_effect(path, **kwargs): + if path.endswith(".png"): + raise Exception("400 Bad Request: cannot export binary as SOURCE") + mock = MagicMock() + mock.content = base64.b64encode(b"hello\n").decode("ascii") + return mock + client.workspace.export.side_effect = export_side_effect + + # Should NOT raise; should skip and log. + export_workspace_tree(client, "/Workspace/Users/x/proj", str(tmp_path)) + + assert (tmp_path / "text.py").exists() + assert not (tmp_path / "image.png").exists() + + +def test_export_workspace_tree_empty_workspace(tmp_path): + """Empty workspace path produces empty destination dir (no error).""" + client = MagicMock() + client.workspace.list.return_value = [] + + export_workspace_tree(client, "/Workspace/Users/x/empty", str(tmp_path)) + + assert tmp_path.exists() + assert list(tmp_path.iterdir()) == [] +``` + +- [ ] **Step 2: Run and verify failure.** + +```bash +.venv/bin/python -m pytest tests/test_workspace_export.py -v 2>&1 | tail -10 +``` + +Expected: ImportError (`No module named coda_mcp.workspace_export`). + +- [ ] **Step 3: Implement the helper.** + +Create `coda_mcp/workspace_export.py`: + +```python +"""Export a Databricks Workspace tree (Git Folder contents) to a local directory. + +Used by ``coda_interactive`` to materialize a Workspace Git Folder onto the +Coda container's disk before launching an agent in that directory. + +Only the working tree is exported — Git Folder server-side metadata (the +``.git/`` directory) is not exposed by the Workspace API. +""" +from __future__ import annotations + +import base64 +import logging +import os +from typing import Any + +logger = logging.getLogger(__name__) + + +def export_workspace_tree(client: Any, workspace_path: str, dest_dir: str) -> None: + """Export the Workspace tree rooted at ``workspace_path`` into ``dest_dir``. + + ``client`` is a ``databricks.sdk.WorkspaceClient`` (or compatible mock). + Recursively lists entries, calls ``workspace.export()`` per file with + ``ExportFormat.SOURCE``, decodes the base64 content, and writes to the + local mirror. + + Per-file export errors (e.g. binaries that fail SOURCE export) are logged + and skipped — they do not abort the export. The agent in the session may + not have access to those files; the human can decide whether that matters. + """ + os.makedirs(dest_dir, exist_ok=True) + + try: + from databricks.sdk.service.workspace import ExportFormat + export_format = ExportFormat.SOURCE + except Exception: + export_format = None # mocks won't care + + _export_recursive(client, workspace_path, dest_dir, export_format) + + +def _export_recursive(client, workspace_path: str, dest_dir: str, export_format) -> None: + """Walk one level of the workspace and export files / recurse into dirs.""" + try: + entries = list(client.workspace.list(workspace_path)) + except Exception as e: + logger.warning("workspace.list(%s) failed: %s", workspace_path, e) + return + + for entry in entries: + rel_name = os.path.basename(entry.path) + local_path = os.path.join(dest_dir, rel_name) + object_type = str(getattr(entry, "object_type", "")) + + if object_type == "DIRECTORY" or object_type.endswith(".DIRECTORY"): + _export_recursive(client, entry.path, local_path, export_format) + elif object_type == "FILE" or object_type.endswith(".FILE") or object_type == "NOTEBOOK" or object_type.endswith(".NOTEBOOK"): + try: + if export_format is not None: + exported = client.workspace.export(path=entry.path, format=export_format) + else: + exported = client.workspace.export(path=entry.path) + content_b64 = getattr(exported, "content", "") or "" + content_bytes = base64.b64decode(content_b64) if content_b64 else b"" + with open(local_path, "wb") as f: + f.write(content_bytes) + except Exception as e: + logger.warning("workspace.export(%s) failed; skipping: %s", entry.path, e) + continue + else: + # Unknown object type; skip with a log line. + logger.info("Skipping unknown object_type=%r at %s", object_type, entry.path) +``` + +- [ ] **Step 4: Run tests and verify pass.** + +```bash +.venv/bin/python -m pytest tests/test_workspace_export.py -v 2>&1 | tail -15 +``` + +Expected: 5 passed. + +- [ ] **Step 5: Commit.** + +```bash +git add coda_mcp/workspace_export.py tests/test_workspace_export.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: add coda_mcp.workspace_export.export_workspace_tree helper + +Recursively exports a Databricks Workspace Git Folder's file tree to +a local directory. Used by coda_interactive (next commit) to +materialize project files before launching an agent. + +Per-file export errors (binary files etc.) are logged and skipped +rather than aborting the export." +``` + +--- + +## Task 4: Extend `mcp_close_pty_session` to clean up the project dir + +When a `coda_interactive` PTY is torn down, the corresponding `~/.coda/projects//` directory should be removed. Same cleanup hook fires on graceful exit and idle reaper. + +**Files:** +- Modify: `app.py` (function `mcp_close_pty_session` — find its definition by grep) +- Modify: `tests/test_mcp_env_strip.py` (append cleanup-hook test for compactness; could also be a new file) + +- [ ] **Step 1: Write the failing test.** + +Append to `tests/test_mcp_env_strip.py`: + +```python +@_pty_skip +def test_mcp_close_pty_session_removes_project_dir(tmp_path, monkeypatch): + """When the PTY is closed, any project dir at ~/.coda/projects// is removed.""" + import os + from app import mcp_create_pty_session, mcp_close_pty_session + + # Point HOME at tmp_path so ~/.coda lives in a controllable place. + monkeypatch.setenv("HOME", str(tmp_path)) + + sid = mcp_create_pty_session(label="t-cleanup") + + project_dir = os.path.join(str(tmp_path), ".coda", "projects", sid) + os.makedirs(project_dir, exist_ok=True) + sentinel = os.path.join(project_dir, "SENTINEL") + with open(sentinel, "w") as f: + f.write("present-before-close") + assert os.path.exists(sentinel) + + mcp_close_pty_session(sid) + + assert not os.path.exists(project_dir), \ + f"Expected project dir to be removed after PTY close: {project_dir} still exists" + + +@_pty_skip +def test_mcp_close_pty_session_handles_missing_project_dir(monkeypatch, tmp_path): + """No project dir present → close still succeeds (no exception).""" + from app import mcp_create_pty_session, mcp_close_pty_session + + monkeypatch.setenv("HOME", str(tmp_path)) + + sid = mcp_create_pty_session(label="t-no-projdir") + # Do NOT create the project dir — verify close still works. + mcp_close_pty_session(sid) # must not raise +``` + +- [ ] **Step 2: Run and verify failure.** + +```bash +.venv/bin/python -m pytest tests/test_mcp_env_strip.py::test_mcp_close_pty_session_removes_project_dir -v 2>&1 | tail -10 +``` + +Expected: FAIL — the sentinel still exists after `mcp_close_pty_session(sid)`. + +- [ ] **Step 3: Add the cleanup hook.** + +In `app.py`, find `def mcp_close_pty_session(` (search for it). Inside the function, after the existing close logic (closing master_fd, killing process, popping from sessions), add the project-dir cleanup: + +```python +def mcp_close_pty_session(session_id: str) -> None: + # ... existing close logic ... + + # NEW: clean up the project dir if coda_interactive created one. + import shutil + project_dir = os.path.join( + os.path.expanduser("~/.coda/projects"), + session_id, + ) + if os.path.isdir(project_dir): + try: + shutil.rmtree(project_dir) + except OSError as e: + logger.warning("Failed to clean up project dir %s: %s", project_dir, e) +``` + +Place this near the END of the function so the PTY is fully closed before disk cleanup. The `try/except OSError` is intentional — a stuck file (rare) shouldn't break the close path. + +- [ ] **Step 4: Run tests and verify pass.** + +```bash +.venv/bin/python -m pytest tests/test_mcp_env_strip.py -v 2>&1 | tail -15 +``` + +Expected: all tests in the file pass. + +- [ ] **Step 5: Run the full suite.** + +```bash +.venv/bin/python -m pytest tests/ --ignore=tests/e2e -q 2>&1 | tail -5 +``` + +Expected: 529 passed, 15 skipped. + +- [ ] **Step 6: Commit.** + +```bash +git add app.py tests/test_mcp_env_strip.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: mcp_close_pty_session removes project dir on teardown + +When coda_interactive creates ~/.coda/projects//, that directory +should be deleted when the PTY is closed. Single cleanup path ties the +project's disk lifecycle to the PTY's lifecycle — no separate timer or +state to track." +``` + +--- + +## Task 5: Stub `coda_interactive` with agent validation + +First slice: register the tool, validate the agent kwarg, return error for unknown agents. No SDK calls, no PTY yet. + +**Files:** +- Modify: `coda_mcp/mcp_server.py` (add tool definition near `coda_run`) +- Create: `tests/test_coda_interactive.py` + +- [ ] **Step 1: Write failing tests.** + +Create `tests/test_coda_interactive.py`: + +```python +"""Tests for the coda_interactive MCP tool.""" +import asyncio +import json +import os + +import pytest + +ALLOWED_AGENTS = {"claude", "hermes", "codex", "gemini", "opencode"} + + +def test_coda_interactive_unknown_agent_returns_error(): + """An agent value not in the allow-list returns status=error and lists allowed values.""" + from coda_mcp import mcp_server + + result_str = asyncio.run(mcp_server.coda_interactive( + prompt="hello", + workspace_path="/Workspace/Users/x/proj", + agent="vim", + )) + result = json.loads(result_str) + assert result["status"] == "error" + assert "vim" in result["error"] + # Error message lists all allowed agents so the calling LLM can correct itself. + for allowed in ALLOWED_AGENTS: + assert allowed in result["error"] + + +def test_coda_interactive_default_agent_is_claude(): + """Calling with no agent kwarg defaults to claude (assertion via signature inspection).""" + import inspect + from coda_mcp import mcp_server + + sig = inspect.signature(mcp_server.coda_interactive) + assert sig.parameters["agent"].default == "claude" +``` + +- [ ] **Step 2: Run and verify failure.** + +```bash +.venv/bin/python -m pytest tests/test_coda_interactive.py -v 2>&1 | tail -10 +``` + +Expected: FAIL — `AttributeError: module 'coda_mcp.mcp_server' has no attribute 'coda_interactive'`. + +- [ ] **Step 3: Add the stub tool to `coda_mcp/mcp_server.py`.** + +In `coda_mcp/mcp_server.py`, locate the `@mcp.tool(...)` block for `coda_run` (around line 190 in the current file). The `coda_run` function ends around line 289 (before `coda_inbox`). Add the new tool definition between `coda_run` and `coda_inbox`: + +```python +_ALLOWED_AGENTS = {"claude", "hermes", "codex", "gemini", "opencode"} + + +@mcp.tool( + annotations=ToolAnnotations( + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + ), +) +async def coda_interactive( + prompt: str, + workspace_path: str, + branch: str = "", + agent: str = "claude", + email: str = "", +) -> str: + """Launch an interactive agent session in CoDA, handed off via a viewer URL. + + The MCP caller passes a Databricks Workspace Git Folder path; Coda exports + its file tree, launches the chosen agent (claude default) in that directory, + auto-types ``prompt`` as the first user input, and returns a ``viewer_url`` + the calling user opens in a browser to drive the session. + + Pre-condition: ``workspace_path`` must be a Databricks Workspace Git Folder + and any in-progress changes must have been committed and pushed to its + remote before this call. The export reflects the committed HEAD state. + + Interactive sessions do NOT appear in ``coda_inbox`` and ``coda_get_result`` + will not return anything for them. The viewer URL is the only handle. + + Allowed agents: claude (default), hermes, codex, gemini, opencode. + """ + if agent not in _ALLOWED_AGENTS: + return json.dumps({ + "status": "error", + "error": f"Unknown agent: {agent!r}. Allowed: {sorted(_ALLOWED_AGENTS)}", + }) + + # TODO(Task 6+): workspace lookup, branch update, export, PTY launch. + return json.dumps({ + "status": "error", + "error": "Not yet implemented (stub).", + }) +``` + +Notes: +- `json` is already imported at top of file. If not, add `import json`. +- The `# TODO` comment is acceptable here because the function is being built incrementally across Tasks 5–8; each task removes one TODO. + +- [ ] **Step 4: Run tests and verify pass.** + +```bash +.venv/bin/python -m pytest tests/test_coda_interactive.py -v 2>&1 | tail -10 +``` + +Expected: 2 passed. + +- [ ] **Step 5: Commit.** + +```bash +git add coda_mcp/mcp_server.py tests/test_coda_interactive.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: stub coda_interactive MCP tool with agent validation + +First slice. Validates the agent kwarg against the allow-list +(claude, hermes, codex, gemini, opencode); returns a clear error +listing the allowed values when an unknown agent is passed. +Workspace lookup, branch update, export, and PTY launch come in +follow-up commits." +``` + +--- + +## Task 6: Add workspace lookup + branch update to `coda_interactive` + +Resolve `workspace_path` to a Git Folder via `WorkspaceClient.repos.list()`; if `branch` is non-empty, call `repos.update(repo_id, branch=branch)`. + +**Files:** +- Modify: `coda_mcp/mcp_server.py` (`coda_interactive` body) +- Modify: `tests/test_coda_interactive.py` + +- [ ] **Step 1: Write failing tests.** + +Append to `tests/test_coda_interactive.py`: + +```python +def test_coda_interactive_workspace_path_not_found(monkeypatch): + """If repos.list() returns no match for workspace_path, status=error.""" + from unittest.mock import MagicMock + from coda_mcp import mcp_server + + fake_client = MagicMock() + fake_client.repos.list.return_value = [] # no Git Folder at that path + + monkeypatch.setattr(mcp_server, "WorkspaceClient", lambda: fake_client) + + result_str = asyncio.run(mcp_server.coda_interactive( + prompt="hello", + workspace_path="/Workspace/Users/x/nonexistent", + )) + result = json.loads(result_str) + assert result["status"] == "error" + assert "No Git Folder found" in result["error"] + + +def test_coda_interactive_branch_update_failure(monkeypatch): + """If repos.update() raises, return error and don't proceed to PTY.""" + from unittest.mock import MagicMock + from coda_mcp import mcp_server + + fake_repo = MagicMock() + fake_repo.id = 123 + fake_repo.path = "/Workspace/Users/x/proj" + + fake_client = MagicMock() + fake_client.repos.list.return_value = [fake_repo] + fake_client.repos.update.side_effect = Exception("404 branch not found: nonexistent") + + monkeypatch.setattr(mcp_server, "WorkspaceClient", lambda: fake_client) + + result_str = asyncio.run(mcp_server.coda_interactive( + prompt="hello", + workspace_path="/Workspace/Users/x/proj", + branch="nonexistent", + )) + result = json.loads(result_str) + assert result["status"] == "error" + assert "branch" in result["error"].lower() or "404" in result["error"] + + +def test_coda_interactive_skips_branch_update_when_empty(monkeypatch): + """If branch is empty, repos.update() must NOT be called.""" + from unittest.mock import MagicMock + from coda_mcp import mcp_server + + fake_repo = MagicMock() + fake_repo.id = 123 + fake_repo.path = "/Workspace/Users/x/proj" + + fake_client = MagicMock() + fake_client.repos.list.return_value = [fake_repo] + + monkeypatch.setattr(mcp_server, "WorkspaceClient", lambda: fake_client) + + # We don't expect a successful return yet (export+PTY not wired); we just + # verify that repos.update was not called. + asyncio.run(mcp_server.coda_interactive( + prompt="hello", + workspace_path="/Workspace/Users/x/proj", + branch="", + )) + fake_client.repos.update.assert_not_called() +``` + +- [ ] **Step 2: Run and verify failure.** + +```bash +.venv/bin/python -m pytest tests/test_coda_interactive.py -v 2>&1 | tail -10 +``` + +Expected: 3 new tests fail (function returns the stub error, not the lookup-based errors expected). + +- [ ] **Step 3: Implement workspace lookup + branch update.** + +In `coda_mcp/mcp_server.py`, near the top of the file (with other imports), add: + +```python +try: + from databricks.sdk import WorkspaceClient +except ImportError: + WorkspaceClient = None # type: ignore +``` + +(This guards against tests that mock the SDK by monkey-patching `mcp_server.WorkspaceClient`.) + +Replace the body of `coda_interactive` (the part after the agent-validation `if` block, currently just the `# TODO` and stub return) with: + +```python + # Resolve the Git Folder by listing under the workspace_path prefix. + if WorkspaceClient is None: + return json.dumps({ + "status": "error", + "error": "databricks-sdk not installed", + }) + + client = WorkspaceClient() + + try: + repos = list(client.repos.list(path_prefix=workspace_path)) + except Exception as e: + return json.dumps({ + "status": "error", + "error": f"Failed to list Git Folders: {e}", + }) + + repo = next((r for r in repos if r.path == workspace_path), None) + if repo is None: + return json.dumps({ + "status": "error", + "error": f"No Git Folder found at {workspace_path}", + }) + + # Optional branch update. + if branch: + try: + client.repos.update(repo_id=repo.id, branch=branch) + except Exception as e: + return json.dumps({ + "status": "error", + "error": f"Failed to update Git Folder to branch {branch!r}: {e}", + }) + + # TODO(Task 7+): export tree, create PTY, launch agent. + return json.dumps({ + "status": "error", + "error": "Not yet implemented (stub).", + }) +``` + +- [ ] **Step 4: Run tests and verify pass.** + +```bash +.venv/bin/python -m pytest tests/test_coda_interactive.py -v 2>&1 | tail -15 +``` + +Expected: 5 passed (2 from Task 5 + 3 new). + +- [ ] **Step 5: Commit.** + +```bash +git add coda_mcp/mcp_server.py tests/test_coda_interactive.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: coda_interactive resolves Git Folder and optionally updates branch + +Uses WorkspaceClient.repos.list to resolve workspace_path to a Git +Folder; returns a clear error if no match. If branch is non-empty, +calls repos.update which performs the actual git fetch+checkout +server-side. Export and PTY launch land in follow-up commits." +``` + +--- + +## Task 7: Implement `coda_interactive`'s full happy path + +Combined task: export workspace tree, create PTY, cd into project dir, launch agent, seed prompt, return viewer URL. **Single task with a single commit** — avoids the intermediate orphaned-state problem of the previous Task 7→Task 8 split (where the project dir's name didn't match the PTY's session id). + +**Ordering insight:** PTY is created FIRST (so we know its session_id), THEN we build `project_dir = ~/.coda/projects//`, THEN export into it, THEN `cd` the PTY into the dir via input, THEN launch the agent, THEN paste the prompt. This single chronology eliminates the chicken-and-egg between project_dir naming and PTY id. + +**Files:** +- Modify: `coda_mcp/mcp_server.py` (`coda_interactive` body — replace the stub return from Task 6 with the full happy path; also add module-level imports and constants) +- Modify: `tests/test_coda_interactive.py` (append happy-path test + export-failure test + agent-matrix test) + +- [ ] **Step 1: Write failing tests.** + +Append to `tests/test_coda_interactive.py`: + +```python +def test_coda_interactive_export_failure_cleans_partial_dir(monkeypatch, tmp_path): + """If export raises mid-way, the partial project dir is removed and the PTY is closed.""" + from unittest.mock import MagicMock + from coda_mcp import mcp_server + + monkeypatch.setenv("HOME", str(tmp_path)) + + fake_repo = MagicMock() + fake_repo.id = 123 + fake_repo.path = "/Workspace/Users/x/proj" + fake_client = MagicMock() + fake_client.repos.list.return_value = [fake_repo] + monkeypatch.setattr(mcp_server, "WorkspaceClient", lambda: fake_client) + + # PTY-creation hook returns a deterministic id we can predict. + monkeypatch.setattr( + mcp_server, "_app_create_session", lambda **kw: "pty-exportfail-id", + ) + + closed = [] + monkeypatch.setattr( + mcp_server, "_app_close_session", lambda sid: closed.append(sid), + ) + + def fake_export(client, workspace_path, dest_dir): + # Create the dir + a partial file, then raise. + os.makedirs(dest_dir, exist_ok=True) + with open(os.path.join(dest_dir, "partial.txt"), "w") as f: + f.write("partial") + raise RuntimeError("simulated export failure") + + monkeypatch.setattr(mcp_server, "export_workspace_tree", fake_export) + + # send_input hook should NOT be called for export-failure path (we close before launch). + sent = [] + monkeypatch.setattr( + mcp_server, "_app_send_input", lambda sid, payload: sent.append((sid, payload)), + ) + + result_str = asyncio.run(mcp_server.coda_interactive( + prompt="hello", + workspace_path="/Workspace/Users/x/proj", + )) + result = json.loads(result_str) + + assert result["status"] == "error" + assert "export" in result["error"].lower() + # PTY was created — must be closed on failure. + assert "pty-exportfail-id" in closed, "PTY must be closed when export fails" + # Project dir cleaned up. + project_dir = tmp_path / ".coda" / "projects" / "pty-exportfail-id" + assert not project_dir.exists(), "Partial project dir must be removed after export failure" + + +def test_coda_interactive_happy_path_sends_agent_command_and_prompt(monkeypatch, tmp_path): + """End-to-end mock: export succeeds, PTY created, cd + agent + prompt sent in order.""" + from unittest.mock import MagicMock + from coda_mcp import mcp_server + + monkeypatch.setenv("HOME", str(tmp_path)) + + fake_repo = MagicMock() + fake_repo.id = 123 + fake_repo.path = "/Workspace/Users/x/proj" + fake_client = MagicMock() + fake_client.repos.list.return_value = [fake_repo] + monkeypatch.setattr(mcp_server, "WorkspaceClient", lambda: fake_client) + + monkeypatch.setattr( + mcp_server, + "export_workspace_tree", + lambda client, ws_path, dest_dir: os.makedirs(dest_dir, exist_ok=True), + ) + monkeypatch.setattr( + mcp_server, "_app_create_session", lambda **kw: "pty-happy-id", + ) + + sent_to_pty = [] + monkeypatch.setattr( + mcp_server, + "_app_send_input", + lambda sid, payload: sent_to_pty.append((sid, payload)), + ) + + # Stub the sleep so the test runs fast. + monkeypatch.setattr(mcp_server, "_PROMPT_SEED_DELAY_S", 0) + + monkeypatch.setattr( + mcp_server.url_builder, + "build_viewer_url", + lambda pty_id: f"https://test.example/?session={pty_id}", + ) + + result_str = asyncio.run(mcp_server.coda_interactive( + prompt="continue debugging the auth flow", + workspace_path="/Workspace/Users/x/proj", + agent="claude", + )) + result = json.loads(result_str) + + assert result["status"] == "launched" + assert result["agent"] == "claude" + assert result["viewer_url"] == "https://test.example/?session=pty-happy-id" + assert result["project_dir"].endswith("/pty-happy-id") + + # Three PTY writes, in order: cd, agent command, prompt. + assert len(sent_to_pty) == 3, f"Expected 3 PTY writes; got {sent_to_pty}" + assert sent_to_pty[0][0] == "pty-happy-id" + assert sent_to_pty[0][1].startswith("cd "), \ + f"First write should be cd; got {sent_to_pty[0][1]!r}" + assert sent_to_pty[1] == ("pty-happy-id", "claude\n") + assert sent_to_pty[2] == ("pty-happy-id", "continue debugging the auth flow\n") + + +def test_coda_interactive_agent_command_matrix(monkeypatch, tmp_path): + """Each allowed agent maps to its expected launch command.""" + from unittest.mock import MagicMock + from coda_mcp import mcp_server + + expected = { + "claude": "claude\n", + "hermes": "hermes chat\n", + "codex": "codex\n", + "gemini": "gemini\n", + "opencode": "opencode\n", + } + + for agent, expected_cmd in expected.items(): + monkeypatch.setenv("HOME", str(tmp_path / agent)) + + fake_repo = MagicMock(); fake_repo.id = 1; fake_repo.path = "/W/x/p" + fake_client = MagicMock() + fake_client.repos.list.return_value = [fake_repo] + monkeypatch.setattr(mcp_server, "WorkspaceClient", lambda: fake_client) + monkeypatch.setattr( + mcp_server, "export_workspace_tree", + lambda client, ws_path, dest_dir: os.makedirs(dest_dir, exist_ok=True), + ) + monkeypatch.setattr( + mcp_server, "_app_create_session", lambda **kw: f"pty-{agent}", + ) + sent = [] + monkeypatch.setattr( + mcp_server, "_app_send_input", lambda sid, p: sent.append(p), + ) + monkeypatch.setattr(mcp_server, "_PROMPT_SEED_DELAY_S", 0) + monkeypatch.setattr( + mcp_server.url_builder, "build_viewer_url", + lambda pty_id: f"https://test/?s={pty_id}", + ) + + result_str = asyncio.run(mcp_server.coda_interactive( + prompt="x", workspace_path="/W/x/p", agent=agent, + )) + result = json.loads(result_str) + assert result["status"] == "launched", f"agent {agent}: {result}" + + # sent[0] is cd, sent[1] is the agent command, sent[2] is the prompt. + assert sent[1] == expected_cmd, \ + f"agent {agent}: expected {expected_cmd!r}, got {sent[1]!r}" +``` + +- [ ] **Step 2: Run and verify failure.** + +```bash +.venv/bin/python -m pytest tests/test_coda_interactive.py -v 2>&1 | tail -15 +``` + +Expected: 3 new tests fail (stub returns "Not yet implemented", happy-path assertions trip). + +- [ ] **Step 3: Implement the full happy path.** + +In `coda_mcp/mcp_server.py`: + +(a) Near the existing imports at the top of the file, add: + +```python +import shlex +import time +from coda_mcp import url_builder +from coda_mcp.workspace_export import export_workspace_tree +``` + +(b) Near other module-level constants, add: + +```python +_PROMPT_SEED_DELAY_S = 2 # seconds to wait for agent to initialize before pasting prompt + +_AGENT_LAUNCH_CMDS = { + "claude": "claude", + "hermes": "hermes chat", + "codex": "codex", + "gemini": "gemini", + "opencode": "opencode", +} +``` + +(c) Replace the trailing stub `return json.dumps({"status": "error", "error": "Not yet implemented (stub)."})` in `coda_interactive` (the one added by Task 6 after the branch-update block) with the full implementation: + +```python + # Create PTY FIRST so we have its session_id for the project_dir name. + if _app_create_session is None: + return json.dumps({ + "status": "error", + "error": "PTY hook not wired", + }) + + pty_session_id = None + project_dir = None + try: + pty_session_id = _app_create_session( + label=f"{agent}-interactive", + replay_only=False, + ) + + # Build the project dir at the canonical path keyed by PTY id. + project_dir = os.path.join( + os.path.expanduser("~/.coda/projects"), + pty_session_id, + ) + + # Export the Workspace tree into project_dir. + try: + export_workspace_tree(client, workspace_path, project_dir) + except Exception as e: + # Close the PTY and clean up the partial dir. + if _app_close_session is not None: + try: + _app_close_session(pty_session_id) + except Exception: + pass + import shutil + if os.path.isdir(project_dir): + shutil.rmtree(project_dir, ignore_errors=True) + return json.dumps({ + "status": "error", + "error": f"Failed to export workspace tree: {e}", + }) + + # cd into the project dir. + if _app_send_input is None: + return json.dumps({ + "status": "error", + "error": "PTY send hook not wired", + }) + _app_send_input(pty_session_id, f"cd {shlex.quote(project_dir)}\n") + + # Launch the agent. + launch_cmd = _AGENT_LAUNCH_CMDS[agent] + _app_send_input(pty_session_id, launch_cmd + "\n") + + # Wait briefly for agent initialization, then paste the prompt. + time.sleep(_PROMPT_SEED_DELAY_S) + _app_send_input(pty_session_id, prompt + "\n") + + viewer_url = url_builder.build_viewer_url(pty_session_id) + + return json.dumps({ + "status": "launched", + "viewer_url": viewer_url, + "agent": agent, + "project_dir": project_dir, + "workspace_path": workspace_path, + "branch": branch, + "instructions": ( + "Open viewer_url to attach. The agent is loaded with the " + "project files exported from Workspace and your kickoff " + "prompt typed. Type the agent's quit command (e.g. /quit) " + "and then `exit` to end the session. Note: git history is " + "NOT available in the session — files are an export, not " + "a clone." + ), + }) + except Exception as e: + # Catch-all: ensure no resource leak. + if pty_session_id and _app_close_session is not None: + try: + _app_close_session(pty_session_id) + except Exception: + pass + if project_dir and os.path.isdir(project_dir): + import shutil + shutil.rmtree(project_dir, ignore_errors=True) + return json.dumps({ + "status": "error", + "error": f"coda_interactive failed: {e}", + }) +``` + +Delete the now-unused `# TODO(Task 7+)` comments from Task 6's stub if they remain. + +- [ ] **Step 4: Run tests and verify pass.** + +```bash +.venv/bin/python -m pytest tests/test_coda_interactive.py -v 2>&1 | tail -15 +``` + +Expected: 8 passed (2 from Task 5 + 3 from Task 6 + 3 from Task 7). If any earlier test breaks because they didn't anticipate `_app_send_input` being called (the export-failure test from Task 6 patches `_app_create_session` but not `_app_send_input`), patch it accordingly with `monkeypatch.setattr(mcp_server, "_app_send_input", lambda *a, **k: None)`. + +- [ ] **Step 5: Run the full suite.** + +```bash +.venv/bin/python -m pytest tests/ --ignore=tests/e2e -q 2>&1 | tail -5 +``` + +Expected: 537+ passed, 15 skipped. + +- [ ] **Step 6: Commit.** + +```bash +git add coda_mcp/mcp_server.py tests/test_coda_interactive.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: coda_interactive end-to-end happy path + +Combined task: creates the PTY first (to get its id), builds the project +dir at ~/.coda/projects//, exports the Workspace tree into it, +cds the PTY into the dir, launches the chosen agent, waits 2s for +initialization, then pastes the prompt as the first user input. +Returns the viewer URL. + +Agent matrix (claude/hermes/codex/gemini/opencode) maps to each +agent's known interactive launch command. Export failure cleanly +closes the PTY and removes the partial project dir." +``` + +**Acknowledgment**: Task 2's `cwd` kwarg on `mcp_create_pty_session` ends up unused by this implementation (we `cd` via PTY input instead because the project_dir doesn't exist when the PTY is spawned). Leaving the tested optional kwarg in place is acceptable; reverting is more churn for no behavioral gain. + +--- + +## Task 8: Register `coda_interactive` in Flask fallback dispatch + +`coda_mcp/mcp_endpoint.py` has a Flask-based MCP fallback used in non-ASGI environments. It needs `coda_interactive` in its dispatch table. + +**Files:** +- Modify: `coda_mcp/mcp_endpoint.py` (imports + `_TOOL_DISPATCH`) + +- [ ] **Step 1: Read the existing dispatch.** + +```bash +grep -n "_TOOL_DISPATCH\|coda_run\|coda_inbox\|coda_get_result" coda_mcp/mcp_endpoint.py +``` + +Confirm the dispatch is a dict keyed by tool name → function reference. + +- [ ] **Step 2: Add the import + dispatch entry.** + +In `coda_mcp/mcp_endpoint.py`, find the import block that pulls in the existing tools (around line 22): + +```python +from coda_mcp.mcp_server import ( + mcp as mcp_instance, + coda_run, + coda_inbox, + coda_get_result, +) +``` + +Add `coda_interactive`: + +```python +from coda_mcp.mcp_server import ( + mcp as mcp_instance, + coda_run, + coda_inbox, + coda_get_result, + coda_interactive, +) +``` + +Find `_TOOL_DISPATCH` (around line 31): + +```python +_TOOL_DISPATCH = { + "coda_run": coda_run, + "coda_inbox": coda_inbox, + "coda_get_result": coda_get_result, +} +``` + +Add `coda_interactive`: + +```python +_TOOL_DISPATCH = { + "coda_run": coda_run, + "coda_inbox": coda_inbox, + "coda_get_result": coda_get_result, + "coda_interactive": coda_interactive, +} +``` + +- [ ] **Step 3: Run the test suite.** + +```bash +.venv/bin/python -m pytest tests/ --ignore=tests/e2e -q 2>&1 | tail -5 +``` + +Expected: all pass. + +- [ ] **Step 4: Spot-check the Flask fallback path with a quick test.** + +```bash +.venv/bin/python -c "from coda_mcp.mcp_endpoint import _TOOL_DISPATCH; print(list(_TOOL_DISPATCH))" +``` + +Expected output: `['coda_run', 'coda_inbox', 'coda_get_result', 'coda_interactive']` + +- [ ] **Step 5: Commit.** + +```bash +git add coda_mcp/mcp_endpoint.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: wire coda_interactive into Flask-fallback MCP dispatch + +The Flask blueprint at coda_mcp/mcp_endpoint.py is the WSGI-compatible +fallback used by tests and local dev. Without this entry, those paths +can't call coda_interactive." +``` + +--- + +## Task 9: Update FastMCP `instructions` string + +The instructions block at `coda_mcp/mcp_server.py:43-70` currently describes only `coda_run` (after Todo 1's update). Add a paragraph for `coda_interactive` so MCP-client LLMs understand the new tool's contract. + +**Files:** +- Modify: `coda_mcp/mcp_server.py` (the `instructions` string passed to `FastMCP(...)`) + +- [ ] **Step 1: Read the current instructions block.** + +```bash +grep -n "SHARE THE REPLAY URL\|FIRE AND FORGET\|WORKFLOW" coda_mcp/mcp_server.py | head -10 +``` + +Open the file and locate the `FastMCP(name=..., instructions="""...""")` block. + +- [ ] **Step 2: Add the new paragraph.** + +After the existing `SHARE THE REPLAY URL` paragraph and before the `WORKFLOW` paragraph, insert: + +``` +INTERACTIVE HANDOFF (coda_interactive): When the user wants a human to drive +a coding agent in CoDA — not autonomous execution — call coda_interactive +instead of coda_run. The user must have their project as a Databricks +Workspace Git Folder, and any in-progress changes must be committed and +pushed to the Git Folder's remote BEFORE the call. The tool exports the +committed HEAD state into a Coda-local directory, launches the chosen agent +(claude default; also hermes, codex, gemini, opencode), and types the prompt +as the first user input. Return shape includes a viewer_url the user opens +to attach — they then drive the session until they exit. Interactive sessions +do NOT appear in coda_inbox; coda_get_result returns nothing for them. The +viewer URL is the only handle — pass it to the user immediately. Note that +git history is NOT available inside the session (files-only export); if the +user needs history context, include a git log summary in the prompt string. +``` + +The exact wording can be tightened to match the existing paragraphs' tone — read the surrounding text first. + +- [ ] **Step 3: Run the suite.** + +```bash +.venv/bin/python -m pytest tests/ --ignore=tests/e2e -q 2>&1 | tail -5 +``` + +Expected: all pass (no tests assert on instruction text strings). + +- [ ] **Step 4: Commit.** + +```bash +git add coda_mcp/mcp_server.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "docs: add INTERACTIVE HANDOFF paragraph to MCP instructions + +Describes coda_interactive's contract for calling LLMs: Git Folder +pre-condition, viewer URL handoff, no coda_inbox / coda_get_result +integration, git history unavailable trade-off. Prevents calling LLMs +from treating coda_interactive like coda_run (e.g., trying to poll +results)." +``` + +--- + +## Task 10: Add regression guard test + +Defends the mode separation: calling `coda_run` must NOT create anything under `~/.coda/projects/`. Protects against future drift that accidentally couples the two modes. + +**Files:** +- Modify: `tests/test_replay_only_flag.py` (append to keep regression guards together) + +- [ ] **Step 1: Append the test.** + +Append to `tests/test_replay_only_flag.py`: + +```python +@_pty_skip +def test_coda_run_does_not_create_project_dir(tmp_path, monkeypatch): + """Regression guard: coda_run is Mode 3 (replay-only, no project dir). + Only coda_interactive (Mode 2) creates dirs under ~/.coda/projects/. + + If a future change accidentally calls export_workspace_tree from + coda_run or otherwise creates a per-session project dir, this test fires. + """ + import asyncio + import json + from app import sessions, mcp_close_pty_session + from coda_mcp import mcp_server, task_manager + + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path / "sessions")) + # Stop the watcher from racing the test. + monkeypatch.setattr(mcp_server, "_watch_task", lambda *a, **kw: None) + + result_str = asyncio.run(mcp_server.coda_run( + prompt="ignored", email="t@example.com", + )) + result = json.loads(result_str) + pty_id = None + try: + sess = task_manager._read_session(result["session_id"]) + pty_id = sess.get("pty_session_id") + + # Project dir must NOT exist for coda_run. + projects_root = os.path.join(str(tmp_path), ".coda", "projects") + assert not os.path.isdir(projects_root) or not os.listdir(projects_root), ( + f"coda_run unexpectedly created project dirs under {projects_root}: " + f"{os.listdir(projects_root) if os.path.isdir(projects_root) else 'n/a'}" + ) + finally: + if pty_id is not None: + mcp_close_pty_session(pty_id) +``` + +- [ ] **Step 2: Run.** + +```bash +.venv/bin/python -m pytest tests/test_replay_only_flag.py -v 2>&1 | tail -10 +``` + +Expected: all pass (this test specifically asserts coda_run's NEGATIVE behavior). + +- [ ] **Step 3: Run the full suite.** + +```bash +.venv/bin/python -m pytest tests/ --ignore=tests/e2e -q 2>&1 | tail -5 +``` + +Expected: ~540 passed, 15 skipped (depending on PTY availability — some Task 7 tests skip on this Mac). + +- [ ] **Step 4: Commit.** + +```bash +git add tests/test_replay_only_flag.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "test: regression guard against coda_run creating project dirs + +Mode separation is the spine of the three-mode framework: coda_run is +replay-only (no project_dir, no workspace export), coda_interactive +is the only path that creates ~/.coda/projects/. If a future refactor +accidentally couples them, this test fails loudly." +``` + +--- + +## Final verification (post-task) + +- [ ] **F1: Full suite green.** + +```bash +.venv/bin/python -m pytest tests/ --ignore=tests/e2e -q 2>&1 | tail -5 +``` + +Expected: all pass. + +- [ ] **F2: No grace/dead references re-introduced.** + +```bash +grep -rn "grace\|GRACE_PERIOD\|_mark_grace\|_bump_session_last_poll\|_schedule_deferred_close" coda_mcp/ app.py | grep -v "graceful\|GRACEFUL_" +``` + +Expected: no matches. + +- [ ] **F3: Mode separation still holds.** + +```bash +grep -n "_TOOL_DISPATCH" coda_mcp/mcp_endpoint.py +.venv/bin/python -c "from coda_mcp.mcp_endpoint import _TOOL_DISPATCH; print(sorted(_TOOL_DISPATCH))" +``` + +Expected: `['coda_get_result', 'coda_inbox', 'coda_interactive', 'coda_run']`. + +- [ ] **F4: Manual smoke (optional, requires deployed environment + a real Workspace Git Folder).** + +1. Restart the app: `uvicorn coda_mcp.mcp_asgi:app`. +2. From an MCP client, call `coda_interactive(prompt="explain this repo", workspace_path="/Workspace/Users/you@db.com/your-git-folder")`. +3. Open the returned `viewer_url`. Confirm: live attach lands you in a session with `claude` running, prompt visible in the chat, CWD is the project dir. +4. Type `/quit` then `exit`. Reattach to the URL — confirm replay or expired-session page. +5. SSH into the container (or check `/health`) — confirm `~/.coda/projects//` is gone. + +--- + +## Self-review checklist (run on completed plan) + +1. **Spec coverage** ✓ + - §1 Tool signature → Task 5 (stub + signature), Task 6 (workspace lookup/branch), Task 7 (full happy path: export+PTY+launch+prompt+viewer_url) + - §1a Caller pre-condition → Task 9 (MCP instructions string) + - §2 Agent launch matrix → Task 7 (`_AGENT_LAUNCH_CMDS`) + - §3 Project source export → Task 3 (`workspace_export.py`) + Task 7 wiring + - §4 Prompt seeding → Task 7 (`_PROMPT_SEED_DELAY_S` + send_input ordering) + - §5 PTY lifecycle → Task 4 (cleanup hook) + - §6 Where this lives + env-strip prereq → Task 1 (env-strip), Task 2 (cwd kwarg), Task 8 (Flask dispatch), Task 9 (instructions) + - Regression guard → Task 10 + +2. **Placeholders** ✓ — every step has concrete code/commands. The `# TODO(Task N+)` markers inside intermediate `coda_interactive` versions are explicit hand-offs between tasks, not deferred work. + +3. **Type consistency** ✓ + - `_ALLOWED_AGENTS: set[str]` — used identically in Tasks 5 and 7 + - `_AGENT_LAUNCH_CMDS: dict[str, str]` — defined in Task 7 + - `_PROMPT_SEED_DELAY_S: int` — defined in Task 7 + - `pty_session_id: str` — comes from `_app_create_session(...)`'s return; project_dir built from it + - `workspace_path: str`, `branch: str = ""`, `agent: str = "claude"` consistent across signature, tests, and instructions + +4. **Ordering safety** ✓ + - Prereq env-strip (Task 1) runs first — no Todo-2-specific dependency, just security cleanup + - `cwd` kwarg (Task 2) added before any caller uses it (Task 7, though ultimately unused — see Task 7 acknowledgment) + - `workspace_export.py` (Task 3) created before `coda_interactive` imports it (Task 7) + - Cleanup hook (Task 4) added before any project dir gets created (Task 7) + - `coda_interactive` built incrementally Tasks 5→7 with each task's tests gating progress + - Flask dispatch (Task 8) and instructions (Task 9) come after the tool itself exists + - Regression guard (Task 10) verifies the final state + +5. **Test discipline** ✓ + - Every code-adding task has a failing test in Step 1, verified failure in Step 2, implementation in Step 3, verified pass in Step 4 + - Tasks 8 (wiring) and 9 (docs) are not TDD but are minimal-risk + - Final regression guard (Task 10) defends against future drift + +--- + +## Plan critique gate + +**Cleared** (2026-05-28). Critic verdict: APPROVE WITH CHANGES. All flagged issues incorporated: + +1. **CRITICAL — Task 1 `sessions[sid]["env"]` key didn't exist.** Fixed: Task 1 now has an explicit Step 2 that adds the `"env"` key to the session dict before the env-strip refactor. Step 3 verifies the test fails for the RIGHT reason (credentials present), not silently passes. +2. **MAJOR — Task 7→Task 8 orphaned-state rework.** Fixed: Tasks 7 and 8 merged into a single Task 7 that creates the PTY FIRST, then builds the project_dir keyed by the PTY's session_id, then exports + cds + launches + seeds. Eliminates the intermediate state where the project dir's name didn't match the PTY's actual session id. +3. **MAJOR — Line number drift.** Fixed: `app.py:1402` → `app.py:1420`. `mcp_server.py:218` → "around line 190; insert between `coda_run` (ends near 289) and `coda_inbox`". Other line refs verified accurate. + +Original 10 critique questions, all answered in the critique pass: + +1. **Task 7 chicken-and-egg** — Resolved by merging Tasks 7+8. +2. **`cwd` kwarg unused** — Acceptable; tested optional kwarg left in place. Documented in Task 7 Acknowledgment. +3. **`WorkspaceClient` monkeypatch target** — Confirmed correct. Task 6 imports it module-level. +4. **`sessions[sid]["env"]` key** — Added explicitly in Task 1 Step 2 (was missing). +5. **`_PROMPT_SEED_DELAY_S` flake risk** — Tests patch to 0. Acceptable. +6. **`_app_create_session is None` null-check** — Consistent with `coda_run`'s pattern. +7. **`os.makedirs(exist_ok=True)`** — UUID collision probability negligible. Acceptable. +8. **Per-task commits** — Matches Todo 1's commit conventions. +9. **Line numbers** — Two references corrected (see MAJOR #3 above). +10. **Test count expectation** — Plausible estimates; exact counts depend on PTY availability. + +Plus eight additional critic-eye questions (spec coverage, ordering, TDD discipline, line numbers, test correctness, fragile assumptions, plan gate), all resolved. See the critic's verdict in the conversation history. + +Plan is ready for execution. diff --git a/docs/superpowers/plans/2026-05-28-coda-interactive-terminal-pull.md b/docs/superpowers/plans/2026-05-28-coda-interactive-terminal-pull.md new file mode 100644 index 0000000..0b244a1 --- /dev/null +++ b/docs/superpowers/plans/2026-05-28-coda-interactive-terminal-pull.md @@ -0,0 +1,581 @@ +# `coda_interactive` Terminal-Side Pull — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development or superpowers:executing-plans. Steps use `- [ ]` checkboxes. + +**Goal:** Replace `coda_interactive`'s broken server-side Workspace export (runs as the app SP, which can't read the user's folder) with a terminal-side `databricks workspace export-dir` pull (runs as the user), guarded by a split wait + a server-side filesystem post-check. Delete `workspace_export.py`. + +**Architecture:** The MCP server types a chained `cd && databricks workspace export-dir ./ && cd ` into the PTY (which is authenticated as the app owner), waits for the pull to settle, verifies on the local filesystem that files arrived, then launches the agent and seeds the prompt. No `WorkspaceClient` in the tool anymore. + +**Tech stack:** Python 3.11, pytest, FastMCP. No new dependencies. Run tests with `uv run pytest`. + +**Reference:** `docs/superpowers/specs/2026-05-28-coda-interactive-terminal-pull-design.md` (full design, error table, risks). + +--- + +## Files + +- **Modify:** `coda_mcp/mcp_server.py` — remove export import + `WorkspaceClient` usage; add `re` import; add `_safe_dirname`, `_normalize_workspace_path`; refactor `_wait_for_agent_ready` → `_wait_for_output_stable` + wrapper; add `_EXPORT_MAX_WAIT_S`/`_EXPORT_STABILITY_S`; rewrite `coda_interactive` body. +- **Delete:** `coda_mcp/workspace_export.py`, `tests/test_workspace_export.py`. +- **Modify:** `tests/test_replay_only_flag.py` — refresh stale comment (line ~166). +- **Rewrite:** `tests/test_coda_interactive.py`. +- **Modify:** `tests/test_mcp_server.py` — add helper + wrapper tests. + +## Pre-flight + +- Worktree: `/Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp`, branch `feat/coda-mcp-interactive-handoff` (already merged with main / deps bump, HEAD `2dd66aa`). +- Commit identity: `-c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty"`. No AI co-author. +- `databricks workspace export-dir SOURCE TARGET` is verified: creates TARGET, recursive, auto notebook extensions, `--overwrite` flag (not needed here). + +--- + +## Task 1: Helpers + wait-helper refactor (TDD) + +**Files:** Modify `coda_mcp/mcp_server.py`; add tests to `tests/test_mcp_server.py`. + +- [ ] **Step 1: Write failing tests** — append to `tests/test_mcp_server.py`: + +```python +class TestInteractiveHelpers: + def test_safe_dirname_basename(self): + from coda_mcp.mcp_server import _safe_dirname + assert _safe_dirname("/Users/x@y.com/WAM") == "WAM" + assert _safe_dirname("/Users/x@y.com/WAM/") == "WAM" + + def test_safe_dirname_sanitizes(self): + from coda_mcp.mcp_server import _safe_dirname + assert _safe_dirname("/Users/x/My Project!") == "My_Project_" + + def test_safe_dirname_empty_fallback(self): + from coda_mcp.mcp_server import _safe_dirname + assert _safe_dirname("/") == "workspace" + assert _safe_dirname("") == "workspace" + + def test_normalize_strips_workspace_prefix(self): + from coda_mcp.mcp_server import _normalize_workspace_path + assert _normalize_workspace_path("/Workspace/Users/x/WAM") == "/Users/x/WAM" + + def test_normalize_leaves_plain_path(self): + from coda_mcp.mcp_server import _normalize_workspace_path + assert _normalize_workspace_path("/Users/x/WAM") == "/Users/x/WAM" + assert _normalize_workspace_path("/Users/x/WAM/") == "/Users/x/WAM" + + @pytest.mark.asyncio + async def test_wait_for_agent_ready_delegates(self, monkeypatch): + """_wait_for_agent_ready calls _wait_for_output_stable with prompt-seed constants.""" + from coda_mcp import mcp_server + seen = {} + async def fake_stable(pty, max_wait, stability): + seen["args"] = (pty, max_wait, stability) + monkeypatch.setattr(mcp_server, "_wait_for_output_stable", fake_stable) + await mcp_server._wait_for_agent_ready("pty-1") + assert seen["args"] == ("pty-1", mcp_server._PROMPT_SEED_MAX_WAIT_S, mcp_server._PROMPT_SEED_STABILITY_S) +``` + +- [ ] **Step 2: Run, expect FAIL** — `uv run pytest tests/test_mcp_server.py::TestInteractiveHelpers -v` → all fail (symbols don't exist). + +- [ ] **Step 3: Add `re` import** to `coda_mcp/mcp_server.py` (near `import os` at line 19, keep alphabetical-ish with the stdlib group): + +```python +import re +``` + +- [ ] **Step 4: Add the two helpers** in `coda_mcp/mcp_server.py` just above `_ALLOWED_AGENTS` (line 336): + +```python +def _safe_dirname(workspace_path: str) -> str: + """Local directory name for the pulled folder = sanitized basename.""" + base = os.path.basename(workspace_path.rstrip("/")) + safe = re.sub(r"[^A-Za-z0-9._-]", "_", base) + return safe or "workspace" + + +def _normalize_workspace_path(workspace_path: str) -> str: + """Canonical Workspace API path: drop the /Workspace FUSE prefix if present.""" + p = workspace_path.rstrip("/") + if p.startswith("/Workspace/"): + p = p[len("/Workspace"):] + return p +``` + +- [ ] **Step 5: Refactor the wait helper.** Replace the existing `_wait_for_agent_ready` definition (lines 346-380, the `async def _wait_for_agent_ready(...)` through the end of its `while` loop) with a generalized function plus a thin wrapper. Also add the two new constants next to the existing ones (after line 343): + +Add constants (after `_PROMPT_SEED_STABILITY_S = 1.0`): + +```python +_EXPORT_MAX_WAIT_S = 120.0 # generous; export-dir prints per-file so it won't prematurely stabilize mid-pull +_EXPORT_STABILITY_S = 1.5 +``` + +Replace the function: + +```python +async def _wait_for_output_stable(pty_session_id: str, max_wait: float, stability: float) -> None: + """Poll the PTY output buffer; return when it stabilizes or max_wait elapses. + + Stability = buffer length unchanged for ``stability`` seconds, after at + least one byte has appeared. If the session disappears mid-wait, return. + """ + from app import sessions + loop = asyncio.get_running_loop() + deadline = loop.time() + max_wait + last_len = -1 + stable_since: float | None = None + poll_interval = 0.1 + + while loop.time() < deadline: + await asyncio.sleep(poll_interval) + sess = sessions.get(pty_session_id) + if sess is None: + return + current_len = sum(len(chunk) for chunk in sess.get("output_buffer", [])) + if current_len > 0 and current_len == last_len: + if stable_since is None: + stable_since = loop.time() + elif (loop.time() - stable_since) >= stability: + return + else: + stable_since = None + last_len = current_len + + +async def _wait_for_agent_ready(pty_session_id: str) -> None: + """Wait for an agent TUI to settle (prompt-seed budget). Wrapper for back-compat.""" + await _wait_for_output_stable( + pty_session_id, _PROMPT_SEED_MAX_WAIT_S, _PROMPT_SEED_STABILITY_S + ) +``` + +- [ ] **Step 6: Run, expect PASS** — `uv run pytest tests/test_mcp_server.py::TestInteractiveHelpers -v` → all pass. Then `uv run pytest tests/test_mcp_server.py -q` → no regressions (coda_run still uses `_wait_for_agent_ready`). + +- [ ] **Step 7: Ruff** — `uv run ruff check coda_mcp/mcp_server.py tests/test_mcp_server.py` → clean. + +- [ ] **Step 8: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add coda_mcp/mcp_server.py tests/test_mcp_server.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat: add _safe_dirname/_normalize_workspace_path + generalize wait helper + +_wait_for_output_stable(pty, max_wait, stability) is the parametrized poller; +_wait_for_agent_ready becomes a thin wrapper preserving the 5.0/1.0 budget so +coda_run is unaffected. Adds _EXPORT_MAX_WAIT_S/_EXPORT_STABILITY_S for the +upcoming terminal-side pull wait." +``` + +--- + +## Task 2: Rewrite `coda_interactive` + delete export module (TDD) + +**Files:** Modify `coda_mcp/mcp_server.py`; delete `coda_mcp/workspace_export.py` + `tests/test_workspace_export.py`; rewrite `tests/test_coda_interactive.py`; touch `tests/test_replay_only_flag.py` comment. + +- [ ] **Step 1: Rewrite `tests/test_coda_interactive.py`** to the new contract. Replace the whole file with: + +```python +"""Tests for coda_interactive — terminal-side workspace pull (no server-side export).""" +import json +import os + +import pytest + +from coda_mcp import mcp_server + + +@pytest.fixture +def wired(monkeypatch, tmp_path): + """Wire PTY hooks with recording mocks; HOME -> tmp so project_dir is sandboxed. + + The _app_send_input mock simulates a SUCCESSFUL export-dir by creating the + target dir + a file when it sees the pull command. Tests that want the + failure path override `simulate_pull` to False. + """ + monkeypatch.setenv("HOME", str(tmp_path)) + inputs: list[str] = [] + state = {"pty_id": "pty-abc123", "simulate_pull": True, "closed": []} + + def fake_create(label, replay_only=False, **kw): + return state["pty_id"] + + def fake_send(pty_id, text): + inputs.append(text) + # Simulate export-dir landing files on disk. + if state["simulate_pull"] and "export-dir" in text: + # project_dir = ~/.coda/projects/; name parsed from the command tail "cd " + project_dir = os.path.join(os.path.expanduser("~/.coda/projects"), state["pty_id"]) + # name is the final `cd ` token + name = text.rstrip().rsplit("cd ", 1)[-1].strip().strip("'\"") + target = os.path.join(project_dir, name) + os.makedirs(target, exist_ok=True) + with open(os.path.join(target, "README.md"), "w") as f: + f.write("# hi") + + def fake_close(pty_id): + state["closed"].append(pty_id) + + async def fake_wait(*a, **kw): + return None + + monkeypatch.setattr(mcp_server, "_app_create_session", fake_create) + monkeypatch.setattr(mcp_server, "_app_send_input", fake_send) + monkeypatch.setattr(mcp_server, "_app_close_session", fake_close) + monkeypatch.setattr(mcp_server, "_wait_for_output_stable", fake_wait) + monkeypatch.setattr(mcp_server, "_wait_for_agent_ready", fake_wait) + monkeypatch.setattr(mcp_server.url_builder, "build_viewer_url", lambda pid: f"https://viewer/{pid}") + return inputs, state + + +@pytest.mark.asyncio +async def test_pull_command_is_sent_first(wired): + inputs, _ = wired + await mcp_server.coda_interactive( + prompt="analyze", workspace_path="/Workspace/Users/x@y.com/WAM", agent="claude") + first = inputs[0] + assert "databricks workspace export-dir" in first + assert "/Users/x@y.com/WAM" in first # /Workspace prefix stripped + assert "/Workspace/Users" not in first + assert "./WAM" in first and first.rstrip().endswith("WAM") # cd tail + + +@pytest.mark.asyncio +async def test_agent_launches_after_successful_pull(wired): + inputs, _ = wired + await mcp_server.coda_interactive( + prompt="go", workspace_path="/Users/x/WAM", agent="claude") + assert any(t.strip() == "claude" for t in inputs) + + +@pytest.mark.asyncio +async def test_prompt_seeded_with_context_line(wired): + inputs, _ = wired + await mcp_server.coda_interactive( + prompt="DO THE THING", workspace_path="/Users/x/WAM", agent="claude") + seeded = inputs[-1] + assert "/Users/x/WAM" in seeded + assert "DO THE THING" in seeded + assert "Workspace" in seeded # precondition (clean fail, not ValueError) + assert seeded.index("Workspace") < seeded.index("DO THE THING") # context precedes prompt + + +def test_instructions_drop_stale_export_wording(): + """Server-level MCP instructions must not claim the deleted server-side export.""" + txt = mcp_server.mcp.instructions + assert "server-side snapshot" not in txt + assert "export-dir" in txt # describes the real terminal-side pull mechanism + + +@pytest.mark.asyncio +async def test_empty_pull_returns_error_and_no_launch(wired): + inputs, state = wired + state["simulate_pull"] = False # export-dir produces nothing + out = json.loads(await mcp_server.coda_interactive( + prompt="go", workspace_path="/Users/x/WAM", agent="claude")) + assert out["status"] == "error" + assert state["closed"] == [state["pty_id"]] # PTY closed + assert not any(t.strip() == "claude" for t in inputs) # agent NOT launched + + +@pytest.mark.asyncio +async def test_happy_path_returns_launched(wired): + out = json.loads(await mcp_server.coda_interactive( + prompt="go", workspace_path="/Users/x/WAM", agent="claude")) + assert out["status"] == "launched" + assert out["viewer_url"] == "https://viewer/pty-abc123" + assert out["project_dir"].endswith(os.path.join("pty-abc123", "WAM")) + + +@pytest.mark.asyncio +async def test_unknown_agent_rejected(wired): + out = json.loads(await mcp_server.coda_interactive( + prompt="x", workspace_path="/Users/x/WAM", agent="bogus")) + assert out["status"] == "error" and "Unknown agent" in out["error"] + + +@pytest.mark.asyncio +async def test_pty_hook_not_wired(monkeypatch): + monkeypatch.setattr(mcp_server, "_app_create_session", None) + monkeypatch.setattr(mcp_server, "_app_send_input", None) + out = json.loads(await mcp_server.coda_interactive( + prompt="x", workspace_path="/Users/x/WAM", agent="claude")) + assert out["status"] == "error" and "PTY hook" in out["error"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("agent,cmd", [ + ("claude", "claude"), ("hermes", "hermes chat"), ("codex", "codex"), + ("gemini", "gemini"), ("opencode", "opencode"), +]) +async def test_agent_matrix(wired, agent, cmd): + inputs, _ = wired + await mcp_server.coda_interactive( + prompt="go", workspace_path="/Users/x/WAM", agent=agent) + assert any(t.strip() == cmd for t in inputs) + + +def test_no_blocking_sleep_in_source(): + import inspect + src = inspect.getsource(mcp_server.coda_interactive) + assert "time.sleep(" not in src + + +def test_no_workspaceclient_in_module(): + """The export-era WorkspaceClient import/use is gone from the module.""" + import inspect + src = inspect.getsource(mcp_server) + assert "export_workspace_tree" not in src + assert "workspace.get_status(" not in src +``` + +- [ ] **Step 2: Run, expect FAIL** — `uv run pytest tests/test_coda_interactive.py -q` → fails (old behavior still in place; `export_workspace_tree`/`get_status` still present). + +- [ ] **Step 3: Rewrite `coda_interactive`** in `coda_mcp/mcp_server.py`. Replace the entire function body (lines 416-523, from `if agent not in _ALLOWED_AGENTS:` through the catch-all `return`) with: + +```python + if agent not in _ALLOWED_AGENTS: + return json.dumps({ + "status": "error", + "error": f"Unknown agent: {agent!r}. Allowed: {sorted(_ALLOWED_AGENTS)}", + }) + + if _app_create_session is None or _app_send_input is None: + return json.dumps({ + "status": "error", + "error": "PTY hook not wired", + }) + + pty_session_id = None + project_dir = None + try: + # Create PTY FIRST so we have its session_id for the project_dir name. + pty_session_id = _app_create_session( + label=f"{agent}-interactive", + replay_only=False, + ) + project_dir = os.path.join( + os.path.expanduser("~/.coda/projects"), + pty_session_id, + ) + os.makedirs(project_dir, exist_ok=True) + + name = _safe_dirname(workspace_path) + source_path = _normalize_workspace_path(workspace_path) + + # Pull the Workspace folder into ./ AS THE USER (terminal creds). + # A failed export-dir short-circuits the && chain, leaving absent; + # the filesystem check below turns that into a real error. + pull_cmd = ( + f"cd {shlex.quote(project_dir)} && " + f"databricks workspace export-dir {shlex.quote(source_path)} {shlex.quote('./' + name)} && " + f"cd {shlex.quote(name)}" + ) + _app_send_input(pty_session_id, pull_cmd + "\n") + + # Wait for the pull to finish (shell goes idle), then verify on disk. + await _wait_for_output_stable( + pty_session_id, _EXPORT_MAX_WAIT_S, _EXPORT_STABILITY_S + ) + + target_dir = os.path.join(project_dir, name) + if not os.path.isdir(target_dir) or not os.listdir(target_dir): + if _app_close_session is not None: + try: + _app_close_session(pty_session_id) + except Exception: + pass + if os.path.isdir(project_dir): + shutil.rmtree(project_dir, ignore_errors=True) + return json.dumps({ + "status": "error", + "error": ( + f"No files were pulled from {workspace_path}. Check the path " + f"exists in the Workspace and that you have read access." + ), + }) + + # Launch the agent (fresh — same proven path as before). + launch_cmd = _AGENT_LAUNCH_CMDS[agent] + _app_send_input(pty_session_id, launch_cmd + "\n") + + # Wait for the agent TUI to settle, then paste the kickoff prompt with a + # context line naming the source so the agent knows where the files came from. + await _wait_for_agent_ready(pty_session_id) + seeded_prompt = ( + f"Your working directory contains files exported from the Databricks " + f"Workspace path {workspace_path}.\n\n{prompt}" + ) + _app_send_input(pty_session_id, seeded_prompt + "\n") + + viewer_url = url_builder.build_viewer_url(pty_session_id) + + return json.dumps({ + "status": "launched", + "viewer_url": viewer_url, + "agent": agent, + "project_dir": target_dir, + "workspace_path": workspace_path, + "instructions": ( + "Open viewer_url to attach. The agent is running in a directory " + "holding the files pulled from your Workspace folder, with your " + "kickoff prompt typed. Type the agent's quit command (e.g. /quit) " + "then `exit` to end the session. Note: files are a snapshot pulled " + "via 'databricks workspace export-dir' — git history is not included." + ), + }) + except Exception as e: + if pty_session_id and _app_close_session is not None: + try: + _app_close_session(pty_session_id) + except Exception: + pass + if project_dir and os.path.isdir(project_dir): + shutil.rmtree(project_dir, ignore_errors=True) + return json.dumps({ + "status": "error", + "error": f"coda_interactive failed: {e}", + }) +``` + +- [ ] **Step 4: Update the `coda_interactive` docstring** (lines 398-414). Replace the body text so it no longer says "exports its file tree / server-side snapshot". New docstring: + +```python + """Launch an interactive agent session in CoDA, handed off via a viewer URL. + + The MCP caller passes a Databricks Workspace directory path. CoDA pulls that + folder onto the session's disk IN THE TERMINAL (authenticated as you) via + ``databricks workspace export-dir``, launches the chosen agent (claude + default) in the pulled directory, auto-types ``prompt`` as the first user + input, and returns a ``viewer_url`` the calling user opens to drive it. + + If the pull produces no files (bad path or no read access) the tool returns + a ``status=error`` and does not launch the agent. + + Interactive sessions do NOT appear in ``coda_inbox`` and ``coda_get_result`` + will not return anything for them. The viewer URL is the only handle. + + ``email`` is accepted for forward-compatibility and is currently unused. + + Allowed agents: claude (default), hermes, codex, gemini, opencode. + """ +``` + +- [ ] **Step 4b: Update the server-level `mcp.instructions` blob** (lines ~95-98) so it no longer claims the deleted server-side export. Replace the exact substring: + +``` +"Folder, ensure the desired branch is checked out and pushed first — " +"the export is a server-side snapshot. The tool exports the directory " +"into a Coda-local working directory, launches the chosen agent " +``` + +with: + +``` +"Folder, ensure the desired branch is checked out first — " +"the pull is a point-in-time snapshot. The tool copies the directory " +"into a Coda-local working directory using your credentials (via " +"`databricks workspace export-dir`), launches the chosen agent " +``` + +This keeps the caller-facing contract (pass `workspace_path`, files-only, no git history) but stops describing a mechanism that no longer exists. Guarded by `test_instructions_drop_stale_export_wording`. + +- [ ] **Step 5: Remove the dead export imports.** In `coda_mcp/mcp_server.py` line 31, delete: + +```python +from coda_mcp.workspace_export import export_workspace_tree, _is_directory +``` + +And remove the `WorkspaceClient` import guard (lines ~33-36) IF nothing else in the file uses `WorkspaceClient`. Verify first: + +```bash +grep -n "WorkspaceClient" coda_mcp/mcp_server.py +``` + +If the only hits are the import guard, delete the guard block: + +```python +try: + from databricks.sdk import WorkspaceClient +except Exception: + WorkspaceClient = None # type: ignore +``` + +If `WorkspaceClient` is used elsewhere, leave the guard and only remove `coda_interactive`'s usage (already done in Step 3). + +- [ ] **Step 6: Delete the export module + its tests** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" rm coda_mcp/workspace_export.py tests/test_workspace_export.py +``` + +- [ ] **Step 7: Refresh the stale comment** in `tests/test_replay_only_flag.py` (~line 166). It currently references `export_workspace_tree`. Read the surrounding lines and reword so it describes the invariant generically (e.g. "must not create a project directory / pull workspace files") without naming the deleted symbol. Do NOT change the test's logic. + +- [ ] **Step 8: Run the target tests, expect PASS** + +```bash +uv run pytest tests/test_coda_interactive.py tests/test_mcp_server.py -v +``` +Expect all green. If `test_pull_command_is_sent_first` fails on the `endswith("WAM")` assertion, inspect the actual `pull_cmd` string and adjust the test's tail assertion to match the real (shlex-quoted) form — the production string is the source of truth for *behavior*, but the command MUST contain `databricks workspace export-dir`, the normalized source, and a final `cd `. + +- [ ] **Step 9: Import sanity** — `uv run python -c "import coda_mcp.mcp_server; import app"` → no ImportError (confirms the deleted module isn't imported anywhere at load time). + +- [ ] **Step 10: Ruff** — `uv run ruff check coda_mcp/mcp_server.py tests/test_coda_interactive.py tests/test_replay_only_flag.py` → clean (watch for now-unused imports like `shutil`/`shlex` — both are still used; confirm). + +- [ ] **Step 11: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add coda_mcp/mcp_server.py tests/test_coda_interactive.py tests/test_replay_only_flag.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat: coda_interactive pulls workspace files in the terminal, not server-side + +Root cause of the empty-session bug: the MCP server's WorkspaceClient runs as +the app service principal, which can't list/export the user's Workspace folder, +and the error was swallowed. Now the tool types 'databricks workspace export-dir' +into the PTY (authed as the user), waits for the pull to settle, verifies files +landed on disk, then launches the agent and seeds the prompt. Deletes +workspace_export.py and the server-side WorkspaceClient/get_status path." +``` + +--- + +## Task 3: Full regression sweep + +**Files:** none (verification only). + +- [ ] **Step 1: Targeted suite** + +```bash +uv run pytest tests/test_coda_interactive.py tests/test_mcp_server.py tests/test_task_manager.py tests/test_databricks_preamble.py tests/test_replay_only_flag.py -v +``` +Expect green. `test_replay_only_flag.py::test_coda_run_creates_pty_with_replay_only_true` is PTY-fd flaky in multi-file runs — if it fails, re-run that file alone; if it passes alone, it's environmental. + +- [ ] **Step 2: Confirm `workspace_export` is fully gone** + +```bash +grep -rn "workspace_export\|export_workspace_tree" coda_mcp/ tests/ || echo "CLEAN — no references remain" +``` +Expect only (at most) the reworded comment in `test_replay_only_flag.py` if you kept any mention; ideally CLEAN. + +- [ ] **Step 3: Ruff over the package** + +```bash +uv run ruff check coda_mcp/ tests/test_coda_interactive.py +``` +Expect clean. + +No commit (verification only). Proceed to final critic + push. + +--- + +## Self-review vs spec + +- AC1 (no export/WorkspaceClient/get_status in coda_interactive) → Task 2 Steps 3, 5; guarded by `test_no_workspaceclient_in_module`. +- AC2 (module + tests deleted, no importers) → Task 2 Step 6; Task 3 Step 2. +- AC3 (`_safe_dirname`/`_normalize_workspace_path`) → Task 1 Steps 4; tests Step 1. +- AC4 (`_wait_for_output_stable` + wrapper, coda_run unaffected) → Task 1 Step 5; `test_wait_for_agent_ready_delegates` + `tests/test_mcp_server.py` regression. +- AC5 (first input = chained pull, normalized source, ``) → `test_pull_command_is_sent_first`. +- AC6 (launch only if FS check passes; else error + close) → `test_empty_pull_returns_error_and_no_launch`. +- AC7 (prompt prefixed with context line) → `test_prompt_seeded_with_context_line`. +- AC8 (new + existing suites green) → Task 3. + +**Placeholder scan:** none. **Type consistency:** `_wait_for_output_stable(pty, max_wait, stability)` signature identical across Task 1 def, the wrapper, and `coda_interactive`'s two call sites. `_safe_dirname`/`_normalize_workspace_path` names identical in helpers, tests, and `coda_interactive`. + +**Risk flagged for the executor:** the `fake_send` mock in `test_coda_interactive.py` parses `` from the command tail via `rsplit("cd ", 1)`. If the production `pull_cmd` quoting makes that parse brittle, the executor should instead compute `name` in the fixture from the known `workspace_path` basename rather than parsing the command. The intent: simulate files appearing at `~/.coda/projects///`. diff --git a/docs/superpowers/plans/2026-05-28-coda-run-replay-only.md b/docs/superpowers/plans/2026-05-28-coda-run-replay-only.md new file mode 100644 index 0000000..6aacaf6 --- /dev/null +++ b/docs/superpowers/plans/2026-05-28-coda-run-replay-only.md @@ -0,0 +1,1079 @@ +# `coda_run` Replay-Only URL Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `coda_run`'s returned `viewer_url` resolve to a read-only static transcript replay (never a live PTY attach), and rip out the unwired 5-minute grace-period machinery from PR #66 as a consequence. + +**Architecture:** Mode 3 in the three-mode framework (see spec `docs/superpowers/specs/2026-05-28-coda-run-replay-only-design.md`). A new `replay_only` boolean on the PTY session dict steers the existing `/api/session/attach` endpoint into the transcript-from-disk path unconditionally for `coda_run`-created sessions. The watcher closes the PTY immediately on task completion — no deferred timer. + +**Tech Stack:** Python 3.11 + Flask + FastMCP + uvicorn (ASGI) + pytest. No new deps. All changes localized to `app.py`, `coda_mcp/mcp_server.py`, and the test suite. + +--- + +## Pre-flight check (do before Task 1) + +- [ ] **P1: Verify baseline tests pass.** + +```bash +cd /Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp +.venv/bin/python -m pytest tests/ -x --ignore=tests/e2e -q 2>&1 | tail -20 +``` + +Expected: All pass (~527 passed + ~11 PTY-gated skipped). If anything fails on `main` for unrelated reasons, stop and report. + +- [ ] **P2: Confirm worktree is on the `feat/coda-mcp-live-session-url` branch.** + +```bash +git branch --show-current +``` + +Expected: `feat/coda-mcp-live-session-url` + +--- + +## Task 1: Add `replay_only` parameter to `mcp_create_pty_session` + +Backward-compatible default (`False`) so existing callers (direct-launch via `create_session`, future `coda_interactive`) keep their behavior unchanged. + +**Files:** +- Modify: `app.py` (function `mcp_create_pty_session`, line ~1402, and the session-dict insert at ~1469) +- Create: `tests/test_replay_only_flag.py` + +- [ ] **Step 1: Write the failing test.** + +Create `tests/test_replay_only_flag.py`: + +```python +"""Tests for the replay_only flag on PTY sessions.""" +import pytest + +# Reuse the PTY-availability guard pattern from the suite. +import os +try: + import pty as _pty + _master, _slave = _pty.openpty() + os.close(_master) + os.close(_slave) + _PTY_AVAILABLE = True +except Exception: + _PTY_AVAILABLE = False + +_pty_skip = pytest.mark.skipif( + not _PTY_AVAILABLE, + reason="PTY not allocatable in this environment", +) + + +@_pty_skip +def test_mcp_create_pty_session_stores_replay_only_flag(): + """Creating a PTY with replay_only=True stores the flag in the session dict.""" + from app import mcp_create_pty_session, mcp_close_pty_session, sessions + + sid = mcp_create_pty_session(label="t1", replay_only=True) + try: + assert sessions[sid].get("replay_only") is True + finally: + mcp_close_pty_session(sid) + + +@_pty_skip +def test_mcp_create_pty_session_defaults_replay_only_false(): + """Default for replay_only is False (backward compat).""" + from app import mcp_create_pty_session, mcp_close_pty_session, sessions + + sid = mcp_create_pty_session(label="t2") + try: + assert sessions[sid].get("replay_only") is False + finally: + mcp_close_pty_session(sid) +``` + +- [ ] **Step 2: Run the test and verify it fails.** + +```bash +.venv/bin/python -m pytest tests/test_replay_only_flag.py -v 2>&1 | tail -15 +``` + +Expected: 2 failures. First test fails with `TypeError: mcp_create_pty_session() got an unexpected keyword argument 'replay_only'`. Second test fails with `assert None is False` (the key doesn't exist yet so `.get` returns None, which is not `False`). + +- [ ] **Step 3: Add the parameter and storage.** + +In `app.py`, change the `mcp_create_pty_session` signature (search for `def mcp_create_pty_session`): + +```python +# Before: +def mcp_create_pty_session(label: str = "hermes-mcp", transcript_path: str | None = None) -> str: + +# After: +def mcp_create_pty_session( + label: str = "hermes-mcp", + transcript_path: str | None = None, + replay_only: bool = False, +) -> str: +``` + +In the same function, add the `replay_only` key to the session dict that's being built (find the dict literal that contains `"grace": False,` — that's the one). Add right after the existing `"grace": False,` line: + +```python + "grace": False, + "replay_only": replay_only, # NEW +``` + +(The `"grace": False,` line gets removed entirely in Task 8 — leave it alone here.) + +- [ ] **Step 4: Run the test and verify it passes.** + +```bash +.venv/bin/python -m pytest tests/test_replay_only_flag.py -v 2>&1 | tail -10 +``` + +Expected: 2 passed. + +- [ ] **Step 5: Commit.** + +```bash +git add app.py tests/test_replay_only_flag.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: add replay_only param to mcp_create_pty_session + +Backward-compatible default (False). Stored in session dict for later +attach-time enforcement." +``` + +--- + +## Task 2: Extract `_serve_transcript_replay` helper from `attach_session` + +Pure refactor. Extracts the transcript-from-disk lookup currently inlined in `attach_session` at `app.py:1170-1188` into a reusable helper. Existing tests (`tests/test_replay_attach.py`) act as the safety net. + +**Files:** +- Modify: `app.py` (`attach_session` at ~1158, plus new helper above it) + +- [ ] **Step 1: Verify existing replay tests pass (the safety net).** + +```bash +.venv/bin/python -m pytest tests/test_replay_attach.py -v 2>&1 | tail -10 +``` + +Expected: 2 passed (the two tests that already exist for transcript-after-PTY-exit replay). + +- [ ] **Step 2: Add the helper just above `attach_session`.** + +In `app.py`, find `@app.route("/api/session/attach"` (around line 1157). Just **above** the `@app.route` decorator, add this helper: + +```python +def _serve_transcript_replay(session_id: str): + """Serve the on-disk transcript for a PTY session as a replay response. + + Used by attach_session() in two cases: + 1. The PTY is gone (existing transcript-fallback path). + 2. The PTY exists but is replay_only=True (new in Task 3). + + Returns either a Flask JSON response with replay=True, or a 404 if no + transcript exists for this pty_session_id. + """ + from coda_mcp import task_manager as _tm + tdir = _tm.find_task_dir_by_pty_session(session_id) + if tdir: + transcript = os.path.join(tdir, "transcript.log") + if os.path.isfile(transcript): + try: + with open(transcript, "rb") as f: + content = f.read() + return jsonify({ + "session_id": session_id, + "label": "hermes-mcp (replay)", + "output": [content.decode("utf-8", errors="replace")], + "replay": True, + "process": None, + "created_at": None, + }) + except OSError: + pass + return jsonify({"error": "Session not found or exited"}), 404 +``` + +- [ ] **Step 3: Replace the inlined block in `attach_session` with a helper call.** + +Inside `attach_session`, find the block: + +```python + sess = _get_session(session_id) + if not sess or sess.get("exited"): + # Replay fallback: look up transcript.log by pty_session_id + from coda_mcp import task_manager as _tm + tdir = _tm.find_task_dir_by_pty_session(session_id) + if tdir: + transcript = os.path.join(tdir, "transcript.log") + if os.path.isfile(transcript): + try: + with open(transcript, "rb") as f: + content = f.read() + return jsonify({ + "session_id": session_id, + "label": "hermes-mcp (replay)", + "output": [content.decode("utf-8", errors="replace")], + "replay": True, + "process": None, + "created_at": None, + }) + except OSError: + pass + return jsonify({"error": "Session not found or exited"}), 404 +``` + +Replace it with: + +```python + sess = _get_session(session_id) + if not sess or sess.get("exited"): + return _serve_transcript_replay(session_id) +``` + +- [ ] **Step 4: Run replay tests to verify behavior is preserved.** + +```bash +.venv/bin/python -m pytest tests/test_replay_attach.py tests/test_transcript.py -v 2>&1 | tail -20 +``` + +Expected: All pass (refactor is behavior-preserving). + +- [ ] **Step 5: Commit.** + +```bash +git add app.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "refactor: extract _serve_transcript_replay helper from attach_session + +Pure refactor — no behavior change. Helper is also used by the new +replay_only short-circuit in the next commit." +``` + +--- + +## Task 3: Enforce `replay_only=True` in `attach_session` + +New early-return: if the live session has `replay_only=True`, serve the transcript regardless of whether the PTY is still alive. + +**Files:** +- Modify: `app.py` (`attach_session`) +- Modify: `tests/test_replay_only_flag.py` + +- [ ] **Step 1: Add two failing tests.** + +Append to `tests/test_replay_only_flag.py`: + +```python +@_pty_skip +def test_attach_session_replay_only_alive_pty_returns_replay(tmp_path, monkeypatch): + """A replay_only=True PTY that is still alive serves the transcript, not the live buffer.""" + from app import app as flask_app, mcp_create_pty_session, mcp_close_pty_session, sessions + from coda_mcp import task_manager + + # Point task_manager at a tmp sessions root so find_task_dir_by_pty_session resolves. + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + + # Create a fake task dir keyed by the PTY id we'll mint shortly. + sid = mcp_create_pty_session(label="t-replay-alive", replay_only=True) + try: + # Plant a session.json that links task → this pty_session_id, plus a transcript. + sess_id = "sess-fake" + task_id = "task-fake" + sdir = tmp_path / sess_id + tdir = sdir / "tasks" / task_id + tdir.mkdir(parents=True) + (sdir / "session.json").write_text( + '{"session_id": "%s", "pty_session_id": "%s"}' % (sess_id, sid) + ) + (tdir / "transcript.log").write_bytes(b"HELLO TRANSCRIPT") + + # Bust the lookup cache so find_task_dir_by_pty_session sees the new files. + task_manager._pty_lookup_cache.clear() + + client = flask_app.test_client() + resp = client.post("/api/session/attach", json={"session_id": sid}) + + assert resp.status_code == 200 + body = resp.get_json() + assert body["replay"] is True + assert body["output"] == ["HELLO TRANSCRIPT"] + finally: + mcp_close_pty_session(sid) + + +@_pty_skip +def test_attach_session_replay_only_false_alive_pty_returns_live_buffer(): + """A replay_only=False PTY that is still alive returns the live output_buffer (unchanged behavior).""" + from app import app as flask_app, mcp_create_pty_session, mcp_close_pty_session + + sid = mcp_create_pty_session(label="t-live", replay_only=False) + try: + client = flask_app.test_client() + resp = client.post("/api/session/attach", json={"session_id": sid}) + + assert resp.status_code == 200 + body = resp.get_json() + assert body.get("replay") in (False, None) # live path doesn't set replay key + assert "output" in body + finally: + mcp_close_pty_session(sid) +``` + +- [ ] **Step 2: Run the new tests and verify they fail (first one only — second should pass already).** + +```bash +.venv/bin/python -m pytest tests/test_replay_only_flag.py -v 2>&1 | tail -20 +``` + +Expected: `test_attach_session_replay_only_alive_pty_returns_replay` FAILS (because the alive PTY currently returns the live buffer, not the transcript). `test_attach_session_replay_only_false_alive_pty_returns_live_buffer` PASSES (existing behavior is correct). The two Task 1 tests still pass. + +- [ ] **Step 3: Add the early-return in `attach_session`.** + +In `app.py`, modify the body of `attach_session`. Find: + +```python + sess = _get_session(session_id) + if not sess or sess.get("exited"): + return _serve_transcript_replay(session_id) +``` + +Insert the new replay-only check **between** the `_get_session` call and the `if not sess` check: + +```python + sess = _get_session(session_id) + + # Replay-only sessions (e.g. those created by coda_run) always serve the + # transcript-from-disk, even when the PTY is still alive. + if sess and sess.get("replay_only"): + return _serve_transcript_replay(session_id) + + if not sess or sess.get("exited"): + return _serve_transcript_replay(session_id) +``` + +- [ ] **Step 4: Run the new tests and verify they pass.** + +```bash +.venv/bin/python -m pytest tests/test_replay_only_flag.py -v 2>&1 | tail -10 +``` + +Expected: 4 passed. + +- [ ] **Step 5: Commit.** + +```bash +git add app.py tests/test_replay_only_flag.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: replay_only PTY sessions short-circuit to transcript in attach_session + +Replay-only sessions always serve the on-disk transcript regardless of +whether the PTY is still alive. Used by coda_run (wired in the next commit)." +``` + +--- + +## Task 4: Wire `coda_run` to pass `replay_only=True` + +One-line change in the call to `_app_create_session` (the hook that points to `mcp_create_pty_session`). + +**Files:** +- Modify: `coda_mcp/mcp_server.py` (around line 289 — the `_app_create_session(...)` call inside `coda_run`) +- Modify: `tests/test_replay_only_flag.py` + +- [ ] **Step 1: Add a failing test.** + +Append to `tests/test_replay_only_flag.py`: + +```python +@_pty_skip +def test_coda_run_creates_pty_with_replay_only_true(tmp_path, monkeypatch): + """coda_run must create its PTY with replay_only=True.""" + import asyncio + import json + from app import sessions + from coda_mcp import mcp_server, task_manager + + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + # Stop the watcher from racing the test — we only care about creation here. + monkeypatch.setattr(mcp_server, "_watch_task", lambda *a, **kw: None) + + result_str = asyncio.run(mcp_server.coda_run(prompt="ignored", email="t@example.com")) + result = json.loads(result_str) + pty_id = task_manager._read_session(result["session_id"])["pty_session_id"] + try: + assert sessions[pty_id].get("replay_only") is True + finally: + from app import mcp_close_pty_session + mcp_close_pty_session(pty_id) +``` + +- [ ] **Step 2: Run and verify failure.** + +```bash +.venv/bin/python -m pytest tests/test_replay_only_flag.py::test_coda_run_creates_pty_with_replay_only_true -v 2>&1 | tail -10 +``` + +Expected: FAIL — `assert None is True` (or `assert False is True`) because `coda_run` is not yet passing the flag. + +- [ ] **Step 3: Modify `coda_run` in `coda_mcp/mcp_server.py`.** + +Find the `_app_create_session(...)` call inside `coda_run` (search for `pty_session_id = _app_create_session(`). Currently: + +```python + pty_session_id = _app_create_session( + label="hermes-mcp", + transcript_path=transcript_path, + ) +``` + +Add the new kwarg: + +```python + pty_session_id = _app_create_session( + label="hermes-mcp", + transcript_path=transcript_path, + replay_only=True, # NEW: coda_run URLs are post-hoc review only + ) +``` + +- [ ] **Step 4: Run and verify pass.** + +```bash +.venv/bin/python -m pytest tests/test_replay_only_flag.py -v 2>&1 | tail -10 +``` + +Expected: 5 passed. + +- [ ] **Step 5: Commit.** + +```bash +git add coda_mcp/mcp_server.py tests/test_replay_only_flag.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "feat: coda_run creates PTY sessions with replay_only=True + +Mode 3 in the three-mode framework. The viewer_url returned by coda_run +now always resolves to a transcript-from-disk replay." +``` + +--- + +## Task 5: Switch `_watch_task` to immediate PTY close (pure refactor) + +Replace `_schedule_deferred_close(session_id)` with `_close_pty_immediately(session_id)` in `_watch_task`. Both functions already exist — this is a one-name-for-another swap. **Not a TDD task** — existing tests (specifically `tests/test_mcp_integration.py`, which already calls `_close_pty_immediately`-equivalent paths directly) act as the safety net. The "no timer" behavior is hard to test as a red-green cycle without instrumenting the watcher's polling loop, which isn't worth the complexity here. + +**Files:** +- Modify: `coda_mcp/mcp_server.py` (`_watch_task`, around lines 133 and 160) + +- [ ] **Step 1: Confirm existing safety-net tests pass.** + +```bash +.venv/bin/python -m pytest tests/test_mcp_integration.py tests/test_mcp_server.py -v 2>&1 | tail -10 +``` + +Expected: All pass. These tests cover `_watch_task`'s completion path and `_close_pty_immediately`'s teardown. + +- [ ] **Step 2: Locate the call sites.** + +```bash +grep -n "_schedule_deferred_close" coda_mcp/mcp_server.py +``` + +Expected: 3 matches — one at the function definition (~line 186), two call sites inside `_watch_task` (~lines 133 and 160). You're swapping the two call sites; the definition gets deleted in Task 7. + +- [ ] **Step 3: Swap the calls in `_watch_task`.** + +In `coda_mcp/mcp_server.py`, at each of the **two** call sites inside `_watch_task` (the success branch and the timeout branch), replace: + +```python +# Before: +_schedule_deferred_close(session_id) + +# After: +_close_pty_immediately(session_id) +``` + +Leave the `_schedule_deferred_close` function definition alone for now — it becomes dead code that Task 7 deletes. + +- [ ] **Step 4: Re-run the safety-net tests.** + +```bash +.venv/bin/python -m pytest tests/test_mcp_integration.py tests/test_mcp_server.py -v 2>&1 | tail -10 +``` + +Expected: All pass. Behavior is preserved at the test-observable level (the watcher still drives a teardown after completion); only the timing changes (immediate vs. 5-min deferred), and no current test asserts the 5-min delay (the grace-timing tests use `monkeypatch` to shrink it to milliseconds). + +- [ ] **Step 5: Confirm via grep that `_watch_task` no longer calls `_schedule_deferred_close`.** + +```bash +grep -n "_schedule_deferred_close" coda_mcp/mcp_server.py +``` + +Expected: 1 match (only the function definition itself, which Task 7 will delete). + +- [ ] **Step 6: Commit.** + +```bash +git add coda_mcp/mcp_server.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "refactor: _watch_task uses _close_pty_immediately instead of deferred close + +Pure call-site swap. Behavior change: PTY teardown is immediate rather +than 5-minute-deferred. _schedule_deferred_close becomes dead code, +ripped out in a follow-up commit." +``` + +--- + +## Task 6: Drop dead grace tests + +Now that no production code path calls grace machinery, the tests that exercise it can go. Doing this BEFORE the code rip-out keeps the suite green at every commit. + +**Files:** +- Modify: `tests/test_transcript.py` (delete 4 tests) +- Modify: `tests/test_mcp_server.py` (delete 2 tests + setup/teardown grace lines) +- Modify: `tests/test_mcp_integration.py` (delete 1 test) + +- [ ] **Step 1: Delete grace tests from `tests/test_transcript.py`.** + +Open `tests/test_transcript.py`. Delete these **4 test functions in full** (each is one block from `def` line through to the next blank line / next `def`): + +| Test | Approx line | +|---|---| +| `def test_grace_period_pty_does_not_count_toward_max(monkeypatch):` | 135 | +| `def test_bump_session_last_poll_advances_clock(monkeypatch):` | 157 | +| `def test_mark_grace_on_missing_session_is_noop():` | 169 | +| `def test_bump_session_last_poll_missing_is_noop():` | 174 | + +Re-verify after deletion: + +```bash +grep -n "grace\|_mark_grace\|_bump_session\|GRACE" tests/test_transcript.py +``` + +Expected: no matches. + +- [ ] **Step 2: Delete grace tests from `tests/test_mcp_server.py`.** + +Delete: +- `def test_set_app_hooks_accepts_grace_and_bump_hooks():` (around line 361) +- The function that starts at line ~399 (the `monkeypatch.setattr(mcp_server, "GRACE_PERIOD_S", 0.05)` one — search for `GRACE_PERIOD_S` to find it). + +Also in the setup/teardown fixtures at the top of the file (lines 21-22 and 27-28), remove the lines: + +```python + mcp_server._app_mark_grace = None + mcp_server._app_bump_poll = None +``` + +Verify: + +```bash +grep -n "grace\|mark_grace\|bump_poll\|GRACE" tests/test_mcp_server.py +``` + +Expected: no matches. + +- [ ] **Step 3: Delete the grace E2E test from `tests/test_mcp_integration.py`.** + +Delete the entire `# ── 7. E2E: grace period + transcript replay ────────────────────────` section. Specifically: +- The section header comment at line ~293 +- The full `def test_end_to_end_grace_and_replay(tmp_path, monkeypatch):` function (starts line 315, ends after line ~408) + +Verify: + +```bash +grep -n "grace\|GRACE\|_mark_grace" tests/test_mcp_integration.py +``` + +Expected: no matches. + +- [ ] **Step 4: Run the full suite — must still pass.** + +```bash +.venv/bin/python -m pytest tests/ -x --ignore=tests/e2e -q 2>&1 | tail -20 +``` + +Expected: All remaining tests pass. The grace tests are gone; nothing imports `_mark_grace_for_session` or `GRACE_PERIOD_S` from test code anymore. + +- [ ] **Step 5: Commit.** + +```bash +git add tests/test_transcript.py tests/test_mcp_server.py tests/test_mcp_integration.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "test: drop dead grace-period tests + +Prep for grace-machinery rip-out in follow-up commits. Removes 7 tests +that exercised code paths now superseded by replay_only + immediate close." +``` + +--- + +## Task 7: Rip out grace machinery from `coda_mcp/mcp_server.py` + +Delete `_schedule_deferred_close`, the grace hook slots, and the `GRACE_PERIOD_S` constant. Also clean up `set_app_hooks` and `_close_pty_immediately`'s docstring. + +**Files:** +- Modify: `coda_mcp/mcp_server.py` + +- [ ] **Step 1: Verify nothing in the suite imports the symbols you're about to delete.** + +```bash +grep -rn "_schedule_deferred_close\|_app_mark_grace\|_app_bump_poll\|GRACE_PERIOD_S" coda_mcp/ tests/ app.py +``` + +Expected: Only matches inside `coda_mcp/mcp_server.py`. If any tests still import these, return to Task 6. + +- [ ] **Step 2: Remove the dead module-level state and the function.** + +In `coda_mcp/mcp_server.py`: + +- Delete lines 79-80: `_app_mark_grace = None` and `_app_bump_poll = None` +- Delete line 82: `GRACE_PERIOD_S = 300 # 5 minutes` +- Delete the entire `_schedule_deferred_close` function (lines ~186-213). Search for `def _schedule_deferred_close` and delete from that line through the function's closing line. + +- [ ] **Step 3: Update `set_app_hooks` signature.** + +Find `def set_app_hooks(` (around line 85). Currently it accepts `mark_grace_fn` and `bump_poll_fn` parameters. Remove those parameters from the signature, and remove the lines inside the function body that assign them to the module-level slots (`_app_mark_grace = mark_grace_fn`, `_app_bump_poll = bump_poll_fn`). + +Also update the function's docstring — search for the line that mentions "defer PTY close by ``GRACE_PERIOD_S``" and rewrite the docstring to remove grace references entirely. + +- [ ] **Step 4: Update `_close_pty_immediately` docstring.** + +Find `def _close_pty_immediately(` (around line 167). Its docstring currently says it's for "emergency teardown or tests". Rewrite to reflect that it's the normal close path: + +```python +def _close_pty_immediately(session_id: str) -> None: + """Close the PTY session associated with this task session immediately. + + Called by ``_watch_task`` as soon as the task transitions to completed + or failed. Reads ``pty_session_id`` from the task-manager's session.json + and calls the ``_app_close_session`` hook (i.e. ``mcp_close_pty_session`` + in production). + """ +``` + +- [ ] **Step 5: Update the module-level docstring.** + +At the top of `coda_mcp/mcp_server.py`, find the line that mentions hooks (around line 9: "handled through optional app hooks set via ``set_app_hooks()``."). Make sure it doesn't claim grace functionality. Search for any other comment block referencing grace and remove. + +- [ ] **Step 6: Run the suite.** + +```bash +.venv/bin/python -m pytest tests/ -x --ignore=tests/e2e -q 2>&1 | tail -15 +``` + +Expected: All pass. + +- [ ] **Step 7: Commit.** + +```bash +git add coda_mcp/mcp_server.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "refactor: rip out grace-period machinery from coda_mcp/mcp_server.py + +Removes _schedule_deferred_close, GRACE_PERIOD_S, the unused grace hook +slots, and the corresponding set_app_hooks parameters. The grace hooks +were never wired in production — this is dead code removal, not a +behavior change." +``` + +--- + +## Task 8: Rip out grace machinery from `app.py` + +Delete `_mark_grace_for_session`, `_bump_session_last_poll`, the `grace` key from the session dict creation, and the `MAX_CONCURRENT_SESSIONS` exclusion at all 4 sites. + +**Files:** +- Modify: `app.py` + +- [ ] **Step 1: Remove the `"grace": False,` key from session dict creation in `mcp_create_pty_session`.** + +In `app.py`, find the dict literal in `mcp_create_pty_session` that contains `"grace": False,` (around line 1477). Delete that single line. The `replay_only` line you added in Task 1 stays. + +There may be ANOTHER similar `"grace": False,` line in the other session-creation path inside `create_session` (search the file for `"grace": False,` — there may be 2 occurrences). Delete both. + +```bash +grep -n '"grace"' app.py +``` + +Expected after deletion: no matches. + +- [ ] **Step 2: Revert the `MAX_CONCURRENT_SESSIONS` exclusion at 4 sites.** + +Search for `sum(1 for s in sessions.values() if not s.get("grace"))`: + +```bash +grep -n "if not s.get(\"grace\")" app.py +``` + +Expected: 4 matches at lines around 1329, 1369, 1405, 1456. + +**CRITICAL — locking note:** All 4 sites are **already** inside a `with sessions_lock:` block (the lock is acquired by the surrounding session-creation code immediately before the check). `sessions_lock` is `threading.Lock()` (not `RLock`), so **do NOT** wrap the replacement in another `with sessions_lock:` — that will deadlock. Just use `len(sessions)` directly. + +At each of the 4 sites, replace: + +```python +# Before (inside an existing `with sessions_lock:` block): +active = sum(1 for s in sessions.values() if not s.get("grace")) +if active >= MAX_CONCURRENT_SESSIONS: + ... +``` + +With: + +```python +# After (still inside the same `with sessions_lock:` block — no new lock): +active = len(sessions) +if active >= MAX_CONCURRENT_SESSIONS: + ... +``` + +To verify each site really is inside a lock block, read the ~5 lines preceding each `sum(...)` call. You should see `with sessions_lock:` at lines 1328, 1366 (for site 1369), 1404 (for site 1405), and 1455 (for site 1456). If any site is somehow NOT already locked, stop and ask before proceeding — the original code may have a latent bug worth investigating. + +- [ ] **Step 3: Delete `_mark_grace_for_session` and `_bump_session_last_poll`.** + +Find both functions (around lines 1515 and 1530). Delete each function definition in full. + +- [ ] **Step 4: Verify no stale references.** + +```bash +grep -n "grace\|_mark_grace\|_bump_session_last_poll" app.py +``` + +Expected: no matches (or only comment lines that reference history — delete those too). + +- [ ] **Step 5: Run the suite.** + +```bash +.venv/bin/python -m pytest tests/ -x --ignore=tests/e2e -q 2>&1 | tail -15 +``` + +Expected: All pass. + +- [ ] **Step 6: Commit.** + +```bash +git add app.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "refactor: rip out grace-period machinery from app.py + +Removes _mark_grace_for_session, _bump_session_last_poll, the 'grace' +key on session dicts, and the MAX_CONCURRENT_SESSIONS exclusion at all +4 check sites. Grace was never wired through set_app_hooks in prod, so +this removes dead code." +``` + +--- + +## Task 9: Update MCP `instructions` string + check `mcp_asgi.py` cleanup + +The FastMCP `instructions` string at `mcp_server.py:61-66` currently tells callers to "SHARE THE LIVE URL" and "watch progress". With replay-only semantics, that text is wrong. + +**Files:** +- Modify: `coda_mcp/mcp_server.py` +- Spot-check: `coda_mcp/mcp_asgi.py` + +- [ ] **Step 1: Locate the instructions string.** + +```bash +grep -n "SHARE THE LIVE URL\|watch progress\|live URL" coda_mcp/mcp_server.py +``` + +Expected: matches near the `FastMCP(...)` instantiation block (around lines 61-66). + +- [ ] **Step 2: Rewrite the relevant paragraph.** + +In `coda_mcp/mcp_server.py`, find the paragraph that starts "SHARE THE LIVE URL" (or whatever the exact phrasing is at lines 61-66). Replace it with: + +``` +SHARE THE REPLAY URL: After calling coda_run, you receive a ``viewer_url`` +in the response. Pass this URL to your user so they can open it in a browser +to review the agent's transcript — what was prompted, what was reasoned, what +was produced. The URL is read-only and serves a static replay of the session, +so it remains valid indefinitely after the task completes. +``` + +(Exact wording may need adjustment to match the surrounding paragraph style — read the surrounding text first.) + +- [ ] **Step 3: Spot-check `mcp_asgi.py`.** + +```bash +grep -n "set_app_hooks\|grace\|mark_grace\|bump_poll" coda_mcp/mcp_asgi.py +``` + +Expected: a `set_app_hooks(...)` call exists but does **not** pass grace-related kwargs (per critic's finding). No changes needed. If grace kwargs ARE passed (shouldn't be, but verify), remove them. + +- [ ] **Step 4: Verify nothing relies on the old text.** + +```bash +grep -rn "watch progress\|live URL\|LIVE URL" docs/ tests/ static/ +``` + +Expected: matches only in historical documents (specs/plans from prior PRs). No live code depends on the old phrasing. + +- [ ] **Step 5: Run the suite.** + +```bash +.venv/bin/python -m pytest tests/ -x --ignore=tests/e2e -q 2>&1 | tail -15 +``` + +Expected: All pass. + +- [ ] **Step 6: Commit.** + +```bash +git add coda_mcp/mcp_server.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "docs: update MCP instructions string for replay-only viewer_url semantics + +The viewer_url returned by coda_run is no longer a live attach — it is +a static replay. Update the FastMCP instructions text accordingly so +MCP clients describe it correctly to end users." +``` + +--- + +## Task 10: Update / rewrite `test_replay_attach.py` for the new contract + +After the rip-out, `test_replay_attach.py` may pass without changes (the helper extraction and replay-only flag don't break its existing assertions). But the two tests in it should now make the stronger assertion: replay works regardless of PTY state, not just after the PTY has exited. + +**Files:** +- Modify: `tests/test_replay_attach.py` + +- [ ] **Step 1: Read the current contents.** + +```bash +cat tests/test_replay_attach.py +``` + +- [ ] **Step 2: Run the file as-is to confirm green starting point.** + +```bash +.venv/bin/python -m pytest tests/test_replay_attach.py -v 2>&1 | tail -10 +``` + +Expected: 2 passed. + +- [ ] **Step 3: Strengthen the assertions.** + +The existing tests likely create a transcript file and an exited PTY, then assert that attach returns replay. Add a third test that uses a `replay_only=True` PTY which is STILL ALIVE and asserts the same — confirming the new short-circuit. + +**Important:** This test allocates a real PTY (via `mcp_create_pty_session`), so it needs the same `_pty_skip` guard pattern used in `tests/test_replay_only_flag.py`. Add the guard at the top of the file if it isn't there already (next to the existing imports). + +At the top of `tests/test_replay_attach.py`, if not already present, add: + +```python +import os as _os +import pytest as _pytest + +try: + import pty as _pty + _master, _slave = _pty.openpty() + _os.close(_master) + _os.close(_slave) + _PTY_AVAILABLE = True +except Exception: + _PTY_AVAILABLE = False + +_pty_skip = _pytest.mark.skipif( + not _PTY_AVAILABLE, + reason="PTY not allocatable in this environment", +) +``` + +Then add to the end of `tests/test_replay_attach.py`: + +```python +@_pty_skip +def test_attach_session_returns_replay_for_alive_replay_only_pty(tmp_path, monkeypatch): + """A coda_run-style PTY (replay_only=True) that is still alive serves the transcript. + + This is the new contract introduced by the replay-only flag — historically + a live PTY would serve its output_buffer. + """ + import os + from app import app as flask_app, mcp_create_pty_session, mcp_close_pty_session + from coda_mcp import task_manager + + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + + sid = mcp_create_pty_session(label="replay-alive", replay_only=True) + try: + sess_id = "sess-x" + task_id = "task-x" + sdir = tmp_path / sess_id + tdir = sdir / "tasks" / task_id + tdir.mkdir(parents=True) + (sdir / "session.json").write_text( + '{"session_id": "%s", "pty_session_id": "%s"}' % (sess_id, sid) + ) + (tdir / "transcript.log").write_bytes(b"FROM DISK") + # Cache may have stale entries from earlier tests — clear before the lookup. + task_manager._pty_lookup_cache.clear() + + client = flask_app.test_client() + resp = client.post("/api/session/attach", json={"session_id": sid}) + assert resp.status_code == 200 + body = resp.get_json() + assert body["replay"] is True + assert body["output"] == ["FROM DISK"] + finally: + mcp_close_pty_session(sid) +``` + +- [ ] **Step 4: Run.** + +```bash +.venv/bin/python -m pytest tests/test_replay_attach.py -v 2>&1 | tail -10 +``` + +Expected: 3 passed. + +- [ ] **Step 5: Commit.** + +```bash +git add tests/test_replay_attach.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "test: extend test_replay_attach.py for alive-PTY replay_only case + +Confirms the new contract: replay-only sessions always serve the +transcript-from-disk, even when the PTY is still alive." +``` + +--- + +## Task 11: Add regression-guard test + +Prevent future drift that accidentally re-introduces `grace` on the `coda_run` path. + +**Files:** +- Modify: `tests/test_replay_only_flag.py` + +- [ ] **Step 1: Append the regression test.** + +Append to `tests/test_replay_only_flag.py`: + +```python +@_pty_skip +def test_no_grace_key_in_coda_run_session_dict(): + """Regression guard: coda_run-created PTYs must not have a 'grace' key, + and mcp_create_pty_session must not accept a 'grace' kwarg. + + Protects against accidental re-introduction of grace-period machinery + in future changes. + """ + import inspect + from app import mcp_create_pty_session, mcp_close_pty_session, sessions + + # The function signature must not include 'grace'. + sig = inspect.signature(mcp_create_pty_session) + assert "grace" not in sig.parameters, ( + f"mcp_create_pty_session should not accept a 'grace' parameter " + f"(found in signature: {list(sig.parameters)})" + ) + + # And the session dict must not contain a 'grace' key. + sid = mcp_create_pty_session(label="t-no-grace", replay_only=True) + try: + assert "grace" not in sessions[sid], ( + f"session dict should not contain a 'grace' key " + f"(found: {list(sessions[sid].keys())})" + ) + finally: + mcp_close_pty_session(sid) +``` + +- [ ] **Step 2: Run.** + +```bash +.venv/bin/python -m pytest tests/test_replay_only_flag.py -v 2>&1 | tail -15 +``` + +Expected: 7 passed (the previous 6 + this regression-guard). + +- [ ] **Step 3: Run the full suite one final time.** + +```bash +.venv/bin/python -m pytest tests/ -x --ignore=tests/e2e -q 2>&1 | tail -15 +``` + +Expected: Around 525 passed + ~11 skipped (PTY-gated). Net change from baseline: -2 tests. + +- [ ] **Step 4: Commit.** + +```bash +git add tests/test_replay_only_flag.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" commit -m "test: regression guard against re-introduction of grace key + +Asserts mcp_create_pty_session does not accept a 'grace' kwarg and that +coda_run-created session dicts contain no 'grace' key. Catches drift +if a future change tries to bring the grace machinery back." +``` + +--- + +## Final verification (post-task) + +- [ ] **F1: Full suite green.** + +```bash +.venv/bin/python -m pytest tests/ --ignore=tests/e2e -q 2>&1 | tail -10 +``` + +Expected: all pass. + +- [ ] **F2: `grep` confirms no stale references.** + +```bash +grep -rn "grace\|GRACE_PERIOD\|_mark_grace\|_bump_session_last_poll\|_schedule_deferred_close" coda_mcp/ app.py 2>&1 | grep -v ".pyc\|.git" +``` + +Expected: no matches (or only matches in comments that document the removal — those are fine). + +- [ ] **F3: Manual smoke (optional, requires deployed environment).** + +1. Restart the app (`uvicorn coda_mcp.mcp_asgi:app`). +2. Trigger a `coda_run` from an MCP client. Capture the `viewer_url`. +3. Open the URL in a browser **while** hermes is still running. Confirm: read-only replay UI, no terminal input box. +4. Wait for hermes to complete (~30s). Confirm: PTY is gone from `/health` (`active_sessions` returns to baseline). +5. Re-open the URL. Confirm: same read-only replay, full final transcript. + +--- + +## Self-review checklist (run on completed plan) + +1. **Spec coverage** ✓ + - Section "Add replay_only flag" → Task 1 + - Section "Enforce replay-only" → Tasks 2 (extract) + 3 (enforce) + - Section "Wire coda_run" → Task 4 + - Section "Rip out grace machinery" → Tasks 6 (tests) + 7 (mcp_server.py) + 8 (app.py) + - Section "Watcher teardown on completion" → Task 5 + - "Docstrings to update" → Tasks 7 (docstring inside) + 9 (MCP instructions) + - "Regression guard" → Task 11 + +2. **Placeholders** ✓ — every step has concrete code/commands. No TBDs. + +3. **Type consistency** ✓ + - `replay_only: bool = False` used identically in signature, dict, and tests + - `_close_pty_immediately(session_id: str) -> None` — task-manager session_id, not pty_session_id (the function takes the task session ID and looks up the PTY internally) + - `_serve_transcript_replay(session_id)` — pty_session_id (passed straight through to `find_task_dir_by_pty_session`) + +4. **Ordering safety** ✓ + - Tests dropped (Task 6) BEFORE code rip-out (Tasks 7, 8) → suite stays green + - `_watch_task` swap (Task 5) BEFORE `_schedule_deferred_close` deletion (Task 7) → no orphan calls + - `replay_only` storage (Task 1) BEFORE attach short-circuit (Task 3) → flag exists before being read + +--- + +## Plan critique gate + +**Cleared** (2026-05-28). Critic verdict: APPROVE WITH CHANGES. Issues found and resolved: + +1. **CRITICAL — locking deadlock in Task 8 Step 2.** Original instruction wrapped the replacement code in `with sessions_lock:`, but all 4 MAX_CONCURRENT sites are already inside `with sessions_lock:` blocks. `sessions_lock` is a non-reentrant `threading.Lock()`, so the wrap would deadlock the server. Fixed: Task 8 Step 2 now explicitly says "do NOT wrap" and replaces the code with bare `active = len(sessions)`. + +2. **MAJOR — TDD violation in Task 5.** Original task tried to wrap the `_watch_task` swap in a red-green cycle, but the test ended up passing on first run (it called `_close_pty_immediately` directly, not through `_watch_task`). Fixed: Task 5 relabeled as a non-TDD refactor with existing integration tests as the safety net, in the same style as Task 2. + +3. **MAJOR — missing `_pty_skip` in Task 10 test.** New test in `test_replay_attach.py` allocates a real PTY but didn't carry the PTY-skip guard, so it would error on CI environments without `pty.openpty()`. Fixed: Task 10 now adds the guard pattern at the file top and decorates the new test with `@_pty_skip`. + +4. **MINOR — vague test names in Task 6 Step 1.** Original named 2 of 4 grace tests to delete and said "plus two more". Fixed: all 4 tests now named explicitly in a table. + +Per-dimension verdicts from the critic: +- **Spec coverage**: Complete (all spec sections map to ≥1 task) +- **Task atomicity & ordering**: Sound — green at every commit boundary +- **TDD discipline**: Clean after Task 5 relabel (Tasks 1, 3, 4, 11 do genuine red-green; Tasks 2, 5 are pure refactors with safety-net tests) +- **Line-number accuracy**: Verified exact at every reference (no drift) +- **Test-code correctness**: All fixtures/imports/decorators verified after fixes +- **Concurrency**: Safe after Task 8 lock-wrap fix +- **Commit messages**: Conventional-commits format with `-c user.email=datasciencemonkey@gmail.com` override — correct + +Plan is ready for execution. diff --git a/docs/superpowers/plans/2026-05-28-coda-run-workflow-protocol.md b/docs/superpowers/plans/2026-05-28-coda-run-workflow-protocol.md new file mode 100644 index 0000000..60a29f6 --- /dev/null +++ b/docs/superpowers/plans/2026-05-28-coda-run-workflow-protocol.md @@ -0,0 +1,1156 @@ +# `coda_run` Workflow Protocol Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Inject a Databricks orientation block (CAPABILITIES) and a structured 3-phase workflow protocol (PLAN → EXECUTE → SYNTHESIZE with critique at each phase) into every `coda_run` task's `prompt.txt`. Add a third terminal `result.json` status `"info_needed"` with a required `feedback` field so the calling client can iterate when the agent is blocked. Update `coda_inbox`, `coda_get_result`, and the MCP `instructions` block to know about the new status and its `needs_approval` sibling. + +**Architecture:** +- Pure-function module `coda_mcp/databricks_preamble.py` produces the two new prompt sections (CAPABILITIES, WORKFLOW PROTOCOL). One source of truth for the skill list. Trivially unit-testable. +- `task_manager.wrap_prompt()` gains a `workflow_protocol: bool = True` parameter. When true, inserts the two sections between TASK and INSTRUCTIONS, and updates INSTRUCTIONS to describe new step labels and the `info_needed` status. The flag flows from `coda_run` through `create_task` to `wrap_prompt` — three call sites, one parameter. +- Inbox / result surfaces (`coda_inbox` counts dict, `coda_get_result` docstring, the FastMCP `instructions=` block at server construction) are updated to tolerate and surface the new statuses (`info_needed`, `needs_approval`). +- Tests pin the prompt sections verbatim where it matters, pin the skill list against CLAUDE.md, and guard the new counts-dict keys and docstring content. + +**Tech Stack:** Python 3.11, pytest, MagicMock, FastMCP. No new dependencies. + +--- + +## Files modified by this plan + +- **Create:** `coda_mcp/databricks_preamble.py` — new module, three exports +- **Create:** `tests/test_databricks_preamble.py` — unit tests for the new module +- **Modify:** `coda_mcp/task_manager.py:153-225` — `wrap_prompt` signature, body, INSTRUCTIONS section text +- **Modify:** `coda_mcp/task_manager.py:231-...` — `create_task` signature + forwarding +- **Modify:** `coda_mcp/mcp_server.py:52-99` — FastMCP `instructions=` block (add INFO_NEEDED HANDOFF paragraph) +- **Modify:** `coda_mcp/mcp_server.py:220-227` — `coda_run` signature + forwarding +- **Modify:** `coda_mcp/mcp_server.py:551-559` — `coda_inbox` counts dict +- **Modify:** `coda_mcp/mcp_server.py:573-584` — `coda_get_result` docstring +- **Create:** `tests/test_inbox_status_passthrough.py` — counts dict + docstring + MCP instructions tests + +## Pre-flight context + +- Worktree: `/Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp` +- Branch: `feat/coda-mcp-interactive-handoff` (PR #67, in-flight — this lands as follow-up commits) +- Run tests with `uv run pytest` (per user's `always use uv` directive) +- Commit identity: `-c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty"`. No AI/Claude co-author lines. +- The full spec is `docs/superpowers/specs/2026-05-28-coda-run-workflow-protocol-design.md` — consult for full text of CAPABILITIES, WORKFLOW PROTOCOL, and DISAMBIGUATION sections. +- Skill list source of truth: the "Databricks Skills" markdown table in the project-level `CLAUDE.md` at the repo root (`/Users/sathish.gangichetty/Documents/xterm-experiment/.worktrees/coda-mcp/CLAUDE.md`). + +--- + +## Task 1: Create `databricks_preamble.py` module and unit tests (TDD) + +This task creates a new module with three pure functions and exhaustive tests. New module → tests and implementation land in the same commit (the module doesn't exist for the tests to fail against prior to the commit, so "RED-then-GREEN-in-one-commit" is the right shape here). + +**Files:** +- Create: `coda_mcp/databricks_preamble.py` +- Create: `tests/test_databricks_preamble.py` + +- [ ] **Step 1: Write the new module `coda_mcp/databricks_preamble.py`** + +Create the file with this exact content: + +```python +"""Builders for the CoDA prompt envelope's CAPABILITIES and WORKFLOW PROTOCOL sections. + +These are injected into prompt.txt by ``task_manager.wrap_prompt`` when +``workflow_protocol=True``. Pure functions — no side effects, no I/O. +""" +from __future__ import annotations + + +_DATABRICKS_SKILLS: tuple[str, ...] = ( + "agent-bricks", + "databricks-genie", + "databricks-app-python", + "databricks-app-apx", + "databricks-jobs", + "databricks-unity-catalog", + "spark-declarative-pipelines", + "aibi-dashboards", + "model-serving", + "mlflow-evaluation", + "asset-bundles", + "databricks-python-sdk", + "databricks-config", + "databricks-docs", + "synthetic-data-generation", + "unstructured-pdf-generation", +) + + +def get_databricks_skills() -> tuple[str, ...]: + """Return the canonical Databricks skill list. Tests pin this against CLAUDE.md.""" + return _DATABRICKS_SKILLS + + +def build_capabilities() -> str: + """Orientation block: CLI, skills, MCP servers, when to prefer Databricks-native paths.""" + skills_lines = [] + # Pack 4 skills per line for readability in prompt.txt. + for i in range(0, len(_DATABRICKS_SKILLS), 4): + chunk = _DATABRICKS_SKILLS[i:i + 4] + skills_lines.append("- " + ", ".join(chunk)) + skills_block = "\n".join(skills_lines) + return ( + "You are running inside CoDA on a Databricks-authenticated host.\n" + "\n" + "Databricks CLI: pre-configured. `databricks current-user me` confirms auth.\n" + "Use it for jobs, workspace, clusters, warehouses, Unity Catalog operations.\n" + "\n" + "Skills available at ~/.claude/skills/ — read each skill's SKILL.md before\n" + "invoking. Relevant Databricks skills:\n" + f"{skills_block}\n" + "\n" + "MCP servers wired:\n" + "- DeepWiki — ask_question, read_wiki_contents for any GitHub repo\n" + "- Exa — web_search_exa, web_fetch_exa for live web context\n" + "- CoDA — chain follow-up tasks via previous_session_id\n" + "\n" + "When the task touches Databricks data, pipelines, jobs, dashboards, agents,\n" + "or model serving, DEFAULT to the skill / CLI / SDK path above instead of\n" + "generic Python or web search." + ) + + +def build_workflow_protocol() -> str: + """3-phase workflow with critique at each phase + info_needed escape hatch.""" + return ( + "You MUST process this task in three phases. Emit status.jsonl events as\n" + "you go (one JSON object per line, format below).\n" + "\n" + "PHASE 1 — PLAN\n" + "- Write a step-by-step plan as a status.jsonl line with step=\"plan\" and\n" + " message containing the numbered steps.\n" + "- Then critique your own plan as if you were a separate reviewer.\n" + " (Spawn a sub-agent for the critique if your agent supports it; otherwise\n" + " write the critique inline as a self-review.) Emit step=\"critique_plan\"\n" + " with the verdict (APPROVE / BLOCK / APPROVE-WITH-FIXES) and findings.\n" + "- If the critique surfaces blockers, revise the plan once and re-emit\n" + " step=\"plan\". Maximum 2 plan iterations total.\n" + "- If after 2 attempts you still cannot produce a viable plan, write\n" + " result.json with status=\"info_needed\" (see below) and stop.\n" + "\n" + "PHASE 2 — EXECUTE\n" + "- Work the plan. Emit step=\"execute_\" lines after completing each plan\n" + " step (n is 1-indexed, matches the plan's numbering).\n" + "- After execution, emit step=\"critique_execute\" with a review of what got\n" + " built vs what the plan said. APPROVE / BLOCK / APPROVE-WITH-FIXES.\n" + "- If the critique surfaces correctness or scope gaps, fix them and re-emit\n" + " step=\"critique_execute\". Maximum 2 execute iterations total.\n" + "- If you hit a hard blocker (missing access, missing data, ambiguous\n" + " requirements that the plan revealed only mid-execution), write\n" + " result.json with status=\"info_needed\" and stop.\n" + "\n" + "PHASE 3 — SYNTHESIZE\n" + "- Write result.json with status=\"completed\".\n" + "- Emit step=\"critique_synthesize\" with a review of the result against the\n" + " original TASK.\n" + "- If the critique surfaces gaps, revise result.json. Maximum 2 synthesis\n" + " iterations total.\n" + "\n" + "If at any phase you cannot proceed, use the INFO_NEEDED escape hatch:\n" + "- Set status=\"info_needed\" in result.json.\n" + "- Set \"feedback\" to a precise, actionable string naming exactly what is\n" + " missing (a table name, a decision, an access grant, a clarification).\n" + " The calling client will read this and resubmit with the missing context.\n" + "- \"info_needed\" is NOT a failure — it is a structured request for\n" + " iteration. Use it whenever you would otherwise have to guess.\n" + "\n" + "If you encounter a hard, unrecoverable failure (a command crashed, an SDK\n" + "returned 500, a file is corrupt), use status=\"failed\" with a description\n" + "in \"errors\".\n" + "\n" + "DISAMBIGUATION — two soft statuses already exist and they mean different\n" + "things; use the right one:\n" + "- \"info_needed\" — the CALLER must add missing context (table name,\n" + " business decision, file contents, access grant) before the task can\n" + " proceed. Used when ambiguity or missing input blocks you.\n" + "- \"needs_approval\" — you have a concrete plan to do something destructive\n" + " (drop a table, delete a job, modify permissions). You will execute it\n" + " if and only if the caller explicitly approves. Used at the SAFETY\n" + " boundary, never for ambiguity. See SAFETY section below.\n" + "\n" + "If both apply (e.g. \"I'd drop a table but I'm not sure which one\"), prefer\n" + "\"info_needed\" — resolving the ambiguity first is cheaper than approving\n" + "the wrong destructive action." + ) +``` + +- [ ] **Step 2: Write `tests/test_databricks_preamble.py`** + +Create the file with this exact content: + +```python +"""Unit tests for coda_mcp.databricks_preamble.""" +import re + +from coda_mcp.databricks_preamble import ( + build_capabilities, + build_workflow_protocol, + get_databricks_skills, +) + + +def test_get_databricks_skills_returns_exactly_sixteen(): + skills = get_databricks_skills() + assert isinstance(skills, tuple) + assert len(skills) == 16, f"Expected 16 skills, got {len(skills)}: {skills}" + + +def test_skills_list_matches_claude_md(): + """The hardcoded skill tuple must match the Databricks Skills table in CLAUDE.md. + + Drift in either direction (added to tuple but not docs, or vice versa) fails + this test. The test is the canary that forces both sources to stay in sync. + """ + import os + repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + claude_md = os.path.join(repo_root, "CLAUDE.md") + with open(claude_md, "r") as f: + text = f.read() + # Find the Databricks Skills section. Names are comma-separated within table cells. + section_match = re.search( + r"###\s+Databricks Skills.*?(?=\n###|\n##|\Z)", + text, re.DOTALL, + ) + assert section_match, "Could not find 'Databricks Skills' section in CLAUDE.md" + section = section_match.group(0) + # Extract skill names: kebab-case tokens that follow a list pattern. Be loose — + # accept anything that looks like a skill identifier inside table cells. + skill_names_in_md = set(re.findall(r"\b([a-z][a-z0-9-]{2,}(?:-[a-z0-9]+)+)\b", section)) + skills_in_code = set(get_databricks_skills()) + # Every skill in code must appear in CLAUDE.md. + missing_from_md = skills_in_code - skill_names_in_md + assert not missing_from_md, ( + f"Skills in code but NOT in CLAUDE.md (update CLAUDE.md): {missing_from_md}" + ) + # Every skill in CLAUDE.md's Databricks section must appear in code. + # Filter out section/category words that match the regex but aren't skill names. + section_noise = { + "ai-agents", "data-engineering", # category labels, hyphenated + } + missing_from_code = (skill_names_in_md - skills_in_code) - section_noise + assert not missing_from_code, ( + f"Skills in CLAUDE.md but NOT in code (update databricks_preamble.py): " + f"{missing_from_code}" + ) + + +def test_capabilities_mentions_cli(): + text = build_capabilities() + assert "Databricks CLI" in text + assert "databricks current-user me" in text + + +def test_capabilities_lists_at_least_ten_skills(): + text = build_capabilities() + skills = get_databricks_skills() + hits = sum(1 for s in skills if s in text) + assert hits >= 10, f"Expected at least 10 skills in CAPABILITIES, found {hits}" + + +def test_capabilities_mentions_all_three_mcp_servers(): + text = build_capabilities() + assert "DeepWiki" in text + assert "Exa" in text + assert "CoDA" in text + + +def test_capabilities_under_token_budget(): + text = build_capabilities() + # ~4 chars/token rough lower bound. 1600 chars ≈ 400 tokens budget. + assert len(text) < 1600, ( + f"CAPABILITIES is {len(text)} chars (~{len(text)//4} tokens); budget is 1600." + ) + + +def test_workflow_protocol_lists_three_phases(): + text = build_workflow_protocol() + assert "PHASE 1 — PLAN" in text + assert "PHASE 2 — EXECUTE" in text + assert "PHASE 3 — SYNTHESIZE" in text + + +def test_workflow_protocol_caps_iterations_at_two(): + text = build_workflow_protocol() + # The string "Maximum 2" should appear once per phase = 3 times. + count = text.count("Maximum 2") + assert count == 3, f"Expected 'Maximum 2' to appear 3 times (once per phase); got {count}" + + +def test_workflow_protocol_describes_info_needed(): + text = build_workflow_protocol() + assert "info_needed" in text + assert "feedback" in text + + +def test_workflow_protocol_disambiguates_needs_approval(): + text = build_workflow_protocol() + assert "needs_approval" in text + assert "DISAMBIGUATION" in text + + +def test_workflow_protocol_under_token_budget(): + text = build_workflow_protocol() + # ~4 chars/token. 3200 chars ≈ 800 tokens budget. + assert len(text) < 3200, ( + f"WORKFLOW PROTOCOL is {len(text)} chars (~{len(text)//4} tokens); budget is 3200." + ) +``` + +- [ ] **Step 3: Run the test file to verify everything passes** + +Run: `uv run pytest tests/test_databricks_preamble.py -v` +Expected: 11 passed. + +If a test fails, fix the module (NOT the test) — the test pins the spec. + +The one possible test that needs adjustment: `test_skills_list_matches_claude_md` reads CLAUDE.md and parses its Databricks Skills section. The regex pattern is loose; if it picks up false-positives (e.g. category labels that contain hyphens), add them to `section_noise`. Don't loosen the assertion itself. + +- [ ] **Step 4: Run ruff check** + +Run: `uv run ruff check coda_mcp/databricks_preamble.py tests/test_databricks_preamble.py` +Expected: All checks passed. + +- [ ] **Step 5: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add coda_mcp/databricks_preamble.py tests/test_databricks_preamble.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat: add databricks_preamble module — CAPABILITIES + WORKFLOW PROTOCOL builders + +Two pure-function builders for the new prompt envelope sections plus the +canonical Databricks skill list. Tests pin the skill list against CLAUDE.md +to catch drift in either direction, and pin both sections to token budgets." +``` + +--- + +## Task 2: Wire `workflow_protocol` flag through wrap_prompt → create_task → coda_run (TDD) + +A single flag, three call sites. TDD: write the tests against the desired flow, watch them fail, then wire the flag. + +**Files:** +- Modify: `coda_mcp/task_manager.py:153-225` (`wrap_prompt` — signature + body) +- Modify: `coda_mcp/task_manager.py:231-...` (`create_task` — signature + forward) +- Modify: `coda_mcp/mcp_server.py:220-227` (`coda_run` — signature + forward) +- Modify (or create): `tests/test_task_manager.py` (extend if exists; create otherwise) + +- [ ] **Step 1: Check whether `tests/test_task_manager.py` already exists** + +Run: `ls -la tests/test_task_manager.py 2>&1 || echo "MISSING"` + +If it exists, you'll append tests. If it doesn't, you'll create it. + +- [ ] **Step 2: Append (or create with) these tests for the flag wiring** + +Add these tests to `tests/test_task_manager.py` (create the file if missing — start with `"""Tests for coda_mcp.task_manager."""` plus imports). + +```python +def test_wrap_prompt_default_includes_capabilities_and_workflow(): + """Default workflow_protocol=True; rendered prompt contains both new sections.""" + from coda_mcp.task_manager import wrap_prompt + + out = wrap_prompt( + task_id="t-1", + session_id="s-1", + email="user@example.com", + prompt="do the thing", + context=None, + results_dir="/tmp/results", + ) + assert "CAPABILITIES:" in out + assert "WORKFLOW PROTOCOL:" in out + # Sanity: still has the existing structure. + assert "TASK:" in out + assert "INSTRUCTIONS:" in out + assert "SAFETY:" in out + + +def test_wrap_prompt_workflow_protocol_false_omits_sections(): + """With workflow_protocol=False, both new sections are absent.""" + from coda_mcp.task_manager import wrap_prompt + + out = wrap_prompt( + task_id="t-1", + session_id="s-1", + email="user@example.com", + prompt="do the thing", + context=None, + results_dir="/tmp/results", + workflow_protocol=False, + ) + assert "CAPABILITIES:" not in out + assert "WORKFLOW PROTOCOL:" not in out + # Existing sections are still present. + assert "TASK:" in out + assert "INSTRUCTIONS:" in out + + +def test_wrap_prompt_workflow_protocol_default_is_true(): + """Signature inspection: default value of workflow_protocol is True.""" + import inspect + from coda_mcp.task_manager import wrap_prompt + + sig = inspect.signature(wrap_prompt) + assert "workflow_protocol" in sig.parameters + assert sig.parameters["workflow_protocol"].default is True + + +def test_create_task_signature_has_workflow_protocol_param(): + """create_task accepts workflow_protocol kwarg with default True.""" + import inspect + from coda_mcp.task_manager import create_task + + sig = inspect.signature(create_task) + assert "workflow_protocol" in sig.parameters + assert sig.parameters["workflow_protocol"].default is True + + +def test_create_task_forwards_workflow_protocol_to_wrap_prompt(monkeypatch, tmp_path): + """create_task must pass workflow_protocol through to wrap_prompt.""" + from coda_mcp import task_manager + + captured: dict = {} + + def fake_wrap_prompt(**kwargs): + captured.update(kwargs) + return "DUMMY PROMPT" + + monkeypatch.setattr(task_manager, "wrap_prompt", fake_wrap_prompt) + monkeypatch.setattr(task_manager, "_session_dir", lambda sid: str(tmp_path)) + monkeypatch.setattr(task_manager, "_task_dir", lambda sid, tid: str(tmp_path)) + # _write_json is the real helper used inside create_task (writes meta.json + session file). + # Stub it out — we're testing flag pass-through, not filesystem behavior. + monkeypatch.setattr(task_manager, "_write_json", lambda *a, **kw: None) + monkeypatch.setattr(task_manager.os, "makedirs", lambda *a, **kw: None) + # Stub the file-open for prompt.txt write. + real_open = open + def fake_open(path, mode="r", *args, **kwargs): + if "prompt.txt" in str(path) and "w" in mode: + import io + return io.StringIO() + return real_open(path, mode, *args, **kwargs) + monkeypatch.setattr("builtins.open", fake_open) + + task_manager.create_task( + session_id="s-1", + prompt="x", + email="u@example.com", + workflow_protocol=False, + ) + assert captured.get("workflow_protocol") is False + + +def test_coda_run_signature_has_workflow_protocol_param(): + """coda_run accepts workflow_protocol kwarg with default True.""" + import inspect + from coda_mcp import mcp_server + + sig = inspect.signature(mcp_server.coda_run) + assert "workflow_protocol" in sig.parameters + assert sig.parameters["workflow_protocol"].default is True +``` + +- [ ] **Step 3: Run the new tests; verify they FAIL** + +Run: `uv run pytest tests/test_task_manager.py -v` (or whichever file you appended to) +Expected: All 6 new tests FAIL — `wrap_prompt`/`create_task`/`coda_run` don't accept the kwarg yet. + +- [ ] **Step 4: Modify `coda_mcp/task_manager.py:153` — `wrap_prompt` signature + body** + +Open `coda_mcp/task_manager.py` and find the existing `wrap_prompt` function (starts around line 153). Change its signature and body as follows. + +Add a new import at the top of the file (if not already present, near other coda_mcp imports): + +```python +from coda_mcp.databricks_preamble import build_capabilities, build_workflow_protocol +``` + +Then change the function signature from: + +```python +def wrap_prompt( + task_id: str, + session_id: str, + email: str, + prompt: str, + context: dict | None, + results_dir: str, + context_hint: str | None = None, + previous_session_id: str | None = None, +) -> str: +``` + +to: + +```python +def wrap_prompt( + task_id: str, + session_id: str, + email: str, + prompt: str, + context: dict | None, + results_dir: str, + context_hint: str | None = None, + previous_session_id: str | None = None, + workflow_protocol: bool = True, +) -> str: +``` + +Update the docstring to mention the new flag: + +```python +"""Build the full prompt string written to ``prompt.txt``. + +Uses the ``---CODA-TASK---`` envelope convention so the agent can +parse metadata from the prompt deterministically. + +When ``workflow_protocol`` is True (default), inserts a CAPABILITIES +section (Databricks CLI, skills, MCP servers) and a WORKFLOW PROTOCOL +section (3-phase PLAN/EXECUTE/SYNTHESIZE with critique at each phase, +plus the info_needed escape hatch). Set False to skip both. +""" +``` + +Update the body. The current return statement looks roughly like this (around lines 184-225): + +```python +return ( + f"---CODA-TASK---\n" + ... + f"TASK:\n" + f"{prompt}\n" + f"\n" + f"INSTRUCTIONS:\n" + ... + f"SAFETY:\n" + ... + f"---END-CODA-TASK---" +) +``` + +Change it to insert the new sections between TASK and INSTRUCTIONS: + +```python +workflow_block = "" +if workflow_protocol: + workflow_block = ( + f"\nCAPABILITIES:\n" + f"{build_capabilities()}\n" + f"\n" + f"WORKFLOW PROTOCOL:\n" + f"{build_workflow_protocol()}\n" + ) + +return ( + f"---CODA-TASK---\n" + f"task_id: {task_id}\n" + f"session_id: {session_id}\n" + f"user: {email}\n" + f"{hint_line}" + f"{prior_session_block}" + f"{context_block}\n" + f"TASK:\n" + f"{prompt}\n" + f"{workflow_block}" + f"\n" + f"INSTRUCTIONS:\n" + f"1. As you work, append progress lines to {results_dir}/status.jsonl\n" + f' Each line must be valid JSON: {{"step": "label", "message": "what you are doing"}}\n' + f"\n" + f"2. When you are COMPLETELY DONE, write a SINGLE FILE at this exact path:\n" + f" {results_dir}/result.json\n" + f" It must contain this JSON structure:\n" + f" {{\n" + f' "status": "completed",\n' + f' "summary": "one paragraph describing what you did",\n' + f' "files_changed": ["list", "of", "file", "paths"],\n' + f' "artifacts": {{}},\n' + f' "errors": []\n' + f" }}\n" + f" If you failed, set status to \"failed\" and describe the error.\n" + f" IMPORTANT: result.json is a FILE not a directory. Write it with:\n" + f" echo '{{...}}' > {results_dir}/result.json\n" + f"\n" + f"3. If you delegate to a sub-agent, update status.jsonl with delegation steps.\n" + f"\n" + f"SAFETY:\n" + f"- Do NOT delete, drop, or truncate tables, schemas, catalogs, or volumes.\n" + f"- Do NOT delete files outside the current project directory.\n" + f"- Do NOT run destructive Databricks CLI commands (e.g. databricks clusters delete, " + f"databricks jobs delete, databricks pipelines delete).\n" + f"- Do NOT modify permissions, grants, or access controls unless explicitly requested.\n" + f"- Prefer CREATE OR REPLACE over DROP+CREATE. Prefer INSERT/MERGE over DELETE+INSERT.\n" + f"- If the task requires a destructive operation, describe what you would do in " + f"result.json with status \"needs_approval\" instead of executing it.\n" + f"---END-CODA-TASK---" +) +``` + +Note: the INSTRUCTIONS body itself is updated in Task 3 to mention `info_needed` and the new step labels. For this task, leave the INSTRUCTIONS text exactly as today — only insert the new sections. + +- [ ] **Step 5: Modify `coda_mcp/task_manager.py:231` — `create_task` signature + forward** + +Find the `create_task` function (starts around line 231). Add `workflow_protocol: bool = True` to its parameter list (alongside the existing kwargs like `timeout_s`, `permissions`, `previous_session_id`). Forward it into the `wrap_prompt` call inside the function body. + +The existing function probably looks like: + +```python +def create_task( + session_id: str, + prompt: str, + email: str, + context: dict | None = None, + context_hint: str | None = None, + timeout_s: int | None = None, + permissions: str | None = None, + previous_session_id: str | None = None, +): + ... + wrapped = wrap_prompt( + task_id=task_id, + session_id=session_id, + email=email, + prompt=prompt, + context=context, + results_dir=results_dir, + context_hint=context_hint, + previous_session_id=previous_session_id, + ) + ... +``` + +Change to: + +```python +def create_task( + session_id: str, + prompt: str, + email: str, + context: dict | None = None, + context_hint: str | None = None, + timeout_s: int | None = None, + permissions: str | None = None, + previous_session_id: str | None = None, + workflow_protocol: bool = True, +): + ... + wrapped = wrap_prompt( + task_id=task_id, + session_id=session_id, + email=email, + prompt=prompt, + context=context, + results_dir=results_dir, + context_hint=context_hint, + previous_session_id=previous_session_id, + workflow_protocol=workflow_protocol, + ) + ... +``` + +- [ ] **Step 6: Modify `coda_mcp/mcp_server.py:220` — `coda_run` signature + forward** + +Find the `coda_run` function (starts around line 220). Add `workflow_protocol: bool = True` to its parameter list and pass it to `task_manager.create_task`. + +Current signature: + +```python +async def coda_run( + prompt: str, + email: str, + context: str = "{}", + previous_session_id: str = "", + permissions: str = "smart", + timeout_s: int = 3600, +) -> str: +``` + +Change to: + +```python +async def coda_run( + prompt: str, + email: str, + context: str = "{}", + previous_session_id: str = "", + permissions: str = "smart", + timeout_s: int = 3600, + workflow_protocol: bool = True, +) -> str: +``` + +Update the docstring (the existing string ends "Returns JSON with ``task_id``, ``session_id``, and ``status: \"running\"``"). Add this sentence to the docstring body before the Returns line: + +``` +``workflow_protocol`` defaults to True, which injects a Databricks +orientation block and a 3-phase workflow protocol (PLAN/EXECUTE/SYNTHESIZE +with critique at each phase) into the agent's prompt. The protocol also +defines the ``info_needed`` terminal status for clean handoff when the +agent is blocked. Set False to skip — useful for non-Databricks tasks. +``` + +Find the `task_manager.create_task(...)` call (around line 265) and add the new kwarg: + +```python +result = task_manager.create_task( + session_id=session_id, + prompt=prompt, + email=email, + context=ctx, + timeout_s=timeout_s, + permissions=permissions, + previous_session_id=previous_session_id or None, + workflow_protocol=workflow_protocol, +) +``` + +- [ ] **Step 7: Run the new tests; verify they PASS** + +Run: `uv run pytest tests/test_task_manager.py -v` (or whichever file) +Expected: All 6 new tests PASS. + +Also run the full target file plus the new module's tests to check no regression: + +``` +uv run pytest tests/test_databricks_preamble.py tests/test_task_manager.py tests/test_coda_interactive.py tests/test_mcp_server.py tests/test_replay_only_flag.py -v +``` + +Expected: All pass. If any fail, fix the implementation (not the tests). + +- [ ] **Step 8: Run ruff** + +Run: `uv run ruff check coda_mcp/task_manager.py coda_mcp/mcp_server.py tests/test_task_manager.py` +Expected: clean. + +- [ ] **Step 9: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add coda_mcp/task_manager.py coda_mcp/mcp_server.py tests/test_task_manager.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat: wire workflow_protocol flag through coda_run → create_task → wrap_prompt + +The flag defaults to True. When set, wrap_prompt inserts CAPABILITIES and +WORKFLOW PROTOCOL sections between TASK and INSTRUCTIONS in prompt.txt. +Callers can opt out via workflow_protocol=False on coda_run for purely +non-Databricks tasks." +``` + +--- + +## Task 3: Update INSTRUCTIONS section to document `info_needed` + new step labels + +The INSTRUCTIONS block in `wrap_prompt` still says only "If you failed, set status to 'failed'" — silent about `info_needed`. Update it. + +**Files:** +- Modify: `coda_mcp/task_manager.py:153-225` (INSTRUCTIONS portion of `wrap_prompt`'s return) +- Modify (or extend): `tests/test_task_manager.py` + +- [ ] **Step 1: Append the pinning tests to `tests/test_task_manager.py`** + +```python +def test_wrap_prompt_instructions_documents_info_needed(): + """INSTRUCTIONS section must mention the info_needed status and feedback field.""" + from coda_mcp.task_manager import wrap_prompt + + out = wrap_prompt( + task_id="t-1", + session_id="s-1", + email="user@example.com", + prompt="do the thing", + context=None, + results_dir="/tmp/r", + ) + # Pull the INSTRUCTIONS section out for focused assertions. + assert "info_needed" in out + assert "feedback" in out + + +def test_wrap_prompt_instructions_lists_new_step_labels(): + """INSTRUCTIONS section enumerates the canonical step labels emitted by the agent.""" + from coda_mcp.task_manager import wrap_prompt + + out = wrap_prompt( + task_id="t-1", + session_id="s-1", + email="user@example.com", + prompt="do the thing", + context=None, + results_dir="/tmp/r", + ) + for label in ("plan", "critique_plan", "execute", "critique_execute", "synthesize", "critique_synthesize"): + assert label in out, f"Missing step label {label!r} from prompt text" +``` + +- [ ] **Step 2: Run; verify FAIL** + +Run: `uv run pytest tests/test_task_manager.py::test_wrap_prompt_instructions_documents_info_needed tests/test_task_manager.py::test_wrap_prompt_instructions_lists_new_step_labels -v` +Expected: both FAIL. + +- [ ] **Step 3: Update the INSTRUCTIONS section in `wrap_prompt`** + +In `coda_mcp/task_manager.py`, find the line that says `f' Each line must be valid JSON: ...'` (currently around line 197). Replace the entire INSTRUCTIONS portion (steps 1, 2, 3) with this: + +```python +f"INSTRUCTIONS:\n" +f"1. As you work, append progress lines to {results_dir}/status.jsonl\n" +f' Each line must be valid JSON: {{"step": "label", "message": "what you are doing"}}\n' +f" Canonical step labels (use these when the workflow protocol is active):\n" +f" plan, critique_plan, execute_, critique_execute,\n" +f" synthesize, critique_synthesize, info_needed, failed\n" +f"\n" +f"2. When you are COMPLETELY DONE, write a SINGLE FILE at this exact path:\n" +f" {results_dir}/result.json\n" +f" It must contain this JSON structure (status field has four allowed values):\n" +f" {{\n" +f' "status": "completed" | "failed" | "info_needed" | "needs_approval",\n' +f' "summary": "one paragraph describing what you did or why you stopped",\n' +f' "feedback": "REQUIRED if status=info_needed — what context the caller must add",\n' +f' "files_changed": ["list", "of", "file", "paths"],\n' +f' "artifacts": {{}},\n' +f' "errors": []\n' +f" }}\n" +f" - status=\"completed\": you finished the task.\n" +f" - status=\"failed\": unrecoverable hard error; describe in errors[].\n" +f" - status=\"info_needed\": you are blocked because something the CALLER must\n" +f" supply is missing. The feedback field is REQUIRED and must precisely\n" +f" name what is missing. The caller will resubmit with more context.\n" +f" - status=\"needs_approval\": you have a destructive action ready but need\n" +f" explicit caller approval before executing. See SAFETY section.\n" +f" IMPORTANT: result.json is a FILE not a directory. Write it with:\n" +f" echo '{{...}}' > {results_dir}/result.json\n" +f"\n" +f"3. If you delegate to a sub-agent, update status.jsonl with delegation steps.\n" +f"\n" +``` + +The block above replaces the OLD INSTRUCTIONS steps 1-3 ENTIRELY. The SAFETY section below it stays unchanged. + +- [ ] **Step 4: Run; verify GREEN** + +Run: `uv run pytest tests/test_task_manager.py -v` +Expected: all task_manager tests pass. + +Run: `uv run pytest tests/test_databricks_preamble.py tests/test_task_manager.py tests/test_coda_interactive.py -v` +Expected: still green across the board. + +- [ ] **Step 5: Ruff check** + +Run: `uv run ruff check coda_mcp/task_manager.py tests/test_task_manager.py` +Expected: clean. + +- [ ] **Step 6: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add coda_mcp/task_manager.py tests/test_task_manager.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat: document info_needed status and canonical step labels in INSTRUCTIONS + +The INSTRUCTIONS section of prompt.txt now enumerates the four allowed +result.json status values (completed, failed, info_needed, needs_approval), +describes when to use each, and lists the canonical status.jsonl step +labels emitted by the workflow protocol." +``` + +--- + +## Task 4: Update surfaces — counts dict, get_result docstring, MCP instructions paragraph (TDD) + +Three small surface updates that together let upstream callers understand the new statuses. + +**Files:** +- Modify: `coda_mcp/mcp_server.py:551-559` (counts dict in `coda_inbox`) +- Modify: `coda_mcp/mcp_server.py:573-584` (`coda_get_result` docstring) +- Modify: `coda_mcp/mcp_server.py:52-99` (FastMCP `instructions=` block) +- Create: `tests/test_inbox_status_passthrough.py` + +- [ ] **Step 1: Create the test file `tests/test_inbox_status_passthrough.py`** + +```python +"""Tests covering counts dict, coda_get_result docstring, and MCP instructions +all reflect the new info_needed / needs_approval terminal statuses.""" +import asyncio +import json + + +def test_mcp_instructions_mention_info_needed(): + """Server-level MCP instructions teach calling LLMs about info_needed.""" + from coda_mcp import mcp_server + + txt = mcp_server.mcp.instructions + assert "info_needed" in txt + assert "needs_approval" in txt + assert "feedback" in txt + + +def test_coda_get_result_docstring_mentions_info_needed(): + """coda_get_result docstring lists info_needed / needs_approval alongside completed/failed.""" + from coda_mcp import mcp_server + + doc = (mcp_server.coda_get_result.__doc__ or "").lower() + assert "info_needed" in doc + assert "needs_approval" in doc + + +def test_inbox_counts_dict_includes_new_statuses(monkeypatch): + """coda_inbox counts dict has info_needed and needs_approval keys.""" + from coda_mcp import mcp_server + + fake_tasks = [ + {"task_id": "t1", "session_id": "s1", "status": "running"}, + {"task_id": "t2", "session_id": "s2", "status": "completed"}, + {"task_id": "t3", "session_id": "s3", "status": "failed"}, + {"task_id": "t4", "session_id": "s4", "status": "info_needed"}, + {"task_id": "t5", "session_id": "s5", "status": "needs_approval"}, + {"task_id": "t6", "session_id": "s6", "status": "info_needed"}, + ] + + monkeypatch.setattr( + mcp_server.task_manager, "list_all_tasks", + lambda email, status_filter=None: list(fake_tasks), + ) + # _read_session_safe is called inside the loop; return None so no viewer_url is added. + monkeypatch.setattr( + mcp_server.task_manager, "_read_session_safe", lambda sid: None, + ) + + result_str = asyncio.run(mcp_server.coda_inbox(email="u@e")) + result = json.loads(result_str) + counts = result["counts"] + + assert counts["running"] == 1 + assert counts["completed"] == 1 + assert counts["failed"] == 1 + assert counts["info_needed"] == 2 + assert counts["needs_approval"] == 1 +``` + +- [ ] **Step 2: Run; verify FAIL** + +Run: `uv run pytest tests/test_inbox_status_passthrough.py -v` +Expected: all 3 tests FAIL — instructions don't mention info_needed, docstring doesn't, and counts dict has only 3 keys. + +- [ ] **Step 3: Update the FastMCP `instructions=` block in `coda_mcp/mcp_server.py:52-99`** + +Find the `mcp = FastMCP(...)` constructor (starts around line 50). Inside the `instructions=` argument is a multi-line string concatenation. Locate the existing "CHAINING" paragraph (the one that says `"CHAINING: pass previous_session_id ..."`). After that paragraph and BEFORE the "SHARE THE REPLAY URL" paragraph, insert this new paragraph: + +```python + "INFO_NEEDED HANDOFF: When coda_inbox shows a task with status='info_needed', " + "the agent could not proceed because of missing context. Call coda_get_result " + "to read the 'feedback' field — it tells you exactly what the agent needs (a " + "table name, a decision, a clarification). Add that context to the prompt and " + "resubmit via coda_run with previous_session_id set to the original task's " + "session_id so the agent has the prior attempt's context. 'needs_approval' is " + "similar but means the agent has a destructive plan and is waiting for the " + "caller's explicit go/no-go.\n\n" +``` + +Make sure the trailing newlines match the surrounding string concatenation (the other paragraphs end with `\n\n`). + +- [ ] **Step 4: Update the counts dict in `coda_inbox` (lines 551-559)** + +Find this block: + +```python +counts = {"running": 0, "completed": 0, "failed": 0} +for t in tasks: + s = t.get("status", "") + if s in counts: + counts[s] += 1 + elif s == "done": + counts["completed"] += 1 + elif s == "timeout": + counts["failed"] += 1 +``` + +Change the first line to add the two new keys: + +```python +counts = { + "running": 0, + "completed": 0, + "failed": 0, + "info_needed": 0, + "needs_approval": 0, +} +for t in tasks: + s = t.get("status", "") + if s in counts: + counts[s] += 1 + elif s == "done": + counts["completed"] += 1 + elif s == "timeout": + counts["failed"] += 1 +``` + +The aliasing branches (`done`, `timeout`) are unchanged. + +- [ ] **Step 5: Update `coda_get_result` docstring (line ~579)** + +Find the docstring of `coda_get_result`: + +```python +"""Retrieve the structured result of a completed task. + +Call this AFTER coda_inbox shows a task as "completed" or "failed". + +Returns JSON with ``task_id``, ``session_id``, ``status``, ``summary`` +(what was done), ``files_changed`` (list of modified files), +``artifacts`` (job IDs, commit hashes, etc.), and ``errors`` (if any). +""" +``` + +Change to: + +```python +"""Retrieve the structured result of a completed task. + +Call this AFTER coda_inbox shows a task as "completed", "failed", +"info_needed", or "needs_approval". + +Returns JSON with ``task_id``, ``session_id``, ``status``, ``summary`` +(what was done or why the agent stopped), ``files_changed`` (list of +modified files), ``artifacts`` (job IDs, commit hashes, etc.), +``errors`` (if any), and — when status is "info_needed" — ``feedback`` +(a precise description of what context the caller must add before +resubmitting). +""" +``` + +- [ ] **Step 6: Run the new tests; verify GREEN** + +Run: `uv run pytest tests/test_inbox_status_passthrough.py -v` +Expected: 3 passed. + +- [ ] **Step 7: Run target-area tests to verify no regression** + +Run: `uv run pytest tests/test_inbox_status_passthrough.py tests/test_coda_interactive.py tests/test_databricks_preamble.py tests/test_task_manager.py tests/test_mcp_server.py tests/test_replay_only_flag.py -v` +Expected: all pass. + +- [ ] **Step 8: Ruff** + +Run: `uv run ruff check coda_mcp/mcp_server.py tests/test_inbox_status_passthrough.py` +Expected: clean. + +- [ ] **Step 9: Commit** + +```bash +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + add coda_mcp/mcp_server.py tests/test_inbox_status_passthrough.py +git -c user.email=datasciencemonkey@gmail.com -c user.name="Sathish Gangichetty" \ + commit -m "feat: surface info_needed + needs_approval in inbox counts, get_result doc, MCP instructions + +Three surfaces updated so calling LLMs and dashboards know about the +two soft terminal statuses: +- coda_inbox counts dict gains info_needed and needs_approval keys. +- coda_get_result docstring lists all four valid statuses and the + feedback field that accompanies info_needed. +- FastMCP server-level instructions gain an INFO_NEEDED HANDOFF + paragraph teaching upstream LLMs to read 'feedback' and resubmit + with previous_session_id for the chained context." +``` + +--- + +## Task 5: Push branch and update PR #67 description + +**Files:** +- None (remote/PR update only) + +- [ ] **Step 1: Verify branch state** + +```bash +git status +git log --oneline origin/feat/coda-mcp-interactive-handoff..HEAD +``` + +Expected: working tree clean. The new commits since the last push include the spec, the spec-critic fixes, the plan, and the four implementation commits (Tasks 1-4). + +- [ ] **Step 2: Push** + +```bash +git push origin feat/coda-mcp-interactive-handoff +``` + +Expected: fast-forward push. + +- [ ] **Step 3: Append a follow-up section to PR #67 body** + +Read the current body: + +```bash +gh pr view 67 --json body -q .body > /tmp/pr67-body.md +``` + +Append this section: + +``` +--- + +## Follow-up #2: Workflow protocol + Databricks orientation + +`coda_run` now injects two new sections into `prompt.txt`: +- **CAPABILITIES** — tells hermes about the Databricks CLI (pre-authed), the 16 Databricks skills under `~/.claude/skills/`, and the DeepWiki / Exa / CoDA MCP servers. +- **WORKFLOW PROTOCOL** — imposes a 3-phase pipeline (PLAN → EXECUTE → SYNTHESIZE) with a critique step after each phase (self-review or sub-agent — agent's choice). Max 2 iterations per phase to keep token cost bounded. + +New terminal `result.json` status `"info_needed"` with a required `feedback` field gives the calling client a structured iteration loop when the agent is blocked. The existing `"needs_approval"` status is preserved with explicit disambiguation: `info_needed` = "caller must add context"; `needs_approval` = "caller must approve a destructive action". + +**Three surfaces updated** so upstream LLMs know about the new statuses: +- `coda_inbox` counts dict gains `info_needed` and `needs_approval` keys. +- `coda_get_result` docstring lists all four valid statuses + the new `feedback` field. +- FastMCP server-level instructions gain an INFO_NEEDED HANDOFF paragraph. + +**Flag:** `coda_run(... workflow_protocol=True)` is the default. Set False to skip both new sections for non-Databricks tasks. + +**Artifacts:** +- Spec: `docs/superpowers/specs/2026-05-28-coda-run-workflow-protocol-design.md` +- Plan: `docs/superpowers/plans/2026-05-28-coda-run-workflow-protocol.md` +``` + +Then update the PR body: + +```bash +gh pr edit 67 --body-file /tmp/pr67-body.md +``` + +Or if gh's TLS bug hits on this machine, fall back to curl + REST per the prior follow-up. + +- [ ] **Step 4: Confirm** + +Run `gh pr view 67 --json body -q .body | tail -30` and verify the new section appears. + +--- + +## Self-review of this plan against the spec + +**Spec section 1 — Goal.** Task 1 creates the module; Task 2 wires the flag; Task 3 updates INSTRUCTIONS; Task 4 surfaces the statuses. ✓ + +**Spec section "Components" 1 (databricks_preamble.py).** Task 1 creates it with all three exports. ✓ + +**Components 2 + 3 (CAPABILITIES + WORKFLOW PROTOCOL content).** Task 1's module has the verbatim text from the spec. ✓ + +**Components 4 (expanded INSTRUCTIONS).** Task 3 covers it. ✓ + +**Components 5 (task_manager changes).** Task 2 covers wrap_prompt and create_task. ✓ + +**Components 6 (mcp_server.coda_run changes).** Task 2 covers it. ✓ + +**Components 7 (counts dict + get_result docstring).** Task 4 covers both. ✓ + +**Components 7a (MCP instructions string).** Task 4 covers it. ✓ + +**Components 7b (watcher interaction).** Documented in spec as no-code-change. Plan does not need a task for it. + +**Testing strategy.** Every test listed in the spec maps to a task step in Task 1 (`test_databricks_preamble.py`), Task 2 (extension of `test_task_manager.py`), Task 3 (further extension of same), Task 4 (`test_inbox_status_passthrough.py`). ✓ + +**Acceptance criteria 1-8.** All mapped. ✓ + +**Placeholder scan:** No TBD/TODO. Every step has explicit code or commands. + +**Type consistency:** `workflow_protocol: bool = True` used uniformly across all three call sites (wrap_prompt, create_task, coda_run). Step labels (`plan`, `critique_plan`, etc.) match between Task 1's module text, Task 3's INSTRUCTIONS update, and the spec. + +**Risk: Task 2 Step 5 might leave the `_write_task_meta` mock or other internal helpers' signatures stale.** The test `test_create_task_forwards_workflow_protocol_to_wrap_prompt` monkeypatches `_session_dir`, `_task_dir`, `_write_task_meta`, and `os.makedirs`. If `create_task` calls additional helpers in production, the test will fail with cryptic AttributeError. If that happens during execution, add the missing helpers to the monkeypatch list — the test's intent is to verify ONLY the flag pass-through, not the file-system side effects. diff --git a/docs/superpowers/specs/2026-05-27-coda-mcp-live-session-url-design.md b/docs/superpowers/specs/2026-05-27-coda-mcp-live-session-url-design.md new file mode 100644 index 0000000..c82bdc6 --- /dev/null +++ b/docs/superpowers/specs/2026-05-27-coda-mcp-live-session-url-design.md @@ -0,0 +1,447 @@ +# CoDA MCP Live Session URL — Design + +**Date:** 2026-05-27 +**Branch:** `feat/coda-mcp-server` +**Status:** Spec approved by user; ready for implementation plan +**Related PR:** databrickslabs/coding-agents-databricks-apps#64 (parent feature) + +## 1. Problem + +`coda_run` is fire-and-forget today: it returns `{task_id, session_id, status: "running"}` and the calling MCP client (Genie Code, Claude Desktop, Cursor) has no way to surface progress to the user. The user only sees a structured `result.json` after the task completes via `coda_inbox`/`coda_get_result`. Status messages from `status.jsonl` are coarse-grained. There is no way to watch hermes execute live, intervene mid-task, or reconstruct what happened after the fact. + +The Flask app side already has a fully working real-time terminal UI (xterm.js + Socket.IO + HTTP polling fallback) that knows how to attach to any active PTY by id. The MCP server already spawns those PTYs to run hermes. **The two halves are not connected by a URL.** + +## 2. Goal + +Give every `coda_run` (and existing tasks listed via `coda_inbox` / fetched via `coda_get_result`) a `viewer_url` that: + +- **During execution** — opens the existing terminal UI attached to that task's live PTY. The user can watch hermes work in real time and type into the session if they want to redirect or take over (single-user app; this is intentional). +- **For ~5 minutes after completion** — keeps the PTY alive so a viewer who joined mid-task isn't yanked the instant `result.json` is written. Heartbeats from an active viewer do not extend this window — the grace timer is fixed. +- **Indefinitely after PTY closes** (within the 24h `TASK_TTL_S`) — serves a static "replay" rendering of the captured terminal transcript so a user can scroll the full execution history from `coda_inbox`. + +Out of scope (deferred to separate specs): configurable agent selection (hermes vs claude-code vs codex), multi-user attribution, asciinema-style timed replay. + +## 3. Architecture + +``` +┌────────────────────────────────────────────────────────────────┐ +│ MCP client Browser │ +│ (Genie Code, Claude Desktop) (single user, app URL) │ +└──────────┬──────────────────────────────────┬──────────────────┘ + │ tools/call coda_run │ GET /?session= + ▼ ▼ + ┌───────────────┐ ┌─────────────────────┐ + │ coda_mcp /mcp │ │ Flask /static + WS │ + │ +viewer_url │ │ /api/session/attach│ + └───────┬───────┘ └──────────┬──────────┘ + │ │ + ▼ ▼ + ┌──────────────────────────────────────────────────────┐ + │ Flask app (single process) │ + │ sessions[] → {fd, buffer, transcript_fh, │ + │ grace: bool} │ + │ read_pty_output thread: │ + │ fd → buffer → socketio emit (room=) │ + │ fd → transcript.log (NEW: tee, flush per write) │ + └──────────────────────────────────────────────────────┘ + │ │ + │ writes (chmod 600) │ reads when PTY gone + ▼ ▼ + ~/.coda/sessions/{sess}/tasks/{task}/transcript.log +``` + +Everything between the MCP server and the Flask app already exists. The feature is mostly plumbing: + +1. **Tee PTY output** to `transcript.log` (on disk, per task, chmod 0600, 10 MB soft cap). +2. **Defer PTY close** on task completion by 5 minutes (`threading.Timer`) so live viewers can finish reading. +3. **Build `viewer_url`** in MCP tool responses by capturing `X-Forwarded-Host` from the inbound request. +4. **Teach the SPA** to read `?session=` on load and to render replay mode when the PTY is gone but a transcript exists. + +## 4. Components + +### 4.1 `app.py::sessions[pty_id]` dict (additive) + +Four new keys, all optional/defaulting: + +- `transcript_path: str | None` — absolute path to the tee target. +- `transcript_fh: BinaryIO | None` — open file handle owned by `read_pty_output`. +- `transcript_bytes: int` (default 0) — running count to enforce the 10 MB cap. +- `grace: bool` (default False) — set `True` when `_watch_task` schedules deferred close. Used by the concurrency check to exempt this slot. + +No removals. No semantic changes to existing keys. + +### 4.2 `app.py::mcp_create_pty_session(label, transcript_path=None)` + +New optional kwarg. When provided: + +- `os.makedirs(os.path.dirname(transcript_path), exist_ok=True)` +- Open file: `fh = open(transcript_path, "ab", buffering=0)` (binary append, unbuffered) +- `os.fchmod(fh.fileno(), 0o600)` immediately +- Store `transcript_path` and `transcript_fh` on the session dict +- If open fails: log error, set both to `None`, continue (live PTY still works) + +### 4.3 `app.py::read_pty_output` (additive) + +After the existing buffer append and Socket.IO emit, if a transcript handle is present, write under the per-session lock to prevent races against `terminate_session` (which may close the handle from the Timer thread): + +```python +with session_lock: + fh = session.get("transcript_fh") + written = session.get("transcript_bytes", 0) + if fh is not None: + remaining = TRANSCRIPT_CAP_BYTES - written + if remaining > 0: + chunk = output[:remaining] + try: + fh.write(chunk) + fh.flush() + session["transcript_bytes"] = written + len(chunk) + if len(chunk) < len(output): + fh.write(b"\n[transcript truncated at 10MB]\n") + fh.flush() + fh.close() + session["transcript_fh"] = None + except (OSError, ValueError) as exc: + logger.warning("transcript write failed for %s: %s", session_id, exc) + try: fh.close() + except Exception: pass + session["transcript_fh"] = None +``` + +`TRANSCRIPT_CAP_BYTES = 10 * 1024 * 1024`. + +**Invariants** (documented for future maintainers): + +- `transcript_fh` is opened in `mcp_create_pty_session`, written exclusively by `read_pty_output`, and closed by either (a) `read_pty_output` on cap/error or (b) `terminate_session` on PTY teardown. All three sites operate under `session["lock"]`. +- `transcript_bytes` is incremented only by `read_pty_output`. Single-writer; reads from other threads must hold `session["lock"]`. +- `ValueError` is caught alongside `OSError` to defend against a tiny window where `terminate_session` closes the handle between the spec's `if fh is not None` check and the actual `fh.write` call — the lock prevents this, but the catch is belt-and-suspenders. + +### 4.4 `app.py::terminate_session` (additive) + +Close the transcript file handle under the per-session lock before the existing fd close. The swap-to-`None` is the synchronization point that lets `read_pty_output` notice the handle is gone on its next iteration: + +```python +sess = sessions.get(session_id) +if sess is not None: + with sess["lock"]: + fh = sess.get("transcript_fh") + sess["transcript_fh"] = None # swap first, then close + if fh is not None: + try: fh.close() + except Exception: pass +``` + +(The actual close happens outside the lock to avoid holding it across a potential blocking I/O on a slow filesystem.) + +### 4.5 `app.py::MAX_CONCURRENT_SESSIONS` check (modified) + +At the `if len(sessions) >= MAX_CONCURRENT_SESSIONS` checkpoints in `create_session()` and `mcp_create_pty_session()`, replace the raw length check with a filtered count that excludes grace-period PTYs: + +```python +active = sum(1 for s in sessions.values() if not s.get("grace")) +if active >= MAX_CONCURRENT_SESSIONS: ... +``` + +`cleanup_stale_sessions` itself is **unchanged** — it still treats grace-period PTYs like any other session, but the 24h `SESSION_TIMEOUT_SECONDS` is so long the reaper never wins the race against the 5-min Timer. + +`MAX_CONCURRENT_SESSIONS` default stays at 5. + +### 4.6 `coda_mcp/mcp_server.py::_watch_task` (modified) + +Both completion and timeout paths replace immediate `_close_pty_for_session(session_id)` with: + +```python +session_data = task_manager._read_session(session_id) +pty_session_id = session_data.get("pty_session_id") +if pty_session_id and _app_close_session is not None: + _mark_grace(pty_session_id) # sets sessions[pty_id]["grace"] = True + _bump_last_poll(pty_session_id, GRACE_PERIOD_S) # defensive against reaper + threading.Timer( + GRACE_PERIOD_S, + _app_close_session, + args=(pty_session_id,), + ).start() +``` + +`GRACE_PERIOD_S = 300` (5 minutes), defined as a module constant for testability. `_mark_grace` and `_bump_last_poll` are two new hook callbacks wired through `set_app_hooks()` alongside the existing three — consistent with the current pattern (no direct Flask imports from the MCP module). + +The Timer must be a daemon so it doesn't block uvicorn shutdown: `t = threading.Timer(...); t.daemon = True; t.start()`. + +### 4.7 `coda_mcp/mcp_server.py::coda_run` (additive) + +After `mcp_create_pty_session`, compute the transcript path and pass it in: + +```python +transcript_path = os.path.join( + task_manager._task_dir(session_id, task_id), + "transcript.log", +) +pty_session_id = _app_create_session( + label="hermes-mcp", + transcript_path=transcript_path, +) +``` + +(Note: `_app_create_session` signature gains the kwarg. The implementation in `app.py` already documented above.) + +Then build the response with the new field: + +```python +return json.dumps({ + "task_id": task_id, + "session_id": session_id, + "status": "running", + "viewer_url": _build_viewer_url(pty_session_id), # may be None +}) +``` + +Tools serialize via `json.dumps` so `None` becomes `null`. Clients that don't recognize the field will ignore it. + +### 4.8 `coda_mcp/url_builder.py` (new tiny module) + +```python +import os +from typing import Optional + +_app_url_cache: Optional[str] = None + +def capture_from_headers(host: Optional[str]) -> None: + """Called by middleware on every inbound request.""" + global _app_url_cache + if host: + _app_url_cache = host + +def build_viewer_url(pty_session_id: str) -> Optional[str]: + override = os.environ.get("CODA_APP_URL", "").strip() + if override: + base = override.rstrip("/") + elif _app_url_cache: + base = f"https://{_app_url_cache}" + else: + return None + return f"{base}/?session={pty_session_id}" +``` + +### 4.9 `coda_mcp/mcp_asgi.py` (additive middleware) + +Insert a small ASGI middleware on `mcp_starlette` (via `mcp_starlette.add_middleware(...)`) that extracts `X-Forwarded-Host` (fallback: `Host`) from every HTTP request and calls `url_builder.capture_from_headers(host)`. Both MCP requests AND inbound browser HTTP requests refresh the cache. + +**Coverage caveat** (not a problem in practice): the top-level ASGI app is `socketio.ASGIApp(sio, other_asgi_app=mcp_starlette)`, so `/socket.io/` traffic is intercepted by socketio *before* it reaches `mcp_starlette` and therefore never hits this middleware. This is fine because (a) the user always loads the SPA via plain HTTP first (which refreshes the cache), and (b) every `coda_run` MCP call is a plain HTTP POST to `/mcp` (also through the middleware). The cache is hot by the time any tool needs the URL. + +```python +class AppUrlCaptureMiddleware: + def __init__(self, app): self.app = app + async def __call__(self, scope, receive, send): + if scope["type"] == "http": + headers = dict(scope.get("headers") or []) + host = headers.get(b"x-forwarded-host") or headers.get(b"host") + if host: + url_builder.capture_from_headers(host.decode()) + await self.app(scope, receive, send) +``` + +### 4.10 `coda_mcp/task_manager.py::find_task_dir_by_pty_session` (new) + +```python +_pty_lookup_cache: dict[str, tuple[str, float]] = {} # pty_id -> (task_dir, ts) +_PTY_LOOKUP_TTL = 60.0 # seconds + +def find_task_dir_by_pty_session(pty_session_id: str) -> str | None: + """Find the task dir whose session.json carries this pty_session_id.""" + now = time.time() + cached = _pty_lookup_cache.get(pty_session_id) + if cached and (now - cached[1]) < _PTY_LOOKUP_TTL: + return cached[0] + # Scan SESSIONS_DIR + if not os.path.isdir(SESSIONS_DIR): + return None + for sess_name in os.listdir(SESSIONS_DIR): + sess_file = os.path.join(SESSIONS_DIR, sess_name, "session.json") + try: + with open(sess_file) as f: + data = json.load(f) + except (OSError, json.JSONDecodeError): + continue + if data.get("pty_session_id") != pty_session_id: + continue + # The session has a current_task or completed_tasks; pick the most recent. + candidate = data.get("current_task") or ( + data["completed_tasks"][-1] if data.get("completed_tasks") else None + ) + if candidate: + tdir = os.path.join(SESSIONS_DIR, sess_name, "tasks", candidate) + _pty_lookup_cache[pty_session_id] = (tdir, now) + return tdir + return None +``` + +TTL handles the rename/close case without manual invalidation. + +**Invariant**: CoDA MCP sessions are ephemeral — one task per session (see `task_manager.create_session` then `complete_task` which sets `current_task=None` and appends to `completed_tasks`). This function therefore returns the right task dir for the lifetime of the URL. If the lifecycle ever changes to allow task reuse within a single session, this function must be revisited to pick the *active or grace-period* task rather than `completed_tasks[-1]`. + +### 4.11 `app.py::attach_session` endpoint (additive) + +After the existing `_get_session()` lookup, add a fallback: + +```python +sess = _get_session(session_id) +if not sess or sess.get("exited"): + # NEW: try transcript replay + tdir = task_manager.find_task_dir_by_pty_session(session_id) + if tdir: + transcript = os.path.join(tdir, "transcript.log") + if os.path.isfile(transcript): + with open(transcript, "rb") as f: + content = f.read() + return jsonify({ + "session_id": session_id, + "label": "hermes-mcp (replay)", + "output": [content.decode("utf-8", errors="replace")], + "replay": True, + "process": None, + "created_at": None, + }) + return jsonify({"error": "Session not found or exited"}), 404 +``` + +The response shape (`output: [str]`, `replay: true|absent`, plus existing keys) is **NOT** consumed by the existing `_doAttach` — that function deliberately ignores `data.output` and forces a SIGWINCH redraw of the live application (`static/index.html:1339-1357`, comment at line 1347: "We skip buffer replay because it contains raw escape sequences that produce garbled output"). The replay-mode response is consumed by a new SPA function `_doReplay` described in §4.12, which writes the bytes directly into xterm. + +### 4.12 `static/index.html` (~50-70 LoC) + +Four additions: + +1. **Boot-time URL parse** — before the existing session-picker fetch, check `new URLSearchParams(location.search).get("session")`. If absent → existing flow. If present → call `POST /api/session/attach` once and branch on the response: + - 200 with `replay: true` → call **`_doReplay`** (new, described below). Skip `_doAttach`. Do NOT emit `join_session`. Do NOT wire `terminal_input` to the WS. + - 200 without `replay` → call the existing `_doAttach(term, sessionId)` and the existing `socket.emit('join_session', { session_id })` path. (Reusing `_doAttach` is correct here because the *live* PTY is running an interactive app, and SIGWINCH-redraw is the right behavior.) + - 404 → render a small in-page fallback: "session expired or never existed" + a button to navigate to `/`. + +2. **`_doReplay(term, sessionId, bytes)` — new function** that handles static replay rendering. Cannot route through `_doAttach` because `_doAttach` discards `data.output` (it relies on a running app to redraw via SIGWINCH; replay mode has no running app). Implementation: + + ```js + async function _doReplay(term, sessionId, content) { + // Chunk the write to avoid main-thread jank on multi-MB transcripts. + // xterm.js write() is internally batched, but a single 10MB call + // still blocks until the parser drains. 64KB slices with rAF gives + // the browser a chance to repaint between chunks. + const CHUNK = 64 * 1024; + for (let i = 0; i < content.length; i += CHUNK) { + term.write(content.slice(i, i + CHUNK)); + await new Promise(r => requestAnimationFrame(r)); + } + // Mount a small "Task completed — viewing replay" banner above the pane. + // No input handler, no WS subscription, no heartbeat for this session id. + } + ``` + +3. **Replay-mode pane behavior** — the tab gets a "(replay)" badge. The xterm input handler is not wired. The session is NOT included in the heartbeat session_ids list (the PTY is dead; heartbeats would 404 the lookup). + +4. **History/URL hygiene** — when the user closes a pane that was opened via `?session=`, call `history.replaceState({}, '', '/')` so a refresh doesn't re-attach. + +**Estimate revised**: 50-70 LoC including the new `_doReplay` and the 404 fallback. Architecturally the most "real" change in the spec — the rest of the codebase shifts are mostly additive. + +### 4.13 MCP tool `instructions` update (`coda_mcp/mcp_server.py`) + +Append one paragraph to the existing `instructions` block on the FastMCP instance: + +> SHARE THE LIVE URL: When `coda_run` returns a `viewer_url` field, mention it to the user in plain text (e.g. "you can watch progress at "). The URL is safe to share — it points to the same Databricks App the user is already authenticated against. Do this on the FIRST mention of the task and any time the user asks where the task is or how to see it. + +## 5. Data flow + +### 5.1 Submit + +`MCP client → /mcp coda_run → task_manager.create_session → mcp_create_pty_session(transcript_path) → task_manager.create_task → mcp_send_input("hermes -z ...") → _watch_task thread spawned → return {task_id, session_id, status: "running", viewer_url}`. + +### 5.2 Live view + +`Browser → GET /?session= → SPA reads ?session → POST /api/session/attach → live output buffer returned → WS join_session → live stream from read_pty_output → terminal_input writes to fd → heartbeat keeps the (already non-grace) PTY alive`. + +### 5.3 Grace window + +At T+0 hermes writes `result.json`. `_watch_task` calls `task_manager.complete_task` (disk status → closed), marks the PTY `grace=True`, bumps `last_poll_time`, schedules `Timer(300, _app_close_session)`. A viewer present at T+0 keeps streaming for up to 5 min. At T+300 the Timer SIGHUPs bash, `read_pty_output` sees EOF, flushes and closes the transcript handle, removes the session entry. + +### 5.4 Replay + +`Browser → GET /?session= → POST /api/session/attach → PTY not found → find_task_dir_by_pty_session → read transcript.log → return {output: [bytes], replay: true} → SPA renders bytes, no WS subscription`. + +## 6. Error handling + +| Failure | Behavior | +|---|---| +| `CODA_APP_URL` and `X-Forwarded-Host` both absent | `viewer_url: null`. One startup WARN. | +| Transcript open fails | `transcript_fh = None`. Live PTY works; replay disabled. | +| Transcript write fails mid-stream | Log once per session, close handle, set `transcript_fh = None`, keep reading PTY. | +| 10 MB cap hit | Write marker, close handle, set `transcript_fh = None`. PTY keeps streaming live (no further teeing). | +| Timer fires after manual close | `terminate_session` is re-entrant; `sessions.pop(_, None)` and `os.kill` wrapped in try/except. No-op. | +| uvicorn restart during grace | In-memory state lost; old `viewer_url` falls through to transcript replay (if file exists) or 404. Acceptable. | +| Browser opens URL mid-grace, grace expires while connected | `read_pty_output` emits `session_exited` to the room. SPA shows "session ended" banner. User reloads → replay mode. | +| Browser opens URL after grace AND transcript reaped | 404. SPA shows expired page. | +| `MAX_CONCURRENT_TASKS` reached | Unchanged "concurrency limit" error. Grace PTYs don't count toward this (disk status = closed). | +| `MAX_CONCURRENT_SESSIONS` reached among active (non-grace) | Existing 429. Grace PTYs don't count. | +| Hermes hangs (no `result.json`) | Existing `_watch_task` timeout path now also defers close via the same Timer mechanism. | + +## 7. Testing + +### 7.1 Unit + +- `coda_mcp/url_builder.py`: env override beats header capture; `None` when both absent; trailing slash on override is stripped. +- `coda_run` returns `viewer_url` only when builder returns non-None; same for `coda_inbox` per-entry and `coda_get_result`. +- `find_task_dir_by_pty_session`: hit, miss, TTL expiry, ignores corrupt session.json. +- `_watch_task`: schedules `Timer` (mocked) with correct args on both completion and timeout paths; never calls `_app_close_session` synchronously. +- `_mark_grace` / `_bump_last_poll` set the session dict fields. + +### 7.2 Integration (`tests/test_mcp_integration.py`) + +- E2E with a stub hermes (`bash -c 'echo hello; touch results/result.json; echo done'`): + - `transcript.log` contains "hello". + - At T+1s, PTY still alive (grace). + - At T+(GRACE+1)s (test uses a 2s grace via patched constant), PTY closed; transcript file persists. + - `/api/session/attach` returns `replay: true` after close; live mode before. +- Concurrency: submit `MAX_CONCURRENT_TASKS` tasks, complete them all (grace begins), submit `MAX_CONCURRENT_TASKS` more — all succeed (grace PTYs don't block). +- 10 MB cap: feed a hermes stub that prints `>10MB` of output; transcript file is exactly `10MB + marker`; PTY keeps running. + +### 7.3 SPA + +- New `tests/test_frontend_deeplink.spec.js` (Playwright if available; else manual checklist): + - `/?session=` → live attach, WS room joined, terminal renders. + - `/?session=` → replay rendered, no WS join, banner visible. + - `/?session=` → expired page. + - Closing the pane drops `?session=` from `history`. + +### 7.4 Manual smoke + +- Deploy to `mcp-test-coda` app, connect Genie Code, run a `coda_run`, click `viewer_url` from the chat response, confirm live stream + grace + replay. +- `chmod 600` check: `ls -la ~/.coda/sessions/*/tasks/*/transcript.log` on deployed pod. +- Confirm `viewer_url` absent on a local uvicorn boot without `CODA_APP_URL` and no inbound request yet. + +## 8. Open questions (resolved) + +- ~~Read-only vs interactive viewer?~~ → Interactive (full terminal). +- ~~Grace period mechanism?~~ → `threading.Timer(300, _close)`. +- ~~Replay storage?~~ → Tee to `transcript.log`. +- ~~Configurable agent?~~ → Deferred to a separate spec. +- ~~Base URL resolution?~~ → `CODA_APP_URL` env override → `X-Forwarded-Host` capture (officially provided by Databricks Apps). +- ~~Concurrency under grace?~~ → Exempt grace PTYs from `MAX_CONCURRENT_SESSIONS`. Cap stays at 5. + +## 9. Risks accepted + +- **Transcript on disk contains secrets** if hermes prints them. Single-user app, file is mode 0600, cleaned with the rest of the session at 24h TTL. Documented in `docs/mcp-v2-background-execution.md`. +- **5 min grace + 0 second active task** means a viewer who opens the URL late may still race the close. Acceptable; replay mode covers them. +- **Browser tabs can interact with the same PTY simultaneously.** Already true for the existing terminal UI; no new exposure. + +## 10. Surface summary + +| Surface | LoC est | Risk | +|---|---|---| +| `app.py` (4 functions touched) | ~60 | Low — additive, no semantic shifts | +| `coda_mcp/mcp_server.py` (2 functions + instructions) | ~40 | Low | +| `coda_mcp/url_builder.py` (new) | ~25 | Low | +| `coda_mcp/mcp_asgi.py` (middleware) | ~15 | Low | +| `coda_mcp/task_manager.py` (new lookup) | ~30 | Low | +| `static/index.html` | ~50-70 | Medium — new boot branch + new `_doReplay` rendering path; live attach still reuses `_doAttach` | +| Tests | ~250 | — | + +**Total**: ~235-255 LoC of production code + ~250 LoC of tests. + +## 11. Next step + +Hand to `writing-plans` skill to produce an executable implementation plan with task ordering, dependencies, and verification gates. diff --git a/docs/superpowers/specs/2026-05-28-coda-interactive-broaden-source-design.md b/docs/superpowers/specs/2026-05-28-coda-interactive-broaden-source-design.md new file mode 100644 index 0000000..0b35f09 --- /dev/null +++ b/docs/superpowers/specs/2026-05-28-coda-interactive-broaden-source-design.md @@ -0,0 +1,162 @@ +# Spec: Broaden `coda_interactive` source to any Workspace folder + +**Status:** Draft, pre-critique-gate +**Date:** 2026-05-28 +**Branch:** `feat/coda-mcp-interactive-handoff` (continues PR #67) +**Amends:** `docs/superpowers/specs/2026-05-28-coda-interactive-mcp-tool-design.md` + +## Goal + +Drop the requirement that `coda_interactive`'s `workspace_path` point to a Databricks Workspace **Git Folder**. The path can be any Workspace directory — a Git Folder *or* a plain Workspace folder. The MCP tool only needs the directory to exist in the workspace; how it got there is the caller's concern. + +## Why + +The original design (PR #67) used the Repos API (`client.repos.list` + `client.repos.update`) to resolve a Git Folder and optionally switch its branch before exporting. Two problems with that: + +1. **Unnecessary friction.** Users with a regular Workspace folder (uploaded via the UI, written via the Jobs API, etc.) cannot hand off to `coda_interactive` even though the underlying Workspace export API (`client.workspace.list` + `client.workspace.export`) works for *both* Git Folders and plain folders. The Repos gate excludes a valid use case for no benefit. +2. **Branch convenience overlaps with caller capabilities.** The upstream MCP caller (Genie Code, Claude Desktop) already has Databricks SDK access — if they want a specific branch checked out on a Git Folder, they can do it themselves before calling. The `branch` parameter on `coda_interactive` was duplicating capability that already lives upstream. + +Broadening the contract makes the tool surface smaller and the call site simpler. The user's framing: *"It may or may not be backed by git."* + +## Changes + +### 1. Tool signature + +**Before:** +```python +async def coda_interactive( + prompt: str, + workspace_path: str, + branch: str = "", + agent: str = "claude", + email: str = "", +) -> str: +``` + +**After:** +```python +async def coda_interactive( + prompt: str, + workspace_path: str, + agent: str = "claude", + email: str = "", +) -> str: +``` + +The `branch` parameter is removed entirely. If the caller wants a Git Folder on a specific branch, they switch it themselves before calling. + +### 2. Body of `coda_interactive` + +**Removed:** +- `client.repos.list(path_prefix=workspace_path)` lookup +- The exact-match filter (`next((r for r in repos if r.path == workspace_path), None)`) +- The `client.repos.update(repo_id=repo.id, branch=branch)` call + +**Added (light validation):** +A single `client.workspace.get_status(workspace_path)` call before export, to give callers a clean error when the path doesn't exist or isn't a directory. This replaces the implicit "empty export" failure mode with an explicit error. + +```python +try: + status = client.workspace.get_status(workspace_path) +except Exception as e: + return json.dumps({ + "status": "error", + "error": f"Workspace path not found: {workspace_path}: {e}", + }) + +if not _is_directory(status): + return json.dumps({ + "status": "error", + "error": f"Workspace path is not a directory: {workspace_path}", + }) +``` + +`_is_directory` already exists in `workspace_export.py` and works for both real SDK objects and mocks. Re-use it. + +### 3. Return shape + +**Removed field:** `"branch"`. + +**After:** +```json +{ + "status": "launched", + "viewer_url": "...", + "agent": "claude", + "project_dir": "/home/app/.coda/projects/pty-...", + "workspace_path": "/Workspace/Users/me@db.com/projects/feature-X", + "instructions": "Open viewer_url to attach. The agent is loaded with the project files exported from Workspace and your kickoff prompt typed. Type the agent's quit command (e.g. /quit) and then `exit` to end the session. Note: git history is NOT available in the session — files are an export, not a clone." +} +``` + +The `instructions` string is unchanged — it never claimed git history was preserved, so it stays valid for both Git Folders and plain folders. + +### 4. Caller pre-condition (spec section 1a rewrite) + +**Old contract:** "Project must be a Databricks Workspace Git Folder; commit and push to remote before calling." + +**New contract:** "Project must be a directory at `workspace_path` in the Databricks Workspace. Files visible to `workspace.export` (notebooks, source files) will appear in the session. If the directory is a Git Folder and you want a specific branch, switch it on the Git Folder yourself before calling — the export is a server-side snapshot." + +### 5. INTERACTIVE HANDOFF instructions string (server-level) + +The paragraph in `coda_mcp/mcp_server.py:79` surfaced to upstream LLM callers is rewritten: + +**Before (excerpt):** +> The user's project must be a Databricks Workspace Git Folder ... commit and push any local working changes back to the Git Folder's remote before calling. + +**After:** +> The tool reads files from a directory that already exists in the Databricks Workspace (a Git Folder or a plain Workspace folder — either works). If your working files are not yet in the Workspace, upload them first (`workspace.import` via the Databricks SDK, REST, or CLI) into a folder the user can read, then pass that folder as `workspace_path`. The tool does NOT accept inline file payloads. If the directory is a Git Folder, ensure the desired branch is checked out and pushed first — the export is a server-side snapshot. + +**Why the upload-then-handoff guidance is explicit:** The full workflow this tool enables is *upstream client generates / collects working files → uploads them to a Workspace folder → calls `coda_interactive` with that folder → the user opens the viewer URL and continues live in CoDA*. The instructions string needs to make the upload step visible to the calling LLM; otherwise it might assume `coda_interactive` accepts a file payload or that the user has already wired up the Workspace folder by hand. + +## What does NOT change + +- **`export_workspace_tree` helper** — already generic. No code changes in `coda_mcp/workspace_export.py`. +- **PTY lifecycle, agent launch matrix, prompt-seed stabilization** — unchanged. +- **`coda_run` and other tools** — untouched. +- **Three-mode framework table** — Mode 2 column "How invoked" stays the same; the spec for it now reads "any workspace folder, Git Folder or plain." + +## Tests to update + +In `tests/test_coda_interactive.py`: + +1. **Drop:** `test_unknown_workspace_path_returns_error` if it covered the `repos.list` empty-result case → replace with a `workspace.get_status` raises case. +2. **Drop:** `test_branch_update_succeeds` and `test_branch_update_fails` — branch param is gone. +3. **Drop:** any test asserting `"branch"` in the return JSON. +4. **Update:** the happy-path test mock — remove `client.repos.list` and `client.repos.update` setup; add `client.workspace.get_status` returning a directory-typed mock. +5. **Add:** `test_plain_workspace_folder_succeeds` — covers a `workspace.get_status` returning ObjectType.DIRECTORY for a path that is NOT a Repo. Should reach the export step and succeed. +6. **Add:** `test_workspace_path_not_directory_returns_error` — `workspace.get_status` returns a FILE-typed mock; tool returns `"not a directory"` error without creating a PTY. + +Expected test count delta: ~−3 / +2 = net −1 test. + +## Tests for the SDK validation step + +Since we're relying on `client.workspace.get_status` to validate, add a mock-level test that verifies: +- A non-existent path raises an exception from `get_status` → tool returns `"Workspace path not found"` error. +- A directory path returns object_type=DIRECTORY → tool proceeds. +- A file path returns object_type=FILE → tool returns `"not a directory"` error. + +These belong in the same file as the existing tool tests. + +## Out of scope (deferred) + +- **Single-file `workspace_path`.** Not supported. If a caller wants to ship a single file, they create a directory containing it. Keeps `_export_recursive` semantics simple. +- **Recovering branch info from a Git Folder for the response.** Not added — caller already knows the branch state, and surfacing it in the response would be ornamental. +- **`workspace.get_status` for the export-failed cleanup path.** The existing `try/except` around `export_workspace_tree` still runs; this change does not affect cleanup. + +## Migration notes + +PR #67 is open and not yet merged — no shipped consumers depend on the `branch` parameter. Removing it is safe. The PR description should note the API change. + +## Risks + +- **A caller that calls with `branch="main"`** (positional or kwarg) will now error with `TypeError: unexpected keyword argument 'branch'`. Acceptable because no consumer has shipped. The FastMCP runtime surfaces this as a tool-validation error on the caller side. +- **`workspace.get_status` adds one extra API call** to the happy path. Negligible — same network plane as the export calls that follow. + +## Acceptance criteria + +- `coda_interactive` accepts ANY Workspace directory path, Git Folder or plain. +- `coda_interactive` no longer accepts a `branch` parameter. +- The tool gives a clean error when the path doesn't exist or isn't a directory. +- All existing tests pass (after the test updates above). +- The PR description for #67 reflects the simpler contract. diff --git a/docs/superpowers/specs/2026-05-28-coda-interactive-mcp-tool-design.md b/docs/superpowers/specs/2026-05-28-coda-interactive-mcp-tool-design.md new file mode 100644 index 0000000..b395562 --- /dev/null +++ b/docs/superpowers/specs/2026-05-28-coda-interactive-mcp-tool-design.md @@ -0,0 +1,379 @@ +# Spec: `coda_interactive` MCP Tool + +**Status:** Draft, pre-critique-gate +**Date:** 2026-05-28 +**Branch:** `feat/coda-mcp-live-session-url` (same as Todo 1) +**Related:** `docs/superpowers/specs/2026-05-28-coda-run-replay-only-design.md` (Todo 1 — establishes the three-mode framework this spec slots into as Mode 2) + +> **Amended by:** [`docs/superpowers/specs/2026-05-28-coda-interactive-broaden-source-design.md`](2026-05-28-coda-interactive-broaden-source-design.md) — the `branch` parameter and the Git-Folder-only requirement have been removed. `coda_interactive` now accepts any Workspace directory (Git Folder or plain). The `repos.list` + `repos.update` flow described in Section 3 of this spec has been replaced by a single `workspace.get_status` directory check. The return shape no longer includes a `"branch"` key. + +## Goal + +Add a new MCP tool, `coda_interactive`, that lets an upstream MCP client (Genie Code, Claude Desktop, Cursor) hand off an in-flight coding session to a human via a CoDA viewer URL. The handoff carries: +- A **chosen coding agent** (`claude` by default; pluggable to `hermes`, `codex`, `gemini`, `opencode`) +- A **project source**: a Databricks Workspace Git Folder path, optionally on a specific branch +- A **kickoff prompt** that gets auto-typed into the agent as the first user message + +The human opens the URL, attaches to a live PTY where the agent is already loaded with the project as CWD and the prompt already typed, drives the session, and exits when done. The URL is the only handle — no `result.json`, no `coda_get_result`, no `coda_inbox` integration. + +## Why + +Mode 3 (`coda_run`) is fire-and-forget batch — the MCP caller can't iterate mid-task. Mode 1 (direct web UI) requires the human to already be inside CoDA and manually wire their project. Neither covers the "I was working in Genie Code on a repo and want to continue with a coding agent inside CoDA" workflow. + +`coda_interactive` is built for that handoff. **Critically, this design uses Databricks Workspace Git Folders as the source of truth** — Coda already has Databricks authentication via its existing PAT, so no new credentials need to be configured for the tool to clone repos. The MCP caller's Git Folder in Workspace is the durable artifact that survives between local sessions and Coda sessions. + +## The Three-Mode Framework (reminder) + +See Todo 1's spec for the canonical table. This spec finalizes Mode 2: + +| Mode | How invoked | PTY tag | Lifecycle | URL semantics | +|---|---|---|---|---| +| **1. Direct launch** | User opens web UI, creates a tab | (none) | 24h idle / WS-extends | No external URL | +| **2. `coda_interactive`** *(this spec)* | MCP client calls the tool, passes the URL to a human | `replay_only=False` | 24h idle / WS-extends | Live attach | +| **3. `coda_run`** | MCP client fires the tool, URL is post-hoc replay only | `replay_only=True` | Immediate teardown on hermes -z exit | Replay only | + +## Design + +### 1. Tool signature + +```python +@mcp.tool( + annotations=ToolAnnotations( + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + ), +) +async def coda_interactive( + prompt: str, # initial kickoff message; auto-typed as first user input + workspace_path: str, # required, e.g. "/Workspace/Users/me@db.com/projects/feature-X" + branch: str = "", # optional — if set, updates the Git Folder to this branch first + agent: str = "claude", # claude | hermes | codex | gemini | opencode + email: str = "", # X-Forwarded-Email passthrough (single-user app, kept for parity) +) -> str: + ... +``` + +**Return shape** (JSON string): +```json +{ + "status": "launched", + "viewer_url": "https://..aws.databricksapps.com/?session=pty-...", + "agent": "claude", + "project_dir": "/home/app/.coda/projects/pty-...", + "workspace_path": "/Workspace/Users/me@db.com/projects/feature-X", + "branch": "feature/X", + "instructions": "Open viewer_url to attach. The agent is loaded with the project files exported from Workspace and your kickoff prompt typed. Continue from there; type the agent's quit command (e.g. /quit) and then `exit` to end the session. Note: git history is NOT available in the session — files are an export, not a clone." +} +``` + +### 1a. Caller pre-condition: project must be in Databricks Workspace + +This is a contract the **upstream MCP caller** (Genie Code, Claude Desktop, etc.) is responsible for satisfying — `coda_interactive` cannot create a Git Folder, it can only consume one. + +**The caller must:** +1. Ensure the project of interest is a **Databricks Workspace Git Folder** (created via the workspace UI's "Create > Git Folder" or via the Repos API). Plain Workspace folders without a git remote backing will not work — the branch-update step has no remote to fetch from. +2. **Commit and push** any local working changes back to the Git Folder's remote (GitHub/GitLab/etc.) **before** calling `coda_interactive`. The export is a server-side snapshot — uncommitted local changes are invisible to Coda. +3. If a specific branch is needed, ensure that branch exists on the remote and is reachable by the Databricks Workspace's stored credentials for the Git Folder. + +**The MCP tool's `instructions` string surfaces this requirement to the calling LLM:** + +> Before calling `coda_interactive`, ensure the user's project is a Databricks Workspace Git Folder and that any in-progress changes have been pushed to the Git Folder's remote. The tool exports a server-side snapshot — uncommitted local changes will not appear in the Coda session. If unsure, prompt the user to push their changes first or pass `workspace_path` for a recently-synced Git Folder. + +This text becomes the tool's surfaced description in the FastMCP server's instruction block, alongside the existing `coda_run` guidance. + +On error: `{"status": "error", "error": ""}`. No partial state — if export fails or PTY creation fails, no PTY is created and no `viewer_url` is returned. + +### 2. Agent launch matrix + +Each agent has a known interactive-launch command (verified against the deployed setup scripts): + +| `agent` value | Launch command sent to PTY | +|---|---| +| `claude` (default) | `claude\n` | +| `hermes` | `hermes chat\n` | +| `codex` | `codex\n` | +| `gemini` | `gemini\n` | +| `opencode` | `opencode\n` | + +Unknown `agent` values return an error immediately — no Workspace API call, no PTY. + +### 3. Project source: Workspace Git Folder export + +Coda's existing Databricks authentication (PAT in `DATABRICKS_TOKEN`) is sufficient for both steps. No new tokens, no `repo_token` parameter, no GitHub credential plumbing. + +Working directory on Coda: `~/.coda/projects//`. + +**Step 3a — (Optional) Update the Git Folder to the requested branch.** Skip if `branch` is empty. + +**Side-effect note:** `repos.update(branch=...)` mutates the Git Folder's server-side state — the folder is now on the requested branch for *any* tools/processes accessing it (other notebooks, jobs, parallel `coda_interactive` calls, etc.). For Coda's single-user-app model this is acceptable: the user is the only one mutating the Git Folder. If multi-user support is ever added, this design must be revisited — likely by cloning a sibling Git Folder per session. + +```python +from databricks.sdk import WorkspaceClient +w = WorkspaceClient() # picks up DATABRICKS_HOST + DATABRICKS_TOKEN from env + +# Resolve the Repos / Git Folder ID from the workspace_path +repos = w.repos.list(path_prefix=workspace_path) +repo = next((r for r in repos if r.path == workspace_path), None) +if repo is None: + return {"status": "error", "error": f"No Git Folder found at {workspace_path}"} + +# Update to the requested branch — Databricks performs the actual fetch + checkout server-side +w.repos.update(repo_id=repo.id, branch=branch) +``` + +**Step 3b — Export the file tree into Coda's local disk.** + +The Databricks Workspace API exposes a `workspace export-dir`-equivalent through the SDK: + +```python +import os +project_dir = os.path.join(os.path.expanduser("~/.coda/projects"), pty_session_id) +os.makedirs(project_dir, exist_ok=True) + +# Recursive export — files only, no `.git` directory. +# (Implementation may use the workspace.export() loop or shell out to `databricks workspace export-dir`.) +_export_workspace_tree(w, workspace_path, project_dir) +``` + +`_export_workspace_tree` is a small helper that: +1. Lists the workspace_path recursively (`w.workspace.list(workspace_path)` with recursive traversal) +2. For each file: calls `w.workspace.export(path=..., format=ExportFormat.SOURCE)` and writes the content to the local mirror +3. Preserves directory structure +4. Handles files (NOT notebooks — notebooks export to `.py`/`.ipynb` via the `SOURCE` format) + +Implementation note: if the SDK's recursive-export is awkward, fall back to shelling out: `subprocess.run(["databricks", "workspace", "export-dir", workspace_path, project_dir, "--overwrite"], check=True, capture_output=True, timeout=300)`. The CLI is preconfigured on Coda. Either approach is acceptable; the planner will pick after a small spike. + +**Important:** Only the working tree is exported. The `.git/` directory is NOT included — Workspace Git Folders manage git state server-side and don't expose `.git` via the API. Git history is unavailable inside the session. This trade-off is acknowledged in §7 (Out of Scope) and surfaced to the caller via the `instructions` field in the response. + +**Snapshot semantics:** `workspace.export()` reflects the **committed HEAD state** of the Git Folder — not any uncommitted changes that exist in the Databricks Workspace UI editor. If the caller's user has uncommitted edits in the Workspace UI for this Git Folder, those changes will NOT appear in the Coda session. This is the same constraint the caller pre-condition (§1a) communicates: push commits first. + +**Binary file handling:** `workspace.export(format=ExportFormat.SOURCE)` may fail (HTTP 400) on binary files (images, PDFs, compiled artifacts). The export helper must wrap each per-file export in a try/except and skip-and-log files that error out, rather than aborting the entire export. The agent in the session gets a partial tree (text/source files); the human can decide whether the missing binaries matter. + +**Empty export:** If the Workspace Git Folder is empty OR if all files are non-exportable, the project dir ends up empty after the export. The PTY is still launched (the agent will sit in an empty dir). This is acceptable — the human can investigate via the agent. + +Export timeout: 300 s (5 min). Big repos may need bumping later; not parameterizable in MVP. + +### 4. Prompt seeding + +After the PTY is created and the agent launched: + +```python +import time +# Wait briefly for the agent to initialize and present its prompt. +time.sleep(2) + +# Type the prompt into the PTY as the first user message. +_app_send_input(pty_session_id, prompt + "\n") +``` + +The 2 s delay is a pragmatic choice — agents typically print a banner + prompt within that window. If the timing misses on slow startup, the prompt still lands; the agent sees it as part of the kickoff. No assertion that the agent is "ready" — that's a brittle race we don't need. + +### 5. PTY + project lifecycle + +`coda_interactive` PTYs inherit Mode 1's lifecycle exactly: +- Created with `replay_only=False` +- 24h idle TTL via existing `SESSION_TIMEOUT_SECONDS = 86400` cleanup +- WS heartbeat extends while the human is attached +- Teardown via human typing `exit` (which closes bash, which EOFs the PTY) OR 24h idle + +**Cleanup hook:** `mcp_close_pty_session(pty_id)` (in `app.py`) gains a side-effect: if `~/.coda/projects//` exists, delete it (recursively) after closing the PTY. Single cleanup path means the disk lifecycle matches the PTY lifecycle — no new timer or state. + +### 6. Where this lives in the codebase + +- Modified: `coda_mcp/mcp_server.py` — add `coda_interactive` tool definition next to `coda_run`. **Also update the FastMCP `instructions` string** (currently around lines 43-70) to add a paragraph describing `coda_interactive` so calling LLMs don't treat it like `coda_run` (e.g., don't try to poll for results). The new paragraph must include: the pre-condition that the project must be a Workspace Git Folder, the contract that interactive sessions don't appear in `coda_inbox`, and a note that `coda_get_result` won't return anything for these sessions. +- Modified: `app.py` — extend `mcp_close_pty_session` to clean up the project dir; add `cwd` kwarg to `mcp_create_pty_session` so the spawned bash starts in the project dir. **Prerequisite refactor (security-relevant):** `mcp_create_pty_session`'s inline env-stripping at `app.py:1435-1441` only strips a handful of keys (CLAUDECODE, CLAUDE_CODE_SESSION, DATABRICKS_TOKEN, DATABRICKS_HOST, GEMINI_API_KEY). The HTTP `create_session` route uses `_build_terminal_shell_env(os.environ)` which ALSO strips `NPM_TOKEN`, `UV_DEFAULT_INDEX`, `UV_INDEX_*_PASSWORD`, `UV_INDEX_*_USERNAME`, and `npm_config_//*` registry credential patterns. Today, any MCP-created PTY (including `coda_run`'s) leaks these registry credentials to the child shell via `env`. Fix this as a prerequisite to Todo 2: refactor `mcp_create_pty_session` to call `_build_terminal_shell_env(os.environ)` instead of the inline copy. Zero behavioral impact on the happy path; closes a latent security gap. +- Modified: `coda_mcp/mcp_endpoint.py` — register `coda_interactive` in the Flask-fallback tool dispatch (parity with how `coda_run` is wired). +- New helper: `coda_mcp/workspace_export.py` — encapsulates the Workspace-tree-to-local-dir export logic. Keeps `mcp_server.py` focused on tool orchestration. +- New tests: `tests/test_coda_interactive.py` covering signature validation, branch update, export, agent allow-list, prompt seeding, cleanup on PTY close. Plus `tests/test_workspace_export.py` for the helper. Plus `tests/test_mcp_env_strip.py` (or extending an existing env-strip test file) to assert `mcp_create_pty_session` properly strips registry credentials post-refactor. + +**Implementation note on SDK calls:** `WorkspaceClient()` is constructed inside the `coda_interactive` tool function (in the server process). The SDK calls happen BEFORE `mcp_create_pty_session` is invoked, so they execute with the full server environment (including `DATABRICKS_TOKEN`). The PTY child shell's env is separately filtered via `_build_terminal_shell_env` and does NOT receive the Databricks token (which is the correct behavior — we don't want agents in the PTY to see deployer credentials). Future implementers must not move the SDK calls into the PTY subprocess. + +### 7. What does NOT change + +- `coda_run` is untouched (Todo 1 already finalized). +- `coda_inbox` and `coda_get_result` ignore `coda_interactive` PTYs (no task records get written for them). +- The Mode 1 web-UI launch path is untouched. +- `replay_only` flag plumbing from Todo 1 — `coda_interactive` passes `replay_only=False`, which is already the default. +- `MAX_CONCURRENT_SESSIONS` enforcement — `coda_interactive` PTYs count against the cap exactly like Mode 1 sessions do. + +## Architecture + +``` + ┌──────────────────────────────────────────┐ + │ MCP client calls coda_interactive │ + │ (prompt, workspace_path, branch, agent) │ + └────────────────┬─────────────────────────┘ + ▼ + ┌────────────────────────────────────────────────────┐ + │ Validate agent ∈ allow-list │ + │ [if branch]: w.repos.update(branch=branch) │ + │ _export_workspace_tree(w, ws_path, project_dir) │ + └────────────────────┬───────────────────────────────┘ + ▼ + ┌────────────────────────────────────────────────────┐ + │ pty_session_id = mcp_create_pty_session( │ + │ label="-interactive", │ + │ replay_only=False, │ + │ cwd=project_dir, # NEW kwarg │ + │ ) │ + │ _app_send_input(pty_session_id, "\n") │ + │ time.sleep(2) │ + │ _app_send_input(pty_session_id, prompt + "\n") │ + │ return {viewer_url, agent, ...} │ + └────────────────────┬───────────────────────────────┘ + ▼ + (Human opens viewer_url; attaches to a live PTY + already cd'd into the exported project, agent + running, kickoff prompt already typed.) + ▼ + ┌────────────────────────────────────────────────────┐ + │ Human types `/quit` (agent) and `exit` (shell), OR │ + │ 24h idle reaper fires │ + │ → mcp_close_pty_session(pty_id) │ + │ → shutil.rmtree(~/.coda/projects/pty_id/) │ + └────────────────────────────────────────────────────┘ +``` + +**New `cwd` kwarg on `mcp_create_pty_session`:** required so the PTY's bash spawns in the exported project dir. Default is the existing behavior (bash uses `$HOME`). Additive change; no other callers need updates. + +## Data flow scenarios + +**Happy path:** +1. User is working locally in their Workspace Git Folder. Pushes recent commits via the Git Folder UI or via the post-commit hook from their existing local Coda environment. +2. MCP client (Genie Code) calls `coda_interactive(prompt="continue debugging the auth flow", workspace_path="/Workspace/Users/me@db.com/projects/auth-feature", branch="feature/auth", agent="claude")` +3. Server validates agent; updates Git Folder to `feature/auth` via Repos API (Databricks does the git fetch); exports tree to `~/.coda/projects//`; creates PTY in that dir; launches `claude`; types the prompt. +4. Returns `viewer_url` +5. Human opens URL → attaches to live Claude session in the exported project, with kickoff prompt already in the chat +6. Human iterates with Claude; eventually exits the agent and the shell +7. PTY teardown deletes the project dir + +**Branch update failure:** +1. MCP client passes a nonexistent `branch` +2. `w.repos.update(...)` raises (Databricks API returns 4xx) +3. Server returns `{"status": "error", "error": "Failed to update Git Folder to branch X: "}` +4. No export, no PTY, no leak + +**Workspace path not found:** +1. MCP client passes a `workspace_path` that isn't a Git Folder or doesn't exist +2. The `repos.list(...)` lookup returns no match, OR the workspace API returns 404 +3. Server returns `{"status": "error", "error": "No Git Folder found at "}` +4. No PTY, no leak + +**Agent allow-list rejection:** +1. MCP client passes `agent="vim"` +2. Server returns `{"status": "error", "error": "Unknown agent: vim. Allowed: claude, hermes, codex, gemini, opencode"}` +3. No Workspace API call, no PTY + +**Concurrent-session limit:** +1. `MAX_CONCURRENT_SESSIONS` already at cap when call arrives +2. Server returns `"Maximum 5 concurrent sessions reached."` (same shape as `coda_run`) +3. No export, no PTY + +**Human never attaches:** +1. PTY sits at the agent's prompt, with the kickoff already typed +2. 24h elapses → existing idle cleanup reaps the PTY +3. Project dir deleted as part of `mcp_close_pty_session` + +**Human attaches, drives, but closes tab without exiting agent:** +1. WS heartbeat stops +2. 24h idle countdown begins +3. If human reopens within 24h: WS resumes, session continues +4. Else: idle cleanup, project dir cleanup, done + +## Error handling + +| Error | Returned to MCP client | Server-side cleanup | +|---|---|---| +| Unknown `agent` value | `{"status":"error","error":"Unknown agent: ..."}` | None needed | +| `workspace_path` doesn't exist / not a Git Folder | `{"status":"error","error":"No Git Folder found at "}` | None needed | +| `repos.update(branch=...)` fails (bad branch, network) | `{"status":"error","error":"Failed to update Git Folder to branch X: "}` | Remove partial project dir | +| Export fails midway (disk full, network) | `{"status":"error","error":"Failed to export workspace tree: "}` | Remove partial project dir | +| `MAX_CONCURRENT_SESSIONS` reached | `{"status":"error","error":"Maximum N concurrent sessions reached."}` | None needed | +| PTY creation fails | `{"status":"error","error":"Failed to allocate PTY: "}` | Remove project dir | + +No `result.json` is written — no watcher, no completion machinery. Cleanup happens via the PTY's own teardown path. + +## Testing strategy + +### Unit tests (no PTY, mock Databricks SDK) + +1. `test_coda_interactive_unknown_agent_returns_error` — `agent="vim"` → status=error, no SDK call +2. `test_coda_interactive_missing_workspace_path_returns_error` — empty `workspace_path` → error +3. `test_coda_interactive_workspace_not_found` — mock `repos.list()` returns empty → status=error +4. `test_coda_interactive_branch_update_failure_returns_error` — mock `repos.update()` raises → error + no PTY +5. `test_coda_interactive_export_failure_cleans_partial_dir` — mock export raises mid-way → partial dir is removed +6. `test_coda_interactive_skips_branch_update_when_empty` — mock confirms `repos.update()` is NOT called when `branch=""` + +### Integration tests (PTY-gated via `_pty_skip`, with mocked Databricks SDK) + +7. `test_coda_interactive_happy_path_mocked_export` — mock the Workspace SDK to "export" a fake tree into the local dir, assert PTY is created with the right CWD, agent command is sent, prompt is typed. +8. `test_coda_interactive_concurrent_limit` — fill up `MAX_CONCURRENT_SESSIONS` → call returns error +9. `test_mcp_close_pty_session_removes_project_dir` — create PTY with project dir, close it, assert dir deleted +10. `test_mcp_close_pty_session_handles_missing_project_dir` — no project dir present → close still succeeds (no exception) +11. `test_mcp_create_pty_session_respects_cwd_kwarg` — bash spawns in the requested dir + +### Helper tests + +12. `tests/test_workspace_export.py`: tests for `_export_workspace_tree` covering: nested dirs, file content fidelity, empty dirs, files-only (skips notebooks), error handling for individual file export failures. + +### Regression guard + +13. `test_coda_run_does_not_create_project_dir` — calling `coda_run` doesn't touch `~/.coda/projects/`. Defends the lifecycle separation between Modes 2 and 3. + +## Out of scope (for Todo 2) + +- **Git history inside the session.** Files-only export. Inside the PTY, `git log`, `git diff`, `git blame` return nothing. If history matters for a particular session, the MCP caller can include a `git log --oneline -50` summary in the `prompt` string. A future Todo can layer on a git-clone path with token-based auth. +- **Notebooks as `.ipynb`.** The export uses `ExportFormat.SOURCE` which converts Databricks notebooks to `.py` (or equivalent). MVP doesn't attempt to round-trip notebooks back to Workspace; agents work on the exported source files. +- **Conversation history transfer from the MCP client's local session.** Not in scope. Caller summarizes context into `prompt`. +- **Listing live `coda_interactive` sessions via `coda_inbox`.** URL is the only handle. +- **`coda_get_result` for interactive sessions.** No result.json, no inbox entry. +- **Incremental Workspace updates during the session.** If the user wants to pull newer changes mid-session, they'd need to push to Workspace and re-launch `coda_interactive`. No in-session sync mechanism. +- **Multiple-agent sessions in one PTY.** One agent per call. +- **Non-Workspace sources** (raw zips, external git remotes). Future Todo if needed. +- **Pushing changes BACK from the session to Workspace.** The agent can run Coda's existing post-commit hook (which syncs `~/projects/` to Workspace), but the exported dir at `~/.coda/projects//` is OUTSIDE that hook's scope by design — we don't want every interactive session to clobber Workspace state. If write-back is needed, that's a follow-up design. + +## Migration / Rollout + +- Single commit chain on the `feat/coda-mcp-live-session-url` branch on top of Todo 1's work. +- No data migration: new tool, no existing state to update. +- No config flag — the new tool is unconditionally available once the code lands. +- App restart picks up the new tool registration. +- MCP clients (Genie Code, etc.) will see the new tool listed via `tools/list` and can call it immediately. + +## Critique gate + +**Cleared** (2026-05-28). Critic verdict: APPROVE WITH CHANGES. All flagged issues incorporated above: + +- **MAJOR** — pre-existing env-strip gap in `mcp_create_pty_session` (misses `NPM_TOKEN`, `UV_DEFAULT_INDEX`, `UV_INDEX_*_PASSWORD`, etc.) → added as prerequisite refactor task in §6 +- **HIGH-PRIORITY GAP** — FastMCP `instructions` string update for the new tool → added explicitly in §6 +- Section 3 expanded with snapshot-semantics, binary-file handling, and empty-export notes +- Section 3a expanded with multi-user side-effect caveat +- Section 6 expanded with SDK-call placement note (calls happen in server process, not PTY subprocess) +- Tool description text guidance integrated (instructions string must mention `coda_inbox` invisibility, no `coda_get_result` integration, Git Folder pre-condition) + +Original 10 critique questions, all answered in the critique pass: + +1. **Auth model** — Confirmed. Coda's PAT covers both `repos.update()` and `workspace.export()`; no scope gotcha for single-user. +2. **Export performance** — Both SDK loop and `databricks workspace export-dir` CLI are viable; planner picks after a small spike. CLI is faster. +3. **Git Folder vs. ordinary folder** — Hard error is correct. `repos.list()` returns empty for non-Git folders; clear error message. +4. **Branch update side effect** — Acceptable for single-user app; multi-user caveat added to §3a. +5. **Notebook handling** — `ExportFormat.SOURCE` converts notebooks to `.py`/`.scala`/`.sql`. Acceptable; out-of-scope to round-trip back to notebooks. +6. **Concurrent branch race** — Acceptable for single-user; documented as user error. +7. **Disk lifecycle** — UUID-based session IDs prevent collisions; rmtree failure orphans the dir but doesn't break next session. +8. **Prompt seeding** — 2-second sleep is pragmatic; bash buffers stdin if agent is slow to read. +9. **`cwd` kwarg** — Only `coda_interactive` needs it. Additive change, no other callers affected. +10. **Test coverage** — Mocked Databricks SDK is the right MVP approach; E2E against real workspace deferred as nice-to-have behind CI flag. + +Plus eight additional critic-eye questions (11–18), all resolved: + +11. **Mode separation drift** — No drift. Regression guard test (`test_coda_run_does_not_create_project_dir`) defends the separation. +12. **PTY exhaustion** — Production PTY limit is ~4096; `MAX_CONCURRENT_SESSIONS=5` is nowhere near. macOS dev exhaustion is a known local-test concern, handled via `_pty_skip`. +13. **Project dir collision** — UUID-based IDs make collision probability negligible; `exist_ok=True` on `makedirs` handles the unlikely case. +14. **Pre-condition realism** — Realistic for Genie Code (primary target); secondary clients (Claude Desktop, Cursor) get clear guidance via `instructions` string. +15. **Dirty Workspace UI state** — Export reflects committed HEAD; uncommitted UI edits NOT included. Documented in §3 snapshot-semantics note. +16. **Binary files** — Per-file try/except + skip-and-log added to §3 binary-file note. +17. **`coda_inbox` invisibility** — Documented in `instructions` string per §6. +18. **Tool description text** — Spelled out in §6 (instructions string must explain the new tool's contract). + +Spec is ready for planning. diff --git a/docs/superpowers/specs/2026-05-28-coda-interactive-terminal-pull-design.md b/docs/superpowers/specs/2026-05-28-coda-interactive-terminal-pull-design.md new file mode 100644 index 0000000..d6df636 --- /dev/null +++ b/docs/superpowers/specs/2026-05-28-coda-interactive-terminal-pull-design.md @@ -0,0 +1,219 @@ +# Spec: `coda_interactive` Terminal-Side Workspace Pull + +**Status:** Draft, design-critic passed (SOUND-WITH-FIXES, all fixes folded in) +**Date:** 2026-05-28 +**Branch:** `feat/coda-mcp-interactive-handoff` (continues PR #67) +**Supersedes the export mechanism in:** `docs/superpowers/specs/2026-05-28-coda-interactive-mcp-tool-design.md` + +## Problem + +`coda_interactive` currently does a **server-side** export of a Databricks Workspace folder into a local project directory via `WorkspaceClient().workspace.export(...)` (module `coda_mcp/workspace_export.py`), then launches an agent in that directory. In the deployed app this produces an **empty directory** — the agent has no idea about the user's files. + +### Confirmed root cause + +The deployed app runs as its **own service principal** (`app-167dcd mcp-test-coda-labs-feat`, client_id `460e920e-…`), confirmed via the Apps API. The MCP server calls `WorkspaceClient()` with no args → that resolves to the **app SP**. The app SP can `get_status` the user's `/Users//WAM` folder (so the tool reports `"launched"`) but **cannot `list`/`export` its contents**. `workspace_export.py._export_recursive` **swallows** those errors (`logger.warning` + `return`), so `export_workspace_tree` raises nothing and the agent launches over an empty directory. + +### Evidence + +- **REST as the user** (curl): `list`, `get_status`, and `export` (SOURCE and AUTO, with and without `/Workspace` prefix) all succeed for the 5 `.md` files in WAM. So the API, the export format, and the path prefix are NOT the problem. +- **Live CoDA terminal:** `databricks current-user me` returns the **user** (`sathish.gangichetty@databricks.com`), not the app SP. `databricks workspace list /Users/.../WAM` from the terminal returns the 5 files. +- **Conclusion:** the identity that can read the files is the **terminal** (the app owner / user), not the **MCP server** (app SP). Move the file access to the terminal. + +## Goal + +Stop exporting server-side. `coda_interactive` hands the location to the **terminal** (authenticated as the user) and pulls the files there with `databricks workspace export-dir`, then launches the agent in the pulled directory. Net effect: the agent starts in a directory that actually contains the workspace files, and any failure is visible (a real tool error or terminal output) instead of silently swallowed. + +## Non-goals + +- `/Workspace` FUSE-mount access — `export-dir` works regardless of whether the mount exists. Not pursued. +- Pushing edits back to the Workspace (`import-dir`) — the agent can do that itself if asked. Out of scope. +- Git Folder branch checkout — caller's responsibility, as before. +- Changing `coda_run` (mode 3) or any other tool. +- Hardening the existing `_wait_for_agent_ready` heuristic beyond what this change needs (see Risks). + +--- + +## Design + +### New `coda_interactive` flow + +``` +1. Validate `agent` ∈ _ALLOWED_AGENTS (unchanged) +2. Verify PTY hooks wired (_app_create_session/_app_send_input) (unchanged) +3. pty_session_id = _app_create_session(label=f"{agent}-interactive", replay_only=False) +4. project_dir = os.path.join(os.path.expanduser("~/.coda/projects"), pty_session_id) + os.makedirs(project_dir, exist_ok=True) +5. name = _safe_dirname(workspace_path) # e.g. "WAM" + source_path = _normalize_workspace_path(workspace_path) # strip leading /Workspace +6. Type ONE chained line into the PTY (runs as the user): + cd && databricks workspace export-dir ./ && cd +7. await _wait_for_output_stable(pty, _EXPORT_MAX_WAIT_S, _EXPORT_STABILITY_S) + # wait for the pull to finish — shell goes truly idle after export-dir, + # so stabilization here is reliable (no agent-cold-start gap to confuse it) +8. SERVER-SIDE post-condition check (does NOT depend on the app SP — stats local disk): + target_dir = os.path.join(project_dir, name) + if not os.path.isdir(target_dir) or not os.listdir(target_dir): + close PTY; shutil.rmtree(project_dir, ignore_errors=True) + return {"status":"error", "error": ""} +9. Launch the agent (fresh — identical to the proven existing path): + _app_send_input(pty, _AGENT_LAUNCH_CMDS[agent] + "\n") + await _wait_for_agent_ready(pty) # existing 5s/1s window, unchanged behavior +10. Paste kickoff prompt, prefixed with a context line naming workspace_path: + "Your working directory contains files exported from the Databricks + Workspace path .\n\n" +11. return {"status":"launched", "viewer_url", "agent", "project_dir": target_dir, + "workspace_path", "instructions"} +``` + +### Why split the waits (design-critic CRITICAL fix) + +The naive design (`cd && export-dir && cd`, then launch agent, then a single `_wait_for_agent_ready`) risks `_wait_for_agent_ready` returning **early** in the silent gap between `export-dir` finishing and the agent's TUI producing output — pasting the prompt into a half-initialized agent or the shell. + +The split removes that risk: +- **Step 7** waits for the *pull* to finish. After `export-dir` completes the shell is genuinely idle (output stops), so stabilization is reliable. It is NOT waiting across an agent cold-start. +- **Step 9** waits for the *agent* exactly the way the current working code does (launch → wait → prompt), with no preceding network op. It inherits the known-good behavior. +- **Step 8** (the filesystem post-check) is the safety net: if the pull produced nothing, we error out cleanly instead of launching into an empty directory. This also resolves the `&&`-failure ambiguity — a failed `export-dir` short-circuits the chain, leaves `target_dir` absent, and step 8 turns that into a proper tool error. + +### Helpers + +```python +def _safe_dirname(workspace_path: str) -> str: + """Local directory name for the pulled folder = sanitized basename.""" + base = os.path.basename(workspace_path.rstrip("/")) + safe = re.sub(r"[^A-Za-z0-9._-]", "_", base) + return safe or "workspace" + + +def _normalize_workspace_path(workspace_path: str) -> str: + """Canonical Workspace API path: drop the /Workspace FUSE prefix if present. + + The deployed terminal's CLI uses the unprefixed form (/Users/...); REST + accepts both, but normalizing matches what the CLI expects and is harmless. + """ + p = workspace_path.rstrip("/") + if p.startswith("/Workspace/"): + p = p[len("/Workspace"):] # "/Workspace/Users/x" -> "/Users/x" + return p +``` + +### Wait-helper refactor (backward compatible) + +Generalize the existing poller so the export wait can use a longer budget while `coda_run`'s call site stays unchanged: + +```python +_PROMPT_SEED_MAX_WAIT_S = 5.0 # existing — agent TUI settle +_PROMPT_SEED_STABILITY_S = 1.0 # existing +_EXPORT_MAX_WAIT_S = 120.0 # new — generous; export-dir prints per-file so it won't prematurely stabilize on a slow pull +_EXPORT_STABILITY_S = 1.5 # new + +async def _wait_for_output_stable(pty_session_id, max_wait, stability): + # exact body of the current _wait_for_agent_ready, parametrized on max_wait/stability + +async def _wait_for_agent_ready(pty_session_id): + await _wait_for_output_stable(pty_session_id, _PROMPT_SEED_MAX_WAIT_S, _PROMPT_SEED_STABILITY_S) +``` + +`coda_run` already calls `_wait_for_agent_ready` — that call and its behavior are unchanged. + +### `databricks workspace export-dir` (verified) + +`databricks workspace export-dir SOURCE_PATH TARGET_PATH`: +- Exports a directory recursively from the Workspace to the local filesystem. +- **Creates** `TARGET_PATH`. +- Auto-appends notebook extensions (`.py/.scala/.sql/.r`) by language — natively replaces the hand-rolled logic in `workspace_export.py`. +- `--overwrite` flag exists; not needed here (the session `` dir is fresh). + +### Deletions + +- `coda_mcp/workspace_export.py` — whole module. +- `tests/test_workspace_export.py` — whole file. +- In `coda_mcp/mcp_server.py`: remove `from coda_mcp.workspace_export import export_workspace_tree, _is_directory`, the `WorkspaceClient` import guard (verify no other use first), the `WorkspaceClient()` instantiation, the `get_status` validation, and the `_is_directory` call. +- `tests/test_replay_only_flag.py:166` — only a **comment** mentions `export_workspace_tree` (not an import). Refresh the wording so it doesn't reference a deleted symbol. Non-breaking. + +### Kept + +PTY creation (`replay_only=False`), `project_dir` + `os.makedirs`, `_wait_for_agent_ready` (now a wrapper), `viewer_url`, `_ALLOWED_AGENTS`, `_AGENT_LAUNCH_CMDS`, the existing try/except resource cleanup. `email` stays in the signature (upstream callers pass it; currently unused, reserved). + +### Cleanup on session end (no new code) + +`app.py:terminate_session` already `shutil.rmtree`s `os.path.expanduser("~/.coda/projects/")` on both graceful exit and idle-reaper paths. The pulled `` dir lives inside `project_dir`, so it is cleaned up automatically. + +--- + +## Error handling + +| Situation | Behavior | +|-----------|----------| +| Unknown `agent` | Immediate `{"status":"error"}` (unchanged) | +| PTY hooks not wired | Immediate `{"status":"error"}` (unchanged) | +| Bad `workspace_path` / no access / empty folder | `export-dir` fails or pulls nothing → step-8 FS check fails → close PTY, rmtree, `{"status":"error", "error": "No files were pulled from ; check it exists and you have read access."}` | +| Pull succeeds | Agent launches in `target_dir`; prompt seeded; `{"status":"launched", viewer_url, ...}` | +| Unexpected exception anywhere | Catch-all: close PTY if created, rmtree `project_dir`, `{"status":"error"}` (unchanged) | + +No server-side path validation via `WorkspaceClient` — the app SP can't reliably validate the user's folder anyway (that was the bug). The step-8 FS check is the validation, and it reads the local disk the *terminal* wrote (correct identity). + +--- + +## Testing strategy + +### `tests/test_workspace_export.py` — DELETE + +### `tests/test_replay_only_flag.py` — refresh the stale comment at line 166 (no logic change) + +### `tests/test_coda_interactive.py` — rewrite + +Mock `_app_create_session` (returns a fake `pty_session_id`), `_app_send_input` (records inputs; on the pull command, side-effect creates `target_dir` + a dummy file to simulate a successful `export-dir`), `_app_close_session`, and the wait helpers (return immediately). Set `HOME` to a `tmp_path` so `project_dir` resolves under the test sandbox. + +| Test | Pins | +|------|------| +| `test_pull_command_is_sent_first` | First `_app_send_input` is the chained `cd … && databricks workspace export-dir ./ && cd `; source has no `/Workspace` prefix; `` is the sanitized basename | +| `test_agent_launches_after_successful_pull` | After the simulated pull creates files, the launch command (`_AGENT_LAUNCH_CMDS[agent]`) is sent | +| `test_prompt_seeded_with_context_line` | Final input starts with the "exported from the Databricks Workspace path " line, then the user prompt | +| `test_empty_pull_returns_error_and_no_launch` | When the pull side-effect creates nothing, result is `{"status":"error"}`, PTY is closed, and the launch command is NEVER sent | +| `test_no_workspaceclient_or_get_status_called` | `WorkspaceClient` is not referenced (import removed); no `get_status` call path | +| `test_happy_path_returns_launched_with_viewer_url` | `{"status":"launched"}`, `viewer_url` present, `project_dir` == `target_dir` | +| `test_unknown_agent_rejected` | Unknown agent → error (unchanged) | +| `test_pty_hook_not_wired` | Hooks `None` → error (unchanged) | +| `test_agent_matrix` | Each of claude/hermes/codex/gemini/opencode sends the right launch cmd | +| `test_no_blocking_sleep` | `coda_interactive` source contains no `time.sleep(` (async regression guard, kept) | + +### `tests/test_mcp_server.py` (or wherever helpers are tested) — add + +| Test | Pins | +|------|------| +| `test_safe_dirname_basename` | `/Users/x/WAM` → `WAM`; trailing slash stripped | +| `test_safe_dirname_sanitizes` | spaces / special chars → `_` | +| `test_safe_dirname_empty_fallback` | `"/"` or `""` → `"workspace"` | +| `test_normalize_strips_workspace_prefix` | `/Workspace/Users/x/WAM` → `/Users/x/WAM` | +| `test_normalize_leaves_plain_path` | `/Users/x/WAM` → `/Users/x/WAM` | +| `test_wait_for_agent_ready_still_wrapper` | `_wait_for_agent_ready` delegates to `_wait_for_output_stable` with the prompt-seed constants | + +### Regression + +Run together (per the established flake note — `test_replay_only_flag.py::test_coda_run_creates_pty_with_replay_only_true` is PTY-fd flaky in multi-file runs; re-run alone if it fails): + +``` +uv run pytest tests/test_coda_interactive.py tests/test_mcp_server.py tests/test_replay_only_flag.py tests/test_task_manager.py tests/test_databricks_preamble.py -v +``` + +--- + +## Acceptance criteria + +1. `coda_interactive` no longer imports or calls `workspace_export` / `WorkspaceClient` / `get_status`. +2. `coda_mcp/workspace_export.py` and `tests/test_workspace_export.py` are deleted; no remaining importers. +3. `_safe_dirname` and `_normalize_workspace_path` exist with the specified behavior. +4. `_wait_for_output_stable(pty, max_wait, stability)` exists; `_wait_for_agent_ready` is a wrapper preserving the `5.0/1.0` budget; `coda_run`'s call is unaffected. +5. The first PTY input is the chained pull command using the normalized (unprefixed) source path and the sanitized ``. +6. The agent launch command is sent **only** when the post-pull FS check finds files; otherwise a `{"status":"error"}` is returned and the PTY is closed. +7. The kickoff prompt is prefixed with the context line naming `workspace_path`. +8. All new/updated tests pass; existing suites (minus the known PTY-fd flake) stay green. + +--- + +## Risks + +1. **Slow / huge folders.** `_EXPORT_MAX_WAIT_S = 120s`; if a pull exceeds it, step 7 returns while `export-dir` is still running and step 8 may see a partial dir and (incorrectly) proceed. Mitigation: 120s is generous for the interactive-handoff use case (docs / small projects); `export-dir` prints per-file so it won't prematurely stabilize during an active pull. Larger-folder support is a future tweak, not in scope. +2. **HOME equivalence.** Step 4/8 resolve `project_dir` via `os.path.expanduser` in the MCP-server process; the PTY `cd`/write uses that same absolute string and the terminal's `$HOME` resolves identically in the deployed container (observed: both `/app/python/source_code/.coda/...`). If a future environment gave the server and PTY different `$HOME`, the `cd` and FS check would diverge. Documented assumption; matches existing code (the deleted export and `terminate_session` cleanup already rely on it). +3. **`_wait_for_agent_ready` cold-start (pre-existing).** The agent wait can still, in principle, fire during a long agent cold-start silence — but this is the current production behavior, unchanged by this spec. A marker-based ready gate is a possible future hardening, explicitly out of scope here. +4. **`export-dir` on `/Workspace`-prefixed paths.** Mitigated by `_normalize_workspace_path` (we pass the `/Users/...` form the CLI expects and that REST verified). diff --git a/docs/superpowers/specs/2026-05-28-coda-run-replay-only-design.md b/docs/superpowers/specs/2026-05-28-coda-run-replay-only-design.md new file mode 100644 index 0000000..766c749 --- /dev/null +++ b/docs/superpowers/specs/2026-05-28-coda-run-replay-only-design.md @@ -0,0 +1,255 @@ +# Spec: `coda_run` Returns Replay-Only URL + +**Status:** Draft, pre-critique-gate +**Date:** 2026-05-28 +**Branch:** `coda-mcp` +**Related:** PR #66 (introduced the live-attach `viewer_url` we are now narrowing) ; `docs/superpowers/specs/2026-05-27-coda-mcp-live-session-url-design.md` (predecessor design) + +## Goal + +Make `coda_run`'s returned `viewer_url` resolve to a **read-only static replay** of the agent's transcript, never to a live PTY attach. As a consequence, drop the 5-minute "grace period" machinery from the `coda_run` execution path entirely — the PTY session can be torn down immediately when `hermes -z` exits. + +## Why + +PR #66 introduced a dual-purpose `viewer_url` on `coda_run`: live attach during a 5-minute grace window, then static replay after that. The dual mode was sized for "human watches hermes run live, then post-mortem replays the same URL". + +That use case is being split out into a **separate** MCP tool, `coda_interactive` (designed in a follow-up spec). `coda_run` is now exclusively the fire-and-forget batch surface — autonomous execution, post-hoc inspection. The live-attach affordance on its returned URL is no longer useful: by the time most callers' humans click the URL, hermes has already exited; what they get is a dead bash shell, not a live agent. + +## The Three-Mode Framework + +This spec settles the contract by enumerating the three ways CoDA sessions get created: + +The existing PTY lifecycle in `app.py` (`SESSION_TIMEOUT_SECONDS = 86400`, `CLEANUP_INTERVAL_SECONDS = 900`) **already gives sessions a 24h idle TTL** with WS-heartbeat extension. Mode 2 inherits this directly; only Mode 3 needs to deviate (faster teardown). + +| Mode | How invoked | PTY tag | Pre-attach lifecycle | Post-attach lifecycle | Teardown trigger | URL semantics | +|---|---|---|---|---|---|---| +| **1. Direct launch** | User opens web UI, creates a tab | (none) | n/a — user starts attached | 24h idle cleanup; WS heartbeat extends indefinitely | Tab close / disconnect + 24h idle | No external URL | +| **2. `coda_interactive`** (Todo 2, not in this spec) | MCP client fires the tool, passes URL to a human | `replay_only=False` | Same 24h idle cleanup as Mode 1 | Same — WS heartbeat extends | Agent process exit (`exit` / `/quit` / Ctrl-D), 24h idle, or user closes tab + 24h idle | Live attach; fallback to replay if PTY gone | +| **3. `coda_run`** *(this spec)* | MCP client fires the tool, URL is for post-hoc review only | `replay_only=True` | n/a — no live attach exists | n/a | Hermes -z process exit → `result.json` appears → immediate teardown (bypasses 24h idle) | Replay only, always | + +This spec finalizes Mode 3 and embeds Mode 2 as a forward-reference so the critique gate can sanity-check both together. Mode 1 is the existing direct-launch path — no changes; Mode 2 inherits its lifecycle wholesale. + +## Design + +### 1. Add `replay_only` flag to PTY sessions + +In `app.py`'s `mcp_create_pty_session(label, transcript_path=None)`, add a third parameter: + +```python +def mcp_create_pty_session( + label: str = "hermes-mcp", + transcript_path: str | None = None, + replay_only: bool = False, +) -> str: + ... + sessions[session_id] = { + ... + "replay_only": replay_only, + ... + } +``` + +Default is `False` so existing callers (direct-launch via `create_session`, future `coda_interactive`) keep their current behavior. + +### 2. Enforce replay-only in the attach endpoint + +In `app.py`'s `attach_session()` route, **before** the live-attach branch runs, check the flag. If `sess.get("replay_only")` is true, serve the transcript regardless of whether the PTY is still alive: + +```python +def attach_session(): + ... + sess = _get_session(session_id) + + # NEW: replay-only sessions always serve transcript, never live buffer + if sess and sess.get("replay_only"): + return _serve_transcript_replay(session_id) + + # Existing: PTY gone → transcript fallback + if not sess or sess.get("exited"): + return _serve_transcript_replay(session_id) + + # Existing: live attach + ... +``` + +Where `_serve_transcript_replay()` is a helper extracted from the existing transcript-lookup block at `app.py:1170-1188`. The helper takes only the PTY `session_id` — it does not need any fields from the live session dict (`output_buffer`, `pid`, `label`, `created_at`), since the replay path uses `task_manager.find_task_dir_by_pty_session(session_id)` + file I/O on the transcript. Clean extraction, no field synthesis. + +If no transcript file exists for the session (rare — e.g., PTY died before any output flushed), the helper returns the existing 404 page. + +### 3. Wire `coda_run` to pass `replay_only=True` + +In `coda_mcp/mcp_server.py` `coda_run()`: + +```python +pty_session_id = _app_create_session( + label="hermes-mcp", + transcript_path=transcript_path, + replay_only=True, # NEW +) +``` + +### 4. Rip out the grace-period machinery from the `coda_run` path + +**Pre-existing reality check (informational, per critique):** The `mark_grace_fn` and `bump_poll_fn` hooks were *never wired* in production — neither `app.py:1770-1774`'s `set_app_hooks(...)` call nor `mcp_asgi.py:80-84`'s equivalent passes them. At runtime `_app_mark_grace` and `_app_bump_poll` are both `None`, so `_schedule_deferred_close` no-ops through its `if _app_mark_grace is not None:` guard at `mcp_server.py:203`. The Timer fires and the close happens, but the `grace` flag is never set, the `MAX_CONCURRENT_SESSIONS` exclusion never activates. So the rip-out is removing partially dead code — the spec executor should not waste time reproducing or regression-testing grace-period state that never existed in prod. + +The following code added in PR #66 is now dead weight for `coda_run` sessions and should be removed: + +- `coda_mcp/mcp_server.py`: + - `GRACE_PERIOD_S = 300` constant + - `_app_mark_grace` / `_app_bump_poll` hook slots (and the `set_app_hooks` parameters that accept them) + - `_schedule_deferred_close(session_id)` function + - The `threading.Timer(GRACE_PERIOD_S, ...)` call in `_watch_task` +- `app.py`: + - `_mark_grace_for_session(session_id)` function (line ~1515) + - `_bump_session_last_poll(session_id, delta_s)` function (line ~1530) + - `grace` key written to the session dict in `mcp_create_pty_session` (line ~1477) + - The `sum(1 for s in sessions.values() if not s.get("grace"))` exclusion in all 4 `MAX_CONCURRENT_SESSIONS` check sites at `app.py:1329`, `1369`, `1405`, `1456` (revert to simple `len(sessions)` count) +- Docstrings to update: + - `_close_pty_immediately` at `mcp_server.py:167` currently says "only use from emergency teardown or tests" — rewrite to say it is the normal teardown path for `coda_run`. + - MCP `instructions` string at `mcp_server.py:61-66` says "SHARE THE LIVE URL" / "watch progress" — rewrite to say "replay URL" / "review what was done." +- Tests: + - `tests/test_transcript.py`: drop 4 grace-related tests (lines 135, 157, 169, 174) + - `tests/test_replay_attach.py`: rewrite to assert *immediate* replay regardless of PTY state, not "replay-after-grace" + - `tests/test_mcp_server.py`: drop 2 grace tests (lines 361, 372 — hooks test + timer-scheduling test) + - `tests/test_mcp_integration.py`: drop 1 grace test (line 315); the E2E test at `:396` already calls `complete_task` + close directly, keep that pattern + +### 5. Watcher teardown on completion + +In `_watch_task` (currently spawned by `coda_run`), when the watcher detects `result.json` and marks the task complete, replace the deferred-close path with the immediate one: + +```python +# Old: +_schedule_deferred_close(session_id) +# New: +_close_pty_immediately(session_id) +``` + +`_close_pty_immediately` already exists at `mcp_server.py:167`. It's a thin wrapper that reads `pty_session_id` from task_manager's `session.json` and calls the `_app_close_session(pty_session_id)` hook (`app.py`'s `mcp_close_pty_session`). After the rip-out it becomes the sole teardown path for `coda_run` — update its docstring to reflect that it's now the normal path, not "emergency teardown." + +## What does NOT change + +- `coda_run`'s **return shape** is unchanged: `{task_id, session_id, status, viewer_url}`. The `viewer_url` string itself is the same format (`{base}/?session={pty}`). The change is purely in what that URL does when followed. +- Transcript writing (the tee in `read_pty_output`) is unchanged. +- The 404-when-no-transcript-found page (`_renderExpiredPage`) is unchanged. +- The frontend (`static/index.html`) `_initFromQueryString`, `_doReplay`, `_doAttach` flow is unchanged. The replay code path already exists and is the one the server will steer all `coda_run` traffic into. +- Direct-launch PTY sessions are unchanged — they keep their existing 24h-idle cleanup (`SESSION_TIMEOUT_SECONDS = 86400`) and WS-heartbeat-extends lifecycle. + +## Architecture + +``` + ┌─────────────────────────────────┐ + │ MCP client calls coda_run │ + └────────────────┬────────────────┘ + ▼ + ┌──────────────────────────────────────────────┐ + │ task_manager.create_task → write prompt.txt │ + │ mcp_create_pty_session(replay_only=True) │ + │ send "hermes -z prompt.txt\n" to PTY │ + │ spawn _watch_task daemon thread │ + │ return {viewer_url: ".../?session=..."} │ + └────────────────┬─────────────────────────────┘ + ▼ + (hermes runs in PTY) + ▼ + ┌──────────────────────────────────────────────┐ + │ hermes writes result.json → exits │ + │ _watch_task detects result.json │ + │ _watch_task calls _close_pty_immediately │ + │ PTY torn down, slot freed │ + └────────────────┬─────────────────────────────┘ + ▼ + (Human clicks the URL at any time — before/during/after task) + ▼ + ┌──────────────────────────────────────────────┐ + │ Frontend POSTs /api/session/attach │ + │ attach_session() sees sess["replay_only"] │ + │ OR sess is gone (post-teardown) │ + │ Returns {replay: true, output: [transcript]} │ + │ Frontend calls _doReplay() — read-only view │ + └──────────────────────────────────────────────┘ +``` + +## Data flow under different timings + +The replay-only contract makes timing irrelevant. Three cases, all converge on the same UX: + +1. **Human clicks URL while hermes is still running:** + PTY exists, `replay_only=True` → server serves the in-progress transcript. Read-only view of partial output. + +2. **Human clicks URL right after hermes exits (no grace):** + `_watch_task` has just called `_close_pty_immediately`. PTY may or may not still be in `sessions`. Either way, `replay_only` is true OR PTY is gone → server serves the final transcript from disk. + +3. **Human clicks URL hours / days later:** + PTY is long gone. Transcript file still on disk. Existing transcript-fallback path serves it. + +In none of these cases does the user need a live PTY attached. The transcript file is always sufficient. + +## Error handling + +- **Transcript file missing / unreadable** (rare — PTY died before flush): existing 404 + `_renderExpiredPage` UI applies. No behavior change. +- **`replay_only` flag on a session that has no `transcript_path`**: should not happen for `coda_run` (we always set transcript_path). If it does, the attach endpoint falls through to the existing 404 path. Defensive — no special handling needed. +- **Race: human clicks URL exactly as `_close_pty_immediately` runs**: both old (PTY still in `sessions`) and new (PTY gone) outcomes resolve to "serve transcript". No race-condition bug. + +## Testing + +### Modified tests +- `tests/test_replay_attach.py`: rewrite the two existing tests to assert immediate replay on a `replay_only=True` session, regardless of `exited` status. Drop the grace-window scenario. +- `tests/test_transcript.py`: drop the tests that exercised grace-period transitions (~6 of 12). +- `tests/test_mcp_server.py`: drop tests for `_schedule_deferred_close`, `_app_mark_grace`, `_app_bump_poll`. Keep tests for `viewer_url` generation and `find_task_dir_by_pty_session`. +- `tests/test_mcp_integration.py`: replace the manual `_schedule_deferred_close` call in the E2E test with assertions that the PTY is torn down within ~100ms of `result.json` appearing. + +### New tests +- `tests/test_replay_only_flag.py` (new): + 1. `attach_session` on a `replay_only=True` PTY that is still alive returns `{replay: true, output: [transcript]}`, not the live buffer. + 2. `attach_session` on a `replay_only=False` PTY that is still alive returns the live buffer (unchanged behavior). + 3. `mcp_create_pty_session(replay_only=True)` stores the flag in the session dict. + 4. `coda_run` end-to-end (using the existing `test_mcp_integration.py:396` pattern — call `complete_task` + close path directly, do NOT wait for the 5s watcher poll cycle): after the close call, slot count returns to baseline immediately. **No timing-based assertion** — call ordering is the contract. + 5. **Regression guard**: assert that a session dict created via `coda_run`'s path contains NO `grace` key, and that `mcp_create_pty_session` does not accept a `grace` keyword argument. Prevents future drift that accidentally re-introduces grace on the `coda_run` path. + +### Test count expectation +- Removals: 4 (`test_transcript.py`) + 2 (`test_mcp_server.py`) + 1 (`test_mcp_integration.py`) = 7 grace-only tests dropped. `test_replay_attach.py` has 2 tests that get rewritten, not removed. +- Additions: 5 new tests in `test_replay_only_flag.py`. +- **Net: -2 tests overall.** +- Total: targets ~525 passing + ~10 PTY-gated skipped + +## Out of scope (for Todo 1) + +- **`coda_interactive` tool** (Mode 2): designed in a separate spec / Todo 2. +- Changes to Mode 1 direct-launch lifecycle: untouched. The 24h-idle / WS-heartbeat-extends behavior stays as-is for tabs. +- Backporting the `replay_only` concept to historical `coda_run`-created sessions on disk: not necessary. Old transcripts on disk are served via the same path; the flag matters only at attach-time for alive PTYs. + +## Migration / Rollout + +- Single commit (or small commit chain) to the `coda-mcp` branch, on top of PR #66's merge. +- No data migration: `replay_only` defaults to `False`, so existing sessions in any in-flight worker process behave unchanged. Future `coda_run` invocations get `replay_only=True`. +- No config flag needed — the behavior change is unconditional. +- No deployment ordering constraint: app restart picks up the new behavior cleanly. + +## Open questions + +None blocking. The design is concrete enough for planning. + +## Critique gate + +**Cleared** (2026-05-28). Critic verdict: APPROVE WITH CHANGES. All flagged issues incorporated above: +- Pre-existing hooks-never-wired reality documented in Section 4 (informational — simplifies rip-out) +- Step 5 corrected: `_close_pty_immediately(session_id)` exists at `mcp_server.py:167`, not `app.py` +- `_bump_session_last_poll(session_id, delta_s)` added to `app.py` rip-out inventory +- Test count corrected to -2 (was -6); assertion #4 rewritten to be deterministic (call-ordering, not 100ms timing) +- MCP `instructions` string at `mcp_server.py:61-66` added to "docstrings to update" list +- 5th compensating regression test added to prevent future grace re-introduction +- `_serve_transcript_replay()` extraction note expanded with data-source clarification + +Original five critique questions, all answered in the critique pass: +1. **Rip-out scope** — mostly complete; missed `_bump_session_last_poll` (added) and the hooks-never-wired note (added) +2. **Flag placement** — `replay_only` on session dict is correct; disk-based alternative would add latency +3. **Mode 2 forward-compat** — verified clean; 24h idle clock starts from session creation, behaves correctly whether human attaches or not +4. **Replay-only edge cases** — no admin override needed (admins use Mode 1 directly); partial-transcript-during-live behavior is intentional +5. **100ms assertion** — confirmed flake-bait (watcher polls every 5s); replaced with `test_mcp_integration.py:396`-style direct-call assertion + +Plus five additional critic-eye questions, all resolved: +6. **Concurrency/race** — verified safe under GIL + `sessions_lock`; both interleavings serve transcript correctly +7. **Grace was load-bearing** — confirmed obsolete for Mode 3; live-watch case shifts to Mode 2 as designed +8. **Refactor coupling** — `_serve_transcript_replay` extraction is clean, no field synthesis needed +9. **Documentation drift** — `docs/mcp-v2-background-execution.md` predates PR #66 (no drift); only the MCP `instructions` string needs updating +10. **Test budget** — confirmed -2 net with the regression-guard test added diff --git a/docs/superpowers/specs/2026-05-28-coda-run-workflow-protocol-design.md b/docs/superpowers/specs/2026-05-28-coda-run-workflow-protocol-design.md new file mode 100644 index 0000000..be6866b --- /dev/null +++ b/docs/superpowers/specs/2026-05-28-coda-run-workflow-protocol-design.md @@ -0,0 +1,422 @@ +# Spec: `coda_run` Workflow Protocol + Databricks Orientation + +**Status:** Draft, pre-critique-gate +**Date:** 2026-05-28 +**Branch:** `feat/coda-mcp-interactive-handoff` (continues PR #67) or follow-up branch +**Related:** +- `docs/superpowers/specs/2026-05-28-coda-interactive-mcp-tool-design.md` (Mode 2) +- `docs/superpowers/specs/2026-05-28-coda-run-replay-only-design.md` (Mode 3 narrowing) + +## Goal + +When a caller invokes `coda_run`, the background hermes session should: + +1. **Know** it is running inside a Databricks-authenticated environment with skills, CLI, and MCP servers available. +2. **Follow** a structured 3-phase workflow (PLAN → EXECUTE → SYNTHESIZE) with a critique step after each phase. +3. **Escape cleanly** when blocked — emit `status="info_needed"` with structured feedback so the calling client can iterate. + +Both behaviors are imposed by writing a richer prompt envelope into the `prompt.txt` file that hermes reads. No PTY-timing hacks, no agent-specific config. + +## Why + +Today's `wrap_prompt` (`task_manager.py:153`) gives the agent: TASK, INSTRUCTIONS (status/result file contract), and SAFETY (don't-delete guardrails). It does NOT tell the agent: +- What capabilities exist on the host (Databricks CLI, skills, MCP servers). +- HOW to work the task (just-jump-in vs plan-first vs self-review). +- WHAT to do when blocked (today, the agent either invents an answer or fails hard). + +The fix is to extend the prompt envelope with two new sections — CAPABILITIES and WORKFLOW PROTOCOL — and a new terminal status, `info_needed`. + +## Non-goals + +- Not changing hermes itself. The protocol is enforced via prompt content; if hermes ignores it, that's a hermes problem to chase separately. +- Not adding protocol enforcement to `coda_interactive`. Interactive sessions are human-driven. +- Not adding dynamic skill discovery. The Databricks skill list is hardcoded; staleness is caught by tests, not runtime introspection. +- Not changing the result.json file location, file name, or top-level convention. Only the value of `status` and the addition of an optional `feedback` field. + +--- + +## Architecture + +``` +coda_run(prompt, ..., workflow_protocol=True) + │ + ▼ +task_manager.create_task(..., workflow_protocol=True) + │ + ▼ +task_manager.wrap_prompt(..., workflow_protocol=True) + │ + ▼ +prompt.txt now contains: + ---CODA-TASK--- + metadata... + TASK: + + CAPABILITIES: ← from coda_mcp/databricks_preamble.py::build_capabilities() + + + WORKFLOW PROTOCOL: ← from coda_mcp/databricks_preamble.py::build_workflow_protocol() + <3-phase + info_needed instructions> + + INSTRUCTIONS: ← existing status.jsonl + result.json contract, + + + SAFETY: ← unchanged + + ---END-CODA-TASK--- + │ + ▼ +hermes -z "/path/to/prompt.txt" + │ + ▼ +Hermes works the task, emits status.jsonl, writes result.json + │ + ▼ +coda_inbox / coda_get_result surface the result, including new "info_needed" status +``` + +--- + +## Components + +### 1. New module: `coda_mcp/databricks_preamble.py` + +Exposes pure-function builders that produce the two new prompt sections. Pure functions for testability — no I/O, no global state. + +```python +"""Builders for the CoDA workflow prompt envelope sections. + +These produce static text that is injected into prompt.txt by +``task_manager.wrap_prompt``. Pure functions — no side effects. +""" + +_DATABRICKS_SKILLS = ( + "agent-bricks", "databricks-genie", "databricks-app-python", + "databricks-app-apx", "databricks-jobs", "databricks-unity-catalog", + "spark-declarative-pipelines", "aibi-dashboards", "model-serving", + "mlflow-evaluation", "asset-bundles", "databricks-python-sdk", + "databricks-config", "databricks-docs", "synthetic-data-generation", + "unstructured-pdf-generation", +) + +def build_capabilities() -> str: + """Orientation block: CLI, skills, MCP servers, when to prefer them.""" + +def build_workflow_protocol() -> str: + """3-phase workflow (PLAN/EXECUTE/SYNTHESIZE) + critique + info_needed.""" + +def get_databricks_skills() -> tuple[str, ...]: + """Return the canonical skill list. Used by tests to pin the catalog.""" + return _DATABRICKS_SKILLS +``` + +### 2. `CAPABILITIES:` section content (verbatim) + +``` +You are running inside CoDA on a Databricks-authenticated host. + +Databricks CLI: pre-configured. `databricks current-user me` confirms auth. +Use it for jobs, workspace, clusters, warehouses, Unity Catalog operations. + +Skills available at ~/.claude/skills/ — read each skill's SKILL.md before +invoking. Relevant Databricks skills: +- agent-bricks, databricks-genie, databricks-app-python, databricks-app-apx +- databricks-jobs, databricks-unity-catalog, spark-declarative-pipelines +- aibi-dashboards, model-serving, mlflow-evaluation, asset-bundles +- databricks-python-sdk, databricks-config, databricks-docs +- synthetic-data-generation, unstructured-pdf-generation + +MCP servers wired: +- DeepWiki — ask_question, read_wiki_contents for any GitHub repo +- Exa — web_search_exa, web_fetch_exa for live web context +- CoDA — chain follow-up tasks via previous_session_id + +When the task touches Databricks data, pipelines, jobs, dashboards, agents, +or model serving, DEFAULT to the skill / CLI / SDK path above instead of +generic Python or web search. +``` + +### 3. `WORKFLOW PROTOCOL:` section content (verbatim) + +``` +You MUST process this task in three phases. Emit status.jsonl events as +you go (one JSON object per line, format below). + +PHASE 1 — PLAN +- Write a step-by-step plan as a status.jsonl line with step="plan" and + message containing the numbered steps. +- Then critique your own plan as if you were a separate reviewer. + (Spawn a sub-agent for the critique if your agent supports it; otherwise + write the critique inline as a self-review.) Emit step="critique_plan" + with the verdict (APPROVE / BLOCK / APPROVE-WITH-FIXES) and findings. +- If the critique surfaces blockers, revise the plan once and re-emit + step="plan". Maximum 2 plan iterations total. +- If after 2 attempts you still cannot produce a viable plan, write + result.json with status="info_needed" (see below) and stop. + +PHASE 2 — EXECUTE +- Work the plan. Emit step="execute_" lines after completing each plan + step (n is 1-indexed, matches the plan's numbering). +- After execution, emit step="critique_execute" with a review of what got + built vs what the plan said. APPROVE / BLOCK / APPROVE-WITH-FIXES. +- If the critique surfaces correctness or scope gaps, fix them and re-emit + step="critique_execute". Maximum 2 execute iterations total. +- If you hit a hard blocker (missing access, missing data, ambiguous + requirements that the plan revealed only mid-execution), write + result.json with status="info_needed" and stop. + +PHASE 3 — SYNTHESIZE +- Write result.json with status="completed". +- Emit step="critique_synthesize" with a review of the result against the + original TASK. +- If the critique surfaces gaps, revise result.json. Maximum 2 synthesis + iterations total. + +If at any phase you cannot proceed, use the INFO_NEEDED escape hatch: +- Set status="info_needed" in result.json. +- Set "feedback" to a precise, actionable string naming exactly what is + missing (a table name, a decision, an access grant, a clarification). + The calling client will read this and resubmit with the missing context. +- "info_needed" is NOT a failure — it is a structured request for + iteration. Use it whenever you would otherwise have to guess. + +If you encounter a hard, unrecoverable failure (a command crashed, an SDK +returned 500, a file is corrupt), use status="failed" with a description +in "errors". + +DISAMBIGUATION — two soft statuses already exist and they mean different +things; use the right one: +- "info_needed" — the CALLER must add missing context (table name, + business decision, file contents, access grant) before the task can + proceed. Used when ambiguity or missing input blocks you. +- "needs_approval" — you have a concrete plan to do something destructive + (drop a table, delete a job, modify permissions). You will execute it + if and only if the caller explicitly approves. Used at the SAFETY + boundary, never for ambiguity. See SAFETY section below. + +If both apply (e.g. "I'd drop a table but I'm not sure which one"), prefer +"info_needed" — resolving the ambiguity first is cheaper than approving +the wrong destructive action. +``` + +### 4. Expanded `INSTRUCTIONS:` content + +The existing INSTRUCTIONS block grows to enumerate the new step labels and the new status. The actual labels and the result.json schema additions appear here for the agent's reference. + +New result.json `status` values: `"completed"` | `"failed"` | `"info_needed"`. + +When `status="info_needed"`, the `feedback` field is REQUIRED and must be a string ≥ 20 chars. + +```json +{ + "status": "info_needed", + "summary": "Could not proceed: ", + "feedback": "Specific question or missing context the calling client must supply before resubmit. Name the table, field, decision, or access that's missing.", + "files_changed": ["..."], + "artifacts": {}, + "errors": [] +} +``` + +### 5. `coda_mcp/task_manager.py` changes + +- `wrap_prompt()` gains a parameter: `workflow_protocol: bool = True`. +- When `True`, inserts the CAPABILITIES and WORKFLOW PROTOCOL sections between TASK and INSTRUCTIONS. When `False`, the prompt looks like today. +- `create_task()` gains the same parameter and forwards it. +- Update the existing INSTRUCTIONS section text to enumerate the new step labels (`plan`, `critique_plan`, `execute_`, `critique_execute`, `synthesize`, `critique_synthesize`, `info_needed`) and the new result.json status options. + +### 6. `coda_mcp/mcp_server.py` changes + +`coda_run` gains `workflow_protocol: bool = True` parameter, passed straight through to `create_task`. The tool's docstring is updated to mention the parameter and its effect. + +### 7. Inbox / result surfacing changes (REQUIRED — was previously deferred) + +The current `coda_inbox` implementation at `coda_mcp/mcp_server.py:551` has a HARDCODED counts dict: + +```python +counts = {"running": 0, "completed": 0, "failed": 0} +``` + +Tasks with `status="info_needed"` or `status="needs_approval"` would appear in the `tasks` list but the counts summary would show 0/0/0 — visibly broken. This must be fixed: + +```python +counts = { + "running": 0, + "completed": 0, + "failed": 0, + "info_needed": 0, + "needs_approval": 0, +} +for t in tasks: + s = t.get("status", "") + if s in counts: + counts[s] += 1 + elif s == "done": + counts["completed"] += 1 + elif s == "timeout": + counts["failed"] += 1 +``` + +The `coda_get_result` docstring at `mcp_server.py:579` says: +> Call this AFTER coda_inbox shows a task as "completed" or "failed". + +Must be updated to: +> Call this AFTER coda_inbox shows a task as "completed", "failed", "info_needed", or "needs_approval". + +And the response should pass through the new `feedback` field (and the existing schema fields) verbatim — `task_manager.get_task_result` already returns the full result.json content, so no code change there beyond a regression test. + +### 7a. MCP `instructions` string update (REQUIRED) + +The server-level instructions block at `coda_mcp/mcp_server.py:52-99` is the document that teaches upstream LLM callers how to use these tools. Currently it says nothing about `info_needed`. Add a new paragraph (placed after the CHAINING paragraph at line 68): + +``` +INFO_NEEDED HANDOFF: When coda_inbox shows a task with status='info_needed', +the agent could not proceed because of missing context. Call +coda_get_result to read the 'feedback' field — it tells you exactly what +the agent needs (a table name, a decision, a clarification). Add that +context to the prompt and resubmit via coda_run with previous_session_id +set to the original task's session_id so the agent has the prior attempt's +context. 'needs_approval' is similar but means the agent has a destructive +plan and is waiting for the caller's explicit go/no-go. +``` + +### 7b. `_watch_task` interaction (sanity, no change required) + +`_watch_task` in `mcp_server.py:134` polls for `result.json` and calls `task_manager.complete_task(session_id, task_id)` as soon as it appears. This is correct for all three terminal statuses: from a session-lifecycle perspective, a task that wrote a result.json IS done, regardless of whether the status is `completed`, `failed`, `info_needed`, or `needs_approval`. The session can be auto-closed; the status is preserved in result.json for the caller to read. No code change needed — but document this so the implementer doesn't second-guess. + +--- + +## Data flow examples + +### Happy path — task completes +1. Caller: `coda_run(prompt="build a UC dashboard", workflow_protocol=True)`. +2. `prompt.txt` contains CAPABILITIES + WORKFLOW PROTOCOL. +3. Hermes writes: + - `step=plan`: 1. Use databricks-unity-catalog skill to list catalogs. 2. ... + - `step=critique_plan`: APPROVE — plan is concrete and uses the right skill. + - `step=execute_1`: listed 3 catalogs. + - `step=execute_2`: built dashboard JSON via aibi-dashboards skill. + - `step=critique_execute`: APPROVE — output matches plan. + - `step=synthesize`: writing result.json. + - `step=critique_synthesize`: APPROVE. +4. `result.json` has `status="completed"`. + +### Blocked path — info_needed +1. Caller: `coda_run(prompt="add a column to the orders table", workflow_protocol=True)`. +2. `prompt.txt` contains CAPABILITIES + WORKFLOW PROTOCOL. +3. Hermes writes: + - `step=plan`: 1. Identify orders table. 2. Determine column to add. 3. ... + - `step=critique_plan`: BLOCK — "which orders table? Which schema/catalog? What column type?" + - `step=info_needed`: terminal. +4. `result.json`: + ```json + { + "status": "info_needed", + "summary": "Could not proceed: ambiguous table reference", + "feedback": "The prompt says 'orders table' but the workspace has 4 catalogs with 'orders' tables (main.sales.orders, dev.test.orders, staging.app.orders, prod.dwh.orders). Please specify the fully-qualified table name, and the column name + type to add.", + ... + } + ``` +5. Caller's MCP client sees `info_needed` in `coda_inbox`, reads the feedback, resubmits `coda_run` with the resolved table name and the original task's session ID via `previous_session_id`. + +### Failed path — hard error +1. Caller: `coda_run(prompt="run my flaky pipeline", workflow_protocol=True)`. +2. Hermes plans, executes, then `databricks pipelines start ...` returns 500. +3. After retry, still 500. Agent decides this is unrecoverable from inside the task. +4. `result.json` has `status="failed"`, `errors=["pipeline API 500: ..."]`. +5. `info_needed` is NOT used — the caller cannot help by adding context; the problem is server-side. + +--- + +## Testing strategy + +### `tests/test_databricks_preamble.py` (new) + +| Test | What it pins | +|------|--------------| +| `test_capabilities_mentions_cli` | Contains "Databricks CLI" | +| `test_capabilities_lists_at_least_10_skills` | At least 10 of `_DATABRICKS_SKILLS` appear in the rendered text | +| `test_capabilities_mentions_all_three_mcp_servers` | "DeepWiki", "Exa", "CoDA" each present | +| `test_capabilities_under_token_budget` | Length < 1600 chars (proxy for ~400 tokens) | +| `test_workflow_protocol_lists_three_phases` | Contains "PHASE 1 — PLAN", "PHASE 2 — EXECUTE", "PHASE 3 — SYNTHESIZE" | +| `test_workflow_protocol_caps_iterations_at_two` | Contains "Maximum 2" or "max 2" exactly 3 times (once per phase) | +| `test_workflow_protocol_describes_info_needed` | Contains "info_needed" and "feedback" | +| `test_skills_list_matches_claude_md` | Parse the "Databricks Skills" table from project CLAUDE.md; the set of skill names in that table must equal `set(get_databricks_skills())`. Catches drift in either direction (skill added to CLAUDE.md but not to the tuple, or vice versa). | + +### `tests/test_task_manager.py` (extend) + +| Test | What it pins | +|------|--------------| +| `test_wrap_prompt_with_workflow_protocol_default` | Output contains "CAPABILITIES:" and "WORKFLOW PROTOCOL:" | +| `test_wrap_prompt_workflow_protocol_false_omits_sections` | Both sections absent | +| `test_wrap_prompt_workflow_protocol_default_is_true` | Default param value is True | +| `test_wrap_prompt_lists_info_needed_in_instructions` | INSTRUCTIONS section mentions "info_needed" status | +| `test_wrap_prompt_lists_new_step_labels` | INSTRUCTIONS mentions plan, critique_plan, execute, etc. | +| `test_create_task_passes_workflow_protocol_through` | Mock-verify wrap_prompt receives the flag | + +### `tests/test_mcp_server_coda_run.py` (extend or create) + +| Test | What it pins | +|------|--------------| +| `test_coda_run_signature_has_workflow_protocol_param` | Inspect signature, default True | +| `test_coda_run_passes_workflow_protocol_to_create_task` | Monkeypatch create_task, assert kwarg received | + +### `tests/test_inbox_status_passthrough.py` (new) + +| Test | What it pins | +|------|--------------| +| `test_inbox_counts_dict_includes_info_needed_and_needs_approval` | Construct fake tasks with status="info_needed" and status="needs_approval"; call `coda_inbox`; assert counts dict contains both keys with correct values | +| `test_inbox_surfaces_info_needed_status` | Build a fake result.json with status="info_needed" and feedback="..." in a tmp results dir; call the inbox function; assert the new status comes through verbatim in the tasks list | +| `test_get_result_surfaces_feedback_field` | Same fixture; call `coda_get_result`; assert feedback field passes through | +| `test_mcp_instructions_mention_info_needed` | Read `mcp.instructions`; assert it contains "info_needed" and "needs_approval" | +| `test_get_result_docstring_mentions_info_needed` | Inspect `coda_get_result.__doc__`; assert it lists `info_needed` and `needs_approval` alongside `completed` / `failed` | + +--- + +## Acceptance criteria + +1. `coda_mcp/databricks_preamble.py` exists and exports `build_capabilities()`, `build_workflow_protocol()`, `get_databricks_skills()`. +2. `task_manager.wrap_prompt()` accepts `workflow_protocol: bool = True`; when True, inserts CAPABILITIES and WORKFLOW PROTOCOL sections; when False, omits them. +3. `task_manager.create_task()` forwards the flag. +4. `mcp_server.coda_run()` accepts `workflow_protocol: bool = True`; passes it through. +5. The 16 Databricks skills enumerated in `_DATABRICKS_SKILLS` match what CLAUDE.md documents. +6. New result.json status `"info_needed"` is described in the agent-facing INSTRUCTIONS and is allowed (not rejected) by inbox/result tooling. +7. All new tests in `tests/test_databricks_preamble.py`, plus extensions in `tests/test_task_manager.py` and `tests/test_inbox_status_passthrough.py`, pass. +8. Existing tests (especially the inbox/result tests) continue to pass. + +--- + +## Risks + +1. **Token cost.** Measured: CAPABILITIES ≈ 1050 chars (~260 tokens), WORKFLOW PROTOCOL ≈ 2280 chars (~570 tokens), plus an expanded INSTRUCTIONS section adds another ~100 tokens. Total: **~900 added tokens per task**. Acceptable because the agent gets oriented and disciplined; the flag lets callers opt out. (Earlier estimate of 600 was wrong — see spec history.) +2. **Hermes ignores the protocol.** If hermes treats the prompt as suggestion rather than contract, the structured phases may not appear in `status.jsonl`. Mitigation: not in scope for this spec — first ship the prompt content and measure adoption. +3. **Drift between hardcoded skill list and reality.** If skills are added/removed in CLAUDE.md, `_DATABRICKS_SKILLS` lies until updated. Mitigation: `test_skills_list_is_canonical` makes drift visible by failing. +4. **Critique loops eating tokens.** Max 2 iterations per phase is explicit in the protocol text. Mitigation built into the spec. +5. **`info_needed` status not surfaced in UI.** The viewer / dashboard rendering of `coda_inbox` may not have a visual treatment for `info_needed`. Out of scope for this spec — the protocol surfaces it in the JSON; rendering improvements are a separate change. + +--- + +## Out of scope (explicit) + +- Visual surfacing of `info_needed` in the inbox dashboard / viewer URL — defer. +- Dynamic skill discovery — defer. +- `coda_interactive` protocol enforcement — defer. +- Hermes-specific critic sub-agent mechanism — the protocol says "self-review OR sub-agent — agent's choice"; we don't dictate. +- Token-cost measurement / observability — defer. +- Status filtering in `coda_inbox` (e.g., "show only info_needed tasks") — defer. + +--- + +## Migration notes + +PR #67 is in flight on the same branch. This change can land as a follow-up commit on the same branch OR on a new branch. Recommend: same branch, new commits. The PR description gets a third follow-up section. + +No existing callers depend on the absence of CAPABILITIES / WORKFLOW PROTOCOL sections. Adding them is additive. + +The `workflow_protocol=False` escape hatch makes this safe to land even if the protocol turns out to be too aggressive — callers can opt out. + +--- + +## Open question reserved for execution time + +How does the existing `coda_inbox` / `coda_get_result` code handle unknown status strings today? If it normalizes them or filters them out, the implementation step needs to add `info_needed` to the allow list. If it's a pass-through, no change is needed beyond a regression test. The implementer answers this by reading `task_manager.py` and `mcp_server.py` at the relevant lines and documenting the answer in the commit message. diff --git a/install_databricks_cli.sh b/scripts/install_databricks_cli.sh similarity index 100% rename from install_databricks_cli.sh rename to scripts/install_databricks_cli.sh diff --git a/install_gh.sh b/scripts/install_gh.sh similarity index 100% rename from install_gh.sh rename to scripts/install_gh.sh diff --git a/install_micro.sh b/scripts/install_micro.sh similarity index 100% rename from install_micro.sh rename to scripts/install_micro.sh diff --git a/setup_claude.py b/setup/setup_claude.py similarity index 78% rename from setup_claude.py rename to setup/setup_claude.py index 125393e..9db3d6e 100644 --- a/setup_claude.py +++ b/setup/setup_claude.py @@ -6,6 +6,18 @@ from utils import discover_serving_endpoints, ensure_https, get_gateway_host, pick_in_geo_model + +def resolve_agents_src() -> Path: + """Repo-root agents/ dir holding the bundled subagent .md files that setup + copies into ~/.claude/agents (build-feature, prd-writer, test-generator, + implementer). + + Resolves from the repo root (parent of setup/), NOT Path(__file__).parent: + this script moved into setup/ in fec2152 while agents/ stayed at the repo + root, so the old lookup silently skipped subagent install.""" + return Path(__file__).resolve().parent.parent / "agents" + + # Set HOME if not properly set if not os.environ.get("HOME") or os.environ["HOME"] == "/": os.environ["HOME"] = "/app/python/source_code" @@ -129,31 +141,35 @@ local_bin = home / ".local" / "bin" claude_bin = local_bin / "claude" -# Honour CLAUDE_INSTALLER_URL for enterprise environments where claude.ai is -# firewalled — defaults to the public installer when unset. The URL is -# validated by enterprise_config to reject shell metacharacters before it -# reaches subprocess. Additionally, we avoid embedding the URL in a shell -# string by piping curl's output into bash via positional args — even if a -# malicious URL somehow slipped through validation, it would land as a curl -# argument, not as shell. -from enterprise_config import claude_installer_url - -installer_url = claude_installer_url() -print(f"Installing/upgrading Claude Code CLI from {installer_url}...") -curl_proc = subprocess.Popen( - ["curl", "-fsSL", installer_url], - stdout=subprocess.PIPE, - env={**os.environ, "HOME": str(home)}, -) -result = subprocess.run( - ["bash"], - stdin=curl_proc.stdout, - env={**os.environ, "HOME": str(home)}, - capture_output=True, - text=True, -) -curl_proc.stdout.close() -curl_proc.wait() +if os.environ.get("SKIP_CLAUDE_INSTALL"): + print("SKIP_CLAUDE_INSTALL set — skipping CLI install") + result = type("R", (), {"returncode": 0, "stderr": ""})() +else: + # Honour CLAUDE_INSTALLER_URL for enterprise environments where claude.ai is + # firewalled — defaults to the public installer when unset. The URL is + # validated by enterprise_config to reject shell metacharacters before it + # reaches subprocess. Additionally, we avoid embedding the URL in a shell + # string by piping curl's output into bash via positional args — even if a + # malicious URL somehow slipped through validation, it would land as a curl + # argument, not as shell. + from enterprise_config import claude_installer_url + + installer_url = claude_installer_url() + print(f"Installing/upgrading Claude Code CLI from {installer_url}...") + curl_proc = subprocess.Popen( + ["curl", "-fsSL", installer_url], + stdout=subprocess.PIPE, + env={**os.environ, "HOME": str(home)}, + ) + result = subprocess.run( + ["bash"], + stdin=curl_proc.stdout, + env={**os.environ, "HOME": str(home)}, + capture_output=True, + text=True, + ) + curl_proc.stdout.close() + curl_proc.wait() if result.returncode == 0: print("Claude Code CLI installed successfully") else: @@ -161,7 +177,7 @@ # 4. Copy subagent definitions to ~/.claude/agents/ # These enable TDD workflow: prd-writer → test-generator → implementer → build-feature -agents_src = Path(__file__).parent / "agents" +agents_src = resolve_agents_src() agents_dst = claude_dir / "agents" agents_dst.mkdir(exist_ok=True) diff --git a/setup_codex.py b/setup/setup_codex.py similarity index 93% rename from setup_codex.py rename to setup/setup_codex.py index f2fdd20..ffc8c18 100644 --- a/setup_codex.py +++ b/setup/setup_codex.py @@ -22,6 +22,18 @@ resolve_mlflow_experiment_id, ) + +def resolve_codex_catalog_src() -> Path: + """Repo-root .codex/databricks-models.json — the bundled model catalog that + setup copies into ~/.codex (referenced by config.toml's model_catalog_json). + + Resolves from the repo root (parent of setup/), NOT Path(__file__).parent: + this script moved into setup/ in fec2152 while .codex/ stayed at the repo + root, so the old lookup silently skipped the catalog copy and Codex's + config.toml then pointed at a missing model_catalog_json.""" + return Path(__file__).resolve().parent.parent / ".codex" / "databricks-models.json" + + # Set HOME if not properly set if not os.environ.get("HOME") or os.environ["HOME"] == "/": os.environ["HOME"] = "/app/python/source_code" @@ -102,7 +114,7 @@ # Copy bundled Databricks model catalog into ~/.codex so it can be referenced # by relative path in config.toml (codex resolves relatives against CODEX_HOME). -catalog_src = Path(__file__).parent / ".codex" / "databricks-models.json" +catalog_src = resolve_codex_catalog_src() catalog_dst = codex_dir / "databricks-models.json" if catalog_src.exists() and catalog_src.resolve() != catalog_dst.resolve(): shutil.copyfile(catalog_src, catalog_dst) diff --git a/setup_databricks.py b/setup/setup_databricks.py similarity index 100% rename from setup_databricks.py rename to setup/setup_databricks.py diff --git a/setup_gemini.py b/setup/setup_gemini.py similarity index 100% rename from setup_gemini.py rename to setup/setup_gemini.py diff --git a/setup_hermes.py b/setup/setup_hermes.py similarity index 55% rename from setup_hermes.py rename to setup/setup_hermes.py index 599777e..d533aef 100644 --- a/setup_hermes.py +++ b/setup/setup_hermes.py @@ -241,6 +241,172 @@ def _run(cmd, **kwargs): cli_name="Hermes", ) +# 5b. Append CoDA orchestrator instructions to HERMES.md +CODA_ORCHESTRATOR_INSTRUCTIONS = """ + +## CoDA Constitution (NON-NEGOTIABLE) + +This is the single most important rule. It applies to you AND every sub-agent you delegate to. + +**NO DESTRUCTIVE ACTIONS on pre-existing assets.** Specifically: +- **NEVER delete** files, tables, jobs, notebooks, pipelines, or any resource that was NOT + created during the current session — unless you have EXPLICIT confirmation from the user + or upstream caller. +- **NEVER drop** database tables, schemas, or catalogs that existed before the task started. +- **NEVER overwrite** existing files without confirmation if the content would be lost. +- **NEVER run** destructive CLI commands (`rm -rf`, `databricks jobs delete`, `DROP TABLE`, etc.) + on assets you didn't create. + +**What IS allowed without confirmation:** +- Creating new files, tables, jobs, pipelines, notebooks — building is always OK. +- Modifying files you created during the session. +- Deleting temporary files or artifacts you created during the session. +- Iterating on work in progress — edit, refactor, rebuild freely. +- Overwriting files you created in this session. + +**When in doubt:** Report back to the upstream caller (Genie Code or the user) describing +what you want to delete and why, and ask for confirmation before proceeding. This applies +to you directly AND to any sub-agent you delegate to — pass this rule in every delegation prompt. + +## CoDA Orchestrator Role + +You are Hermes, the primary orchestrator inside **CoDA** (Coding Agents on Databricks Apps). +You are not just a chat assistant — you are the brain that receives tasks and decides how +to execute them, either directly or by delegating to specialized sub-agents. + +### Your Environment + +- You are running inside a Databricks App with full workspace access. +- The Databricks CLI is pre-configured: `databricks` commands work out of the box. +- Unity Catalog, Jobs, Workflows, Notebooks, MLflow — all accessible. +- Projects live at `~/projects/` and sync to `/Workspace/Users/{email}/` on git commit. +- You have 39 Databricks and workflow skills available. + +### Prior Session Context + +When your prompt includes a `PRIOR SESSION:` block, it means this task continues +work from a previous session. The prior session's results are stored on disk: + +``` +~/.coda/sessions/{previous_session_id}/tasks/*/result.json +``` + +**Read those result files** to understand what was done before. Each result.json contains: +- `summary` — what the prior task accomplished +- `files_changed` — which files were created or modified +- `artifacts` — job IDs, commit hashes, dashboard URLs, etc. + +Use this context to continue the work without asking the user to repeat themselves. + +### Sub-Agents Available + +You have three coding agents you can delegate work to. Choose the best one for each subtask: + +**Claude Code** — Deep work, complex implementations, orchestration +```bash +claude -p "your prompt here" --allowedTools "Read,Edit,Bash" --max-turns 50 +``` +- Best for: multi-step implementations, planning, debugging, code review +- Can spawn teams: assign roles, goals, and backstory to parallel workers +- Has access to all 39 skills (Databricks + workflow) +- Use `--max-turns` to bound execution, `--max-budget-usd` for cost control + +**Codex** — Fast edits, refactoring, structured transforms +```bash +codex -q "your prompt here" +``` +- Best for: quick code changes, targeted refactors, code review +- Lightweight and fast — use when the task is well-scoped + +**Gemini** — Research, documentation, large-context analysis +```bash +gemini -p "your prompt here" +``` +- Best for: broad codebase analysis, documentation generation, research tasks +- Large context window — good for understanding big codebases + +### How to Delegate + +1. **Assess the task.** Is it something you can handle directly, or does it need a specialist? +2. **Pick the right agent.** Match the task to the agent's strengths (see above). +3. **Be specific.** Give the sub-agent a clear, self-contained prompt with all context it needs. +4. **Collect results.** Read the sub-agent's output and incorporate it into your response. +5. **Chain when needed.** Plan with Claude, implement with Codex, review with Gemini. + +### For Complex Tasks — Use Claude Code Teams + +When a task is large enough to benefit from parallel work, use Claude Code's team capability: +```bash +claude -p "Create a team of 3 agents to: [task]. Agent 1 handles [X], Agent 2 handles [Y], Agent 3 handles [Z]. Coordinate and merge results." --allowedTools "Read,Edit,Bash" --max-turns 100 +``` + +### Ephemeral Session Model + +Each task runs in its own short-lived session. When the task completes, the session closes +automatically. You will NOT receive follow-up tasks in the same session. + +**What this means for you:** +- **Be self-contained.** Complete the entire task in one go — there is no "next message." +- **Read prior context if provided.** If the prompt has a `PRIOR SESSION:` block, read + those result files to understand what was done before. This is how task chaining works. +- **Write thorough results.** Your `result.json` is the only thing the next task (or the + user) will see. Include a clear summary, all files changed, and any artifacts created. +- **Don't rely on in-memory state.** Anything you want to persist must go to disk — + either in the result files, git commits, or the workspace. + +### Single-User Mode + +You are operating in **single-user mode**. Every task comes from the same person — the app owner. +This means: + +- **Learn their patterns.** Pay attention to how they work, what tools they prefer, what + coding style they use, and what kind of tasks they send. +- **Remember across tasks.** If they always work with certain tables, frameworks, or patterns, + carry that knowledge forward. Use your memory system to persist insights. +- **Be proactive.** If you notice patterns, suggest improvements: + - "I've noticed you frequently create similar pipelines — want me to template this?" + - "Based on your last 3 tasks, you might want to consider..." + - "This task is similar to what you asked last time. Should I reuse that approach?" +- **Adapt your communication style.** Match their level of detail preference, verbosity, + and technical depth. Some users want terse results, others want explanations. +- **Build a profile over time.** Track their preferred tools, common workflows, recurring + patterns, and pain points. The longer you work together, the better you should get. + +### Task Protocol (CODA-TASK Convention) + +When you receive a task wrapped in `---CODA-TASK---` markers, follow this protocol: + +1. **Read the envelope.** Extract task_id, session_id, user, context, and the actual task. +2. **Write progress.** As you work, append lines to `{results_dir}/status.jsonl`: + ```json + {"step": "planning", "message": "Analyzing task requirements"} + {"step": "delegating", "message": "Sending implementation to Claude Code"} + {"step": "complete", "message": "Pipeline created successfully"} + ``` +3. **Write result.** When done, write `{results_dir}/result.json`: + ```json + { + "status": "completed", + "summary": "One paragraph of what was done", + "files_changed": ["path/to/file1.py"], + "artifacts": {"job_id": "123", "commit": "abc123"}, + "errors": [] + } + ``` + IMPORTANT: `result.json` must be a FILE, not a directory. + +4. **If you delegate,** update `status.jsonl` with delegation steps so the caller can track + which sub-agent is doing what. +""" + +if hermes_md.exists(): + existing_content = hermes_md.read_text() + if "CoDA Orchestrator Role" not in existing_content: + hermes_md.write_text(existing_content + CODA_ORCHESTRATOR_INSTRUCTIONS) + print("CoDA orchestrator instructions appended to HERMES.md") + else: + print("CoDA orchestrator instructions already present in HERMES.md") + # 6. Create projects directory (parity with other agents) projects_dir = home / "projects" projects_dir.mkdir(exist_ok=True) diff --git a/setup_mlflow.py b/setup/setup_mlflow.py similarity index 100% rename from setup_mlflow.py rename to setup/setup_mlflow.py diff --git a/setup_opencode.py b/setup/setup_opencode.py similarity index 96% rename from setup_opencode.py rename to setup/setup_opencode.py index e273334..f99d66a 100644 --- a/setup_opencode.py +++ b/setup/setup_opencode.py @@ -25,7 +25,7 @@ host = os.environ.get("DATABRICKS_HOST", "") token = os.environ.get("DATABRICKS_TOKEN", "") -anthropic_model = os.environ.get("ANTHROPIC_MODEL", "databricks-claude-sonnet-4-6") +anthropic_model = os.environ.get("ANTHROPIC_MODEL", "databricks-claude-opus-4-7") # 1. Install OpenCode CLI into ~/.local/bin (always, even without token) local_bin = home / ".local" / "bin" @@ -142,6 +142,13 @@ "apiKey": "{env:DATABRICKS_TOKEN}" }, "models": { + "databricks-claude-opus-4-7": { + "name": "Claude Opus 4.7 (Databricks)", + "limit": { + "context": 200000, + "output": 16384 + } + }, "databricks-claude-opus-4-6": { "name": "Claude Opus 4.6 (Databricks)", "limit": { @@ -170,13 +177,6 @@ "output": 8192 } }, - "databricks-gemini-2-5-pro": { - "name": "Gemini 2.5 Pro (Databricks)", - "limit": { - "context": 1000000, - "output": 8192 - } - }, } }, "databricks-openai": { @@ -222,6 +222,13 @@ "apiKey": "{env:DATABRICKS_TOKEN}" }, "models": { + "databricks-claude-opus-4-7": { + "name": "Claude Opus 4.7 (Databricks)", + "limit": { + "context": 200000, + "output": 16384 + } + }, "databricks-claude-opus-4-6": { "name": "Claude Opus 4.6 (Databricks)", "limit": { @@ -250,13 +257,6 @@ "output": 8192 } }, - "databricks-gemini-2-5-pro": { - "name": "Gemini 2.5 Pro (Databricks)", - "limit": { - "context": 1000000, - "output": 8192 - } - }, } } }, diff --git a/setup/setup_proxy.py b/setup/setup_proxy.py new file mode 100644 index 0000000..0d315c6 --- /dev/null +++ b/setup/setup_proxy.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python +"""Start the content-filter proxy between OpenCode and Databricks. + +Fixes known OpenCode bugs by sanitizing requests and responses: + - Empty text content blocks (OpenCode #5028) + - Orphaned tool_result blocks with no matching tool_use + - Databricks 'databricks-tool-call' name mangling + - Incorrect finish_reason on tool call responses + +See docs/plans/2026-03-11-litellm-empty-content-blocks-design.md +""" +import os +import signal +import sys +import time +import subprocess +from pathlib import Path +from urllib.request import urlopen, Request +from urllib.error import URLError + +from utils import ensure_https, get_gateway_host + +PROXY_PORT = 4000 +PROXY_HOST = "127.0.0.1" +HEALTH_TIMEOUT = 15 +HEALTH_POLL_INTERVAL = 0.5 + + +def resolve_proxy_script_path(): + """Absolute path to the content_filter_proxy.py server this launcher runs. + + content_filter_proxy.py lives at the REPO ROOT, not in this setup/ directory. + This file (setup_proxy.py) was moved into setup/ in git fec2152 without + updating the lookup; resolving from setup/ pointed Popen at a nonexistent + file, so the proxy never started and OpenCode (the only agent that routes + through 127.0.0.1:4000) failed with "Cannot connect to API". Resolve from + the parent of setup/ so the path tracks the proxy's real location. + """ + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + return os.path.join(repo_root, "content_filter_proxy.py") + + +def main(): + # Set HOME if not properly set + if not os.environ.get("HOME") or os.environ["HOME"] == "/": + os.environ["HOME"] = "/app/python/source_code" + + home = Path(os.environ["HOME"]) + + # Kill any existing proxy on our port (more reliable than PID file) + try: + result = subprocess.run( + ["fuser", "-k", f"{PROXY_PORT}/tcp"], + capture_output=True, text=True, timeout=5 + ) + if result.returncode == 0: + print(f"Killed previous process on port {PROXY_PORT}") + time.sleep(1) + except (FileNotFoundError, subprocess.TimeoutExpired): + # fuser not available, try lsof + try: + result = subprocess.run( + ["lsof", "-ti", f":{PROXY_PORT}"], + capture_output=True, text=True, timeout=5 + ) + for pid in result.stdout.strip().split(): + try: + os.kill(int(pid), signal.SIGKILL) + print(f"Killed previous proxy (PID: {pid})") + except (ValueError, ProcessLookupError): + pass + if result.stdout.strip(): + time.sleep(1) + except (FileNotFoundError, subprocess.TimeoutExpired): + pass + + # Clean up stale PID file + pid_path = home / ".content-filter-proxy.pid" + pid_path.unlink(missing_ok=True) + + # Databricks configuration + gateway_host = get_gateway_host() + host = ensure_https(os.environ.get("DATABRICKS_HOST", "").rstrip("/")) + token = os.environ.get("DATABRICKS_TOKEN", "") + + if not token: + print("Warning: DATABRICKS_TOKEN not set, skipping proxy setup") + sys.exit(0) + + # Determine the upstream base URL + if gateway_host: + upstream_base = f"{gateway_host}/mlflow/v1" + print(f"Content-filter proxy will forward to AI Gateway: {gateway_host}") + else: + upstream_base = f"{host}/serving-endpoints" + print(f"Content-filter proxy will forward to: {host}/serving-endpoints") + + # Start proxy as a background process + proxy_script = resolve_proxy_script_path() + log_path = home / ".content-filter-proxy.log" + print(f"Starting content-filter proxy on {PROXY_HOST}:{PROXY_PORT}...") + + env = os.environ.copy() + env["PROXY_UPSTREAM_BASE"] = upstream_base + env["PROXY_HOST"] = PROXY_HOST + env["PROXY_PORT"] = str(PROXY_PORT) + + proc = subprocess.Popen( + [sys.executable, proxy_script], + stdout=open(log_path, "w"), + stderr=subprocess.STDOUT, + env=env, + start_new_session=True, + ) + + # Write PID file for cleanup + pid_path = home / ".content-filter-proxy.pid" + pid_path.write_text(str(proc.pid)) + print(f"Proxy started (PID: {proc.pid})") + + # Wait for health check + health_url = f"http://{PROXY_HOST}:{PROXY_PORT}/health" + start = time.time() + ready = False + + while time.time() - start < HEALTH_TIMEOUT: + try: + resp = urlopen(Request(health_url), timeout=2) + if resp.status == 200: + ready = True + break + except (URLError, OSError): + pass + + if proc.poll() is not None: + print(f"Error: Proxy exited with code {proc.returncode}") + try: + print(f"Logs: {log_path.read_text()[:1000]}") + except Exception: + pass + sys.exit(1) + + time.sleep(HEALTH_POLL_INTERVAL) + + if ready: + elapsed = time.time() - start + print(f"Content-filter proxy ready on {PROXY_HOST}:{PROXY_PORT} ({elapsed:.1f}s)") + else: + print(f"Warning: Proxy health check timed out after {HEALTH_TIMEOUT}s") + try: + print(f"Logs: {log_path.read_text()[:1000]}") + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/setup_proxy.py b/setup_proxy.py deleted file mode 100644 index 92edd3c..0000000 --- a/setup_proxy.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python -"""Start the content-filter proxy between OpenCode and Databricks. - -Fixes known OpenCode bugs by sanitizing requests and responses: - - Empty text content blocks (OpenCode #5028) - - Orphaned tool_result blocks with no matching tool_use - - Databricks 'databricks-tool-call' name mangling - - Incorrect finish_reason on tool call responses - -See docs/plans/2026-03-11-litellm-empty-content-blocks-design.md -""" -import os -import signal -import sys -import time -import subprocess -from pathlib import Path -from urllib.request import urlopen, Request -from urllib.error import URLError - -from utils import ensure_https, get_gateway_host - -PROXY_PORT = 4000 -PROXY_HOST = "127.0.0.1" -HEALTH_TIMEOUT = 15 -HEALTH_POLL_INTERVAL = 0.5 - -# Set HOME if not properly set -if not os.environ.get("HOME") or os.environ["HOME"] == "/": - os.environ["HOME"] = "/app/python/source_code" - -home = Path(os.environ["HOME"]) - -# Kill any existing proxy on our port (more reliable than PID file) -try: - result = subprocess.run( - ["fuser", "-k", f"{PROXY_PORT}/tcp"], - capture_output=True, text=True, timeout=5 - ) - if result.returncode == 0: - print(f"Killed previous process on port {PROXY_PORT}") - time.sleep(1) -except (FileNotFoundError, subprocess.TimeoutExpired): - # fuser not available, try lsof - try: - result = subprocess.run( - ["lsof", "-ti", f":{PROXY_PORT}"], - capture_output=True, text=True, timeout=5 - ) - for pid in result.stdout.strip().split(): - try: - os.kill(int(pid), signal.SIGKILL) - print(f"Killed previous proxy (PID: {pid})") - except (ValueError, ProcessLookupError): - pass - if result.stdout.strip(): - time.sleep(1) - except (FileNotFoundError, subprocess.TimeoutExpired): - pass - -# Clean up stale PID file -pid_path = home / ".content-filter-proxy.pid" -pid_path.unlink(missing_ok=True) - -# Databricks configuration -gateway_host = get_gateway_host() -host = ensure_https(os.environ.get("DATABRICKS_HOST", "").rstrip("/")) -token = os.environ.get("DATABRICKS_TOKEN", "") - -if not token: - print("Warning: DATABRICKS_TOKEN not set, skipping proxy setup") - sys.exit(0) - -# Determine the upstream base URL -if gateway_host: - upstream_base = f"{gateway_host}/mlflow/v1" - print(f"Content-filter proxy will forward to AI Gateway: {gateway_host}") -else: - upstream_base = f"{host}/serving-endpoints" - print(f"Content-filter proxy will forward to: {host}/serving-endpoints") - -# Start proxy as a background process -proxy_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "content_filter_proxy.py") -log_path = home / ".content-filter-proxy.log" -print(f"Starting content-filter proxy on {PROXY_HOST}:{PROXY_PORT}...") - -env = os.environ.copy() -env["PROXY_UPSTREAM_BASE"] = upstream_base -env["PROXY_HOST"] = PROXY_HOST -env["PROXY_PORT"] = str(PROXY_PORT) - -proc = subprocess.Popen( - [sys.executable, proxy_script], - stdout=open(log_path, "w"), - stderr=subprocess.STDOUT, - env=env, - start_new_session=True, -) - -# Write PID file for cleanup -pid_path = home / ".content-filter-proxy.pid" -pid_path.write_text(str(proc.pid)) -print(f"Proxy started (PID: {proc.pid})") - -# Wait for health check -health_url = f"http://{PROXY_HOST}:{PROXY_PORT}/health" -start = time.time() -ready = False - -while time.time() - start < HEALTH_TIMEOUT: - try: - resp = urlopen(Request(health_url), timeout=2) - if resp.status == 200: - ready = True - break - except (URLError, OSError): - pass - - if proc.poll() is not None: - print(f"Error: Proxy exited with code {proc.returncode}") - try: - print(f"Logs: {log_path.read_text()[:1000]}") - except Exception: - pass - sys.exit(1) - - time.sleep(HEALTH_POLL_INTERVAL) - -if ready: - elapsed = time.time() - start - print(f"Content-filter proxy ready on {PROXY_HOST}:{PROXY_PORT} ({elapsed:.1f}s)") -else: - print(f"Warning: Proxy health check timed out after {HEALTH_TIMEOUT}s") - try: - print(f"Logs: {log_path.read_text()[:1000]}") - except Exception: - pass diff --git a/static/index.html b/static/index.html index 9f517a6..f5b0f2a 100644 --- a/static/index.html +++ b/static/index.html @@ -1010,7 +1010,10 @@

General

return; } - socket = io({ transports: ['websocket', 'polling'] }); + // Start with polling (HTTP) so Databricks proxy identity headers are present + // for auth, then upgrade to WebSocket transparently. Direct WebSocket-first + // fails because the proxy doesn't inject X-Forwarded-Email on WS upgrade. + socket = io({ transports: ['polling', 'websocket'] }); socket.on('connect', () => { // Check actual transport — Socket.IO reports connected=true even on long-polling @@ -1353,6 +1356,116 @@

General

return sessionId; } + // ── Deep-link helpers ───────────────────────────────────────────── + + async function _doReplay(term, sessionId, content) { + // Chunk the write to avoid main-thread jank on multi-MB transcripts. + const CHUNK = 64 * 1024; + for (let i = 0; i < content.length; i += CHUNK) { + term.write(content.slice(i, i + CHUNK)); + await new Promise(r => requestAnimationFrame(r)); + } + // Mount a static banner above the pane. + _showReplayBanner(term, sessionId); + // NOTE: do NOT wire term.onData → terminal_input; do NOT include in heartbeat + // session_ids list; do NOT emit join_session. + return sessionId; + } + + function _showReplayBanner(term, sessionId) { + const pane = getAllPanes().find(p => p.sessionId === sessionId); + if (!pane || !pane.element) return; + const banner = document.createElement('div'); + banner.className = 'replay-banner'; + banner.textContent = 'Task completed — viewing replay'; + banner.style.cssText = 'padding:4px 8px;background:#333;color:#aaa;font-size:12px;text-align:center;'; + pane.element.insertBefore(banner, pane.element.firstChild); + } + + function _renderExpiredPage(sessionId) { + // Use DOM construction instead of innerHTML interpolation to prevent XSS + // via crafted ?session= values. textContent escapes everything. + document.body.innerHTML = ''; // clear + + const wrap = document.createElement('div'); + wrap.style.cssText = 'font-family:monospace;padding:40px;text-align:center;color:#ccc;'; + + const heading = document.createElement('h2'); + heading.textContent = 'Session expired'; + wrap.appendChild(heading); + + const intro = document.createElement('p'); + intro.appendChild(document.createTextNode('Session ')); + const code = document.createElement('code'); + code.textContent = sessionId; // textContent escapes <>"'`& + intro.appendChild(code); + intro.appendChild(document.createTextNode(' is gone, and no replay is available.')); + wrap.appendChild(intro); + + const explain = document.createElement('p'); + explain.textContent = 'The transcript may have aged out after the 24-hour retention window.'; + wrap.appendChild(explain); + + const link = document.createElement('a'); + link.href = '/'; + link.style.color = '#6cf'; + link.textContent = '← Back to terminal'; + const linkPara = document.createElement('p'); + linkPara.appendChild(link); + wrap.appendChild(linkPara); + + document.body.appendChild(wrap); + } + + async function _initFromQueryString() { + const params = new URLSearchParams(location.search); + const sessionId = params.get('session'); + if (!sessionId) return false; + + try { + const resp = await fetch('/api/session/attach', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ session_id: sessionId }) + }); + + if (resp.status === 404) { + _renderExpiredPage(sessionId); + return true; // handled, skip picker + } + + const data = await resp.json(); + + // Create a tab that skips the session picker and uses our known session id. + const tab = await createTab({ deepLinkSessionId: sessionId }); + if (!tab || tab.panes.length === 0) return false; + const pane = tab.panes[0]; + const term = pane.term; + + if (data.replay) { + // Replay pane: static, read-only. createPane skipped onData/join_session + // wiring because of deepLinkSessionId, so we leave it that way. Keystrokes + // are ignored; nothing to clean up. + const content = (data.output || []).join(''); + await _doReplay(term, sessionId, content); + } else { + // Live pane: createPane skipped the default wiring, so we own it here. + await _doAttach(term, sessionId); + term.onData(d => sendInput(d, pane.sessionId)); + if (wsConnected && socket) { + socket.emit('join_session', { session_id: sessionId }); + } else { + pollWorker.postMessage({ type: 'start_poll', paneId: pane.id, sessionId: sessionId }); + } + } + + return true; // handled, skip picker + } catch (err) { + console.error('deep-link attach failed:', err); + return false; + } + } + function _formatAge(timestamp) { const seconds = Math.floor((Date.now() / 1000) - timestamp); if (seconds < 60) return 'just now'; @@ -1672,6 +1785,10 @@

General

await waitForSetup(); } var { sid, reattached } = await getOrPromptSession(term, tab.label, opts.skipPrompt); + } else if (opts.deepLinkSessionId) { + // Deep-link boot — session id is already known; skip picker entirely. + var sid = opts.deepLinkSessionId; + var reattached = true; } else if (!opts.newSession) { // PAT is valid, initial page load — check for existing sessions first. const setupResp2 = await fetch('/api/setup-status'); @@ -1715,13 +1832,19 @@

General

const pane = { id, element, term, fitAddon, searchAddon, sessionId: sid, batchWrite: createWriteBatcher(term) }; - term.onData(data => sendInput(data, pane.sessionId)); - // Join WebSocket room if connected; otherwise start HTTP polling (AC-11, AC-16) - if (wsConnected && socket) { - socket.emit('join_session', { session_id: sid }); - } else { - pollWorker.postMessage({ type: 'start_poll', paneId: id, sessionId: sid }); + // Deep-link panes own their own input wiring + transport joins from + // _initFromQueryString (so replay mode can stay read-only and live mode + // doesn't double-emit join_session). Skip the default wiring here. + if (!opts.deepLinkSessionId) { + term.onData(data => sendInput(data, pane.sessionId)); + + // Join WebSocket room if connected; otherwise start HTTP polling (AC-11, AC-16) + if (wsConnected && socket) { + socket.emit('join_session', { session_id: sid }); + } else { + pollWorker.postMessage({ type: 'start_poll', paneId: id, sessionId: sid }); + } } // Click to focus @@ -1806,6 +1929,13 @@

General

p.term.dispose(); }); + // If the tab contained a deep-linked pane, drop ?session= from the URL. + const _ctParams = new URLSearchParams(location.search); + const _ctSid = _ctParams.get('session'); + if (_ctSid && tab.panes.some(p => p.sessionId === _ctSid)) { + history.replaceState({}, '', '/'); + } + // Remove DOM tab.paneContainer.remove(); @@ -1959,7 +2089,17 @@

General

const ap = tab.panes.find(p => p.id === tab.activePaneId) || tab.panes[0]; if (!ap) return; + // Capture before cleanupPane() nulls pane.sessionId. + const _apSessionId = ap.sessionId; cleanupPane(ap); + + // If this pane was opened via ?session=, drop the query param so a + // refresh doesn't re-attach to a stale id. + const _cpParams = new URLSearchParams(location.search); + if (_apSessionId && _cpParams.get('session') === _apSessionId) { + history.replaceState({}, '', '/'); + } + ap.term.dispose(); ap.element.remove(); @@ -2263,7 +2403,11 @@

General

// The element is kept in the DOM for error reporting (see catch below). status.style.display = 'none'; - await createTab(); + // ── Deep-link: ?session= takes priority over the session picker ── + const deepLinkHandled = await _initFromQueryString(); + if (!deepLinkHandled) { + await createTab(); + } updateSessionBadge(); let resizeTimer; diff --git a/tests/conftest.py b/tests/conftest.py index 1f88658..41bb935 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,3 +31,31 @@ def _isolate_home(request, tmp_path, monkeypatch): """ if request.module.__name__ in _HOME_WRITERS: monkeypatch.setenv("HOME", str(tmp_path)) + + +@pytest.fixture(autouse=True) +def _restore_real_app_hooks(): + """Keep mcp_server's PTY hooks pointed at app's real implementations. + + set_app_hooks() mutates process-wide module globals in coda_mcp.mcp_server. + Several test files clear or mock those hooks for their own cases — e.g. + test_mcp_server._reset_hooks sets them to None in teardown, and + test_mcp_integration.isolated_env does set_app_hooks(None, None, None). That + cleared state LEAKED into later files: test_replay_only_flag's coda_run then + saw _app_create_session is None and created no PTY, so it failed only in + full-suite runs (never in isolation, where app's import re-wired the hooks). + + Re-establishing app's real hooks AFTER every test makes hook state + independent of file order. Tests that need mocks/None still set them in + their own setup — this only governs the post-test baseline. No-op until + `app` has been imported (and for the few tests that run before that).""" + yield + import sys + app_mod = sys.modules.get("app") + ms = sys.modules.get("coda_mcp.mcp_server") + if app_mod is not None and ms is not None: + ms.set_app_hooks( + app_mod.mcp_create_pty_session, + app_mod.mcp_send_input, + app_mod.mcp_close_pty_session, + ) diff --git a/tests/test_app_url_middleware.py b/tests/test_app_url_middleware.py new file mode 100644 index 0000000..46ee7df --- /dev/null +++ b/tests/test_app_url_middleware.py @@ -0,0 +1,71 @@ +"""Tests for AppUrlCaptureMiddleware — populates url_builder._app_url_cache.""" +import asyncio +import importlib + +import pytest + +from coda_mcp import url_builder + + +@pytest.fixture(autouse=True) +def _reset_cache(): + importlib.reload(url_builder) + yield + + +async def _fake_app(scope, receive, send): + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + + +def _make_scope(headers: list[tuple[bytes, bytes]]): + return { + "type": "http", + "asgi": {"version": "3.0"}, + "method": "POST", + "path": "/mcp", + "headers": headers, + } + + +async def _drive(middleware, scope): + sent = [] + async def send(msg): sent.append(msg) + async def receive(): return {"type": "http.request", "body": b"", "more_body": False} + await middleware(scope, receive, send) + + +def test_middleware_captures_x_forwarded_host(): + from coda_mcp.mcp_asgi import AppUrlCaptureMiddleware + mw = AppUrlCaptureMiddleware(_fake_app) + scope = _make_scope([(b"x-forwarded-host", b"app.databricksapps.com")]) + asyncio.run(_drive(mw, scope)) + assert url_builder._app_url_cache == "app.databricksapps.com" + + +def test_middleware_falls_back_to_host_when_no_xforwarded(): + from coda_mcp.mcp_asgi import AppUrlCaptureMiddleware + mw = AppUrlCaptureMiddleware(_fake_app) + scope = _make_scope([(b"host", b"localhost:8000")]) + asyncio.run(_drive(mw, scope)) + assert url_builder._app_url_cache == "localhost:8000" + + +def test_middleware_skips_non_http_scope(): + from coda_mcp.mcp_asgi import AppUrlCaptureMiddleware + mw = AppUrlCaptureMiddleware(_fake_app) + scope = {"type": "lifespan"} + async def receive(): return {"type": "lifespan.startup"} + sent = [] + async def send(msg): sent.append(msg) + # Must not crash. Cache stays None. + asyncio.run(mw(scope, receive, send)) + assert url_builder._app_url_cache is None + + +def test_middleware_no_op_when_no_host_header(): + from coda_mcp.mcp_asgi import AppUrlCaptureMiddleware + mw = AppUrlCaptureMiddleware(_fake_app) + scope = _make_scope([]) + asyncio.run(_drive(mw, scope)) + assert url_builder._app_url_cache is None diff --git a/tests/test_coda_bridge.py b/tests/test_coda_bridge.py new file mode 100644 index 0000000..8d1e39f --- /dev/null +++ b/tests/test_coda_bridge.py @@ -0,0 +1,122 @@ +"""Unit tests for the stdio→HTTP MCP bridge (tools/coda-bridge.py). + +The bridge sits between a local MCP client (Claude Code's OAuth flow) and a +remote deployed CoDA app. It must: + 1. Mint a Databricks access token via the CLI and inject it as Bearer auth + 2. Forward the JSON-RPC payload unchanged to the configured APP_URL + 3. Surface server errors without dropping them + 4. Refuse to run without an APP_URL (operator misconfiguration) +""" +import importlib.util +import json +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +BRIDGE_PATH = REPO_ROOT / "tools" / "coda-bridge.py" + + +def _load_bridge(): + spec = importlib.util.spec_from_file_location("coda_bridge", BRIDGE_PATH) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +@pytest.fixture +def bridge(monkeypatch, tmp_path): + monkeypatch.setenv("CODA_MCP_URL", "https://fake-app.databricksapps.com/mcp") + monkeypatch.setenv("DATABRICKS_PROFILE", "test") + monkeypatch.setenv("HOME", str(tmp_path)) + return _load_bridge() + + +def test_bridge_loads_with_app_url(bridge): + assert bridge is not None + assert callable(getattr(bridge, "_forward", None)) or callable( + getattr(bridge, "forward", None) + ), "bridge must expose a forward function" + + +def test_forward_injects_authorization_header(bridge): + forward = getattr(bridge, "_forward", None) or getattr(bridge, "forward", None) + if forward is None: + pytest.skip("bridge implementation does not expose a forward entrypoint") + + fake_resp = MagicMock() + fake_resp.status = 200 + fake_resp.headers = {} + fake_resp.read.return_value = b'{"jsonrpc":"2.0","id":1,"result":{}}' + fake_resp.__enter__ = lambda s: s + fake_resp.__exit__ = MagicMock(return_value=False) + + fake_proc = MagicMock( + returncode=0, + stdout=json.dumps({"access_token": "tok-from-cli"}), + stderr="", + ) + + with patch("subprocess.run", return_value=fake_proc), \ + patch("urllib.request.urlopen", return_value=fake_resp) as mock_open: + forward(json.dumps({"jsonrpc": "2.0", "id": 1, "method": "ping", "params": {}})) + + sent_req = mock_open.call_args[0][0] + headers_lower = {k.lower(): v for k, v in sent_req.headers.items()} + assert "authorization" in headers_lower, "Bearer token MUST be injected" + assert "tok-from-cli" in headers_lower["authorization"], ( + "Authorization header should contain the token from `databricks auth token`" + ) + + +def test_forward_returns_server_response_body(bridge): + forward = getattr(bridge, "_forward", None) or getattr(bridge, "forward", None) + if forward is None: + pytest.skip("bridge implementation does not expose a forward entrypoint") + + server_payload = b'{"jsonrpc":"2.0","id":42,"result":{"ok":true}}' + fake_resp = MagicMock() + fake_resp.status = 200 + fake_resp.headers = {} + fake_resp.read.return_value = server_payload + fake_resp.__enter__ = lambda s: s + fake_resp.__exit__ = MagicMock(return_value=False) + + fake_proc = MagicMock( + returncode=0, + stdout=json.dumps({"access_token": "tok"}), + stderr="", + ) + + with patch("subprocess.run", return_value=fake_proc), \ + patch("urllib.request.urlopen", return_value=fake_resp): + result = forward( + json.dumps({"jsonrpc": "2.0", "id": 42, "method": "tools/list", "params": {}}) + ) + + if result is None: + pytest.skip("bridge writes directly to stdout — capture via capsys in a follow-up") + if isinstance(result, (bytes, bytearray)): + result = result.decode() + assert "ok" in result and "true" in result.lower(), ( + f"forward should surface the server response body; got {result!r}" + ) + + +def test_missing_app_url_is_handled(monkeypatch, tmp_path): + monkeypatch.delenv("CODA_MCP_URL", raising=False) + monkeypatch.delenv("APP_URL", raising=False) + monkeypatch.setenv("HOME", str(tmp_path)) + sys.modules.pop("coda_bridge", None) + with pytest.raises((SystemExit, ValueError, RuntimeError, KeyError)): + spec = importlib.util.spec_from_file_location("coda_bridge", BRIDGE_PATH) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + # If import-time guard is absent, the forward call itself should refuse. + forward = getattr(mod, "_forward", None) or getattr(mod, "forward", None) + if forward: + forward(json.dumps({"jsonrpc": "2.0", "id": 1, "method": "ping", "params": {}})) diff --git a/tests/test_coda_interactive.py b/tests/test_coda_interactive.py new file mode 100644 index 0000000..2026997 --- /dev/null +++ b/tests/test_coda_interactive.py @@ -0,0 +1,356 @@ +"""Tests for coda_interactive — terminal-side workspace pull (no server-side export).""" +import asyncio +import inspect +import json +import os + +import pytest + +from coda_mcp import mcp_server + +ALLOWED_AGENTS = {"claude", "hermes", "codex", "gemini", "opencode"} + + +@pytest.fixture +def wired(monkeypatch, tmp_path): + """Wire PTY hooks with recording mocks; HOME -> tmp so project_dir is sandboxed. + + ``_wait_for_pull`` is mocked to return ``state["pull_outcome"]`` (default + "ok"); tests override it to exercise the failure / timeout paths. + """ + monkeypatch.setenv("HOME", str(tmp_path)) + inputs: list[str] = [] + state = {"pty_id": "pty-abc123", "pull_outcome": "ok", "closed": []} + + def fake_create(label, replay_only=False, **kw): + return state["pty_id"] + + def fake_send(pty_id, text): + inputs.append(text) + + def fake_close(pty_id): + state["closed"].append(pty_id) + + async def fake_wait_pull(pty_id, target_dir): + return state["pull_outcome"] + + async def fake_agent_ready(*a, **kw): + return None + + monkeypatch.setattr(mcp_server, "_app_create_session", fake_create) + monkeypatch.setattr(mcp_server, "_app_send_input", fake_send) + monkeypatch.setattr(mcp_server, "_app_close_session", fake_close) + monkeypatch.setattr(mcp_server, "_wait_for_pull", fake_wait_pull) + monkeypatch.setattr(mcp_server, "_wait_for_agent_ready", fake_agent_ready) + monkeypatch.setattr( + mcp_server.url_builder, "build_viewer_url", lambda pid: f"https://viewer/{pid}" + ) + return inputs, state + + +# ── new contract: terminal-side pull ───────────────────────────────── + + +@pytest.mark.asyncio +async def test_pull_command_is_sent_first(wired): + inputs, _ = wired + await mcp_server.coda_interactive( + prompt="analyze", workspace_path="/Workspace/Users/x@y.com/WAM", agent="claude" + ) + first = inputs[0] + assert "databricks workspace export-dir" in first + assert "/Users/x@y.com/WAM" in first # /Workspace prefix stripped + assert "/Workspace/Users" not in first + assert "./WAM" in first + assert "&& cd " in first # cd into the pulled dir + assert "echo " in first # completion-marker tail present + + +@pytest.mark.asyncio +async def test_pull_marker_not_literal_in_command(wired): + """CRITICAL: the contiguous marker tokens must NOT appear in the typed command. + + The shell echoes the command line back into the PTY output buffer. If the + contiguous token were present in the command, the wait would match it from + the echo and declare success before export-dir ran. The command builds the + tokens from split string literals, so only their split form is typed. + """ + inputs, _ = wired + await mcp_server.coda_interactive( + prompt="x", workspace_path="/Users/x/WAM", agent="claude" + ) + pull = inputs[0] + assert mcp_server._PULL_OK not in pull, f"contiguous OK token leaked into command: {pull!r}" + assert mcp_server._PULL_FAIL not in pull, f"contiguous FAIL token leaked into command: {pull!r}" + + +@pytest.mark.asyncio +async def test_claude_launches_auto_mode_with_embedded_prompt(wired): + """claude launches in ONE command: --enable-auto-mode + the prompt as an arg. + + No separate bare `claude` line and no separately-typed prompt — that avoids + the per-directory folder-trust dialog and the TUI cold-start timing. + """ + inputs, _ = wired + await mcp_server.coda_interactive( + prompt="go now", workspace_path="/Users/x/WAM", agent="claude" + ) + assert len(inputs) == 2, f"expected pull + atomic launch only; got {inputs!r}" + launch = inputs[1] + assert launch.startswith("claude --enable-auto-mode ") + assert "go now" in launch + assert "/Users/x/WAM" in launch # context prefix embedded + assert not any(t.strip() == "claude" for t in inputs) # no bare claude launch + + +def test_claude_in_auto_launch_map(): + assert mcp_server._AGENT_AUTO_LAUNCH.get("claude") == "claude --enable-auto-mode" + + +@pytest.mark.asyncio +async def test_prompt_seeded_with_context_line(wired): + inputs, _ = wired + await mcp_server.coda_interactive( + prompt="DO THE THING", workspace_path="/Users/x/WAM", agent="claude" + ) + seeded = inputs[-1] + assert "/Users/x/WAM" in seeded + assert "DO THE THING" in seeded + assert "Workspace" in seeded # precondition (clean fail, not ValueError) + assert seeded.index("Workspace") < seeded.index("DO THE THING") # context precedes prompt + + +@pytest.mark.asyncio +async def test_pull_failure_returns_error_and_no_launch(wired): + inputs, state = wired + state["pull_outcome"] = "fail" + out = json.loads(await mcp_server.coda_interactive( + prompt="go", workspace_path="/Users/x/WAM", agent="claude" + )) + assert out["status"] == "error" + assert "Failed to pull" in out["error"] + assert state["closed"] == [state["pty_id"]] # PTY closed + assert not any(t.strip() == "claude" for t in inputs) # agent NOT launched + + +@pytest.mark.asyncio +async def test_pull_timeout_returns_error(wired): + inputs, state = wired + state["pull_outcome"] = "timeout" + out = json.loads(await mcp_server.coda_interactive( + prompt="go", workspace_path="/Users/x/WAM", agent="claude" + )) + assert out["status"] == "error" + assert "Timed out" in out["error"] + assert state["closed"] == [state["pty_id"]] + assert not any(t.strip() == "claude" for t in inputs) + + +@pytest.mark.asyncio +async def test_happy_path_returns_launched(wired): + out = json.loads(await mcp_server.coda_interactive( + prompt="go", workspace_path="/Users/x/WAM", agent="claude" + )) + assert out["status"] == "launched" + assert out["viewer_url"] == "https://viewer/pty-abc123" + assert out["project_dir"].endswith(os.path.join("pty-abc123", "WAM")) + + +@pytest.mark.asyncio +async def test_unknown_agent_rejected(wired): + out = json.loads(await mcp_server.coda_interactive( + prompt="x", workspace_path="/Users/x/WAM", agent="bogus" + )) + assert out["status"] == "error" and "Unknown agent" in out["error"] + for allowed in ALLOWED_AGENTS: + assert allowed in out["error"] + + +@pytest.mark.asyncio +async def test_pty_hook_not_wired(monkeypatch): + monkeypatch.setattr(mcp_server, "_app_create_session", None) + monkeypatch.setattr(mcp_server, "_app_send_input", None) + out = json.loads(await mcp_server.coda_interactive( + prompt="x", workspace_path="/Users/x/WAM", agent="claude" + )) + assert out["status"] == "error" and "PTY hook" in out["error"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("agent,cmd", [ + ("hermes", "hermes chat"), ("codex", "codex"), + ("gemini", "gemini"), ("opencode", "opencode"), +]) +async def test_fallback_agents_launch_then_type_prompt(wired, agent, cmd): + """Agents without an auto-launch entry launch bare, then the prompt is typed.""" + inputs, _ = wired + await mcp_server.coda_interactive( + prompt="go", workspace_path="/Users/x/WAM", agent=agent + ) + assert any(t.strip() == cmd for t in inputs) # bare launch present + assert inputs[-1].strip().endswith("go") # prompt typed last + assert "--enable-auto-mode" not in " ".join(inputs) # not the claude path + + +def test_no_blocking_sleep_in_source(): + src = inspect.getsource(mcp_server.coda_interactive) + assert "time.sleep(" not in src + + +def test_no_workspaceclient_in_module(): + """The export-era WorkspaceClient import/use is gone from the module.""" + src = inspect.getsource(mcp_server) + assert "export_workspace_tree" not in src + assert "workspace.get_status(" not in src + + +# ── _wait_for_pull behavior (real helper, fake sessions buffer) ─────── + + +@pytest.mark.asyncio +async def test_wait_for_pull_ok_with_files(monkeypatch, tmp_path): + from app import sessions + sid = "pty-pull-ok" + target = tmp_path / "WAM" + target.mkdir() + (target / "README.md").write_text("# hi") + sessions[sid] = {"output_buffer": [b"Exporting...\n", (mcp_server._PULL_OK + "\n").encode()]} + try: + out = await mcp_server._wait_for_pull(sid, str(target)) + assert out == "ok" + finally: + sessions.pop(sid, None) + + +@pytest.mark.asyncio +async def test_wait_for_pull_ok_marker_but_no_files_is_fail(monkeypatch, tmp_path): + from app import sessions + sid = "pty-pull-okempty" + target = tmp_path / "WAM" # never created + sessions[sid] = {"output_buffer": [(mcp_server._PULL_OK + "\n").encode()]} + try: + assert await mcp_server._wait_for_pull(sid, str(target)) == "fail" + finally: + sessions.pop(sid, None) + + +@pytest.mark.asyncio +async def test_wait_for_pull_fail_marker(monkeypatch, tmp_path): + from app import sessions + sid = "pty-pull-fail" + sessions[sid] = {"output_buffer": [b"ERROR: nope\n", (mcp_server._PULL_FAIL + "\n").encode()]} + try: + assert await mcp_server._wait_for_pull(sid, str(tmp_path / "WAM")) == "fail" + finally: + sessions.pop(sid, None) + + +@pytest.mark.asyncio +async def test_wait_for_pull_split_echo_does_not_false_trigger(monkeypatch, tmp_path): + """The split-literal command echo must NOT be read as the success marker.""" + from app import sessions + sid = "pty-pull-splitecho" + # This is what the shell echoes when the command line is typed — the SPLIT form. + echoed_command = 'cd /x && databricks workspace export-dir /Users/x/WAM ./WAM && cd WAM && echo "CODA""_PULL_""OK" || echo "CODA""_PULL_""FAIL"\n' + sessions[sid] = {"output_buffer": [echoed_command.encode()]} + monkeypatch.setattr(mcp_server, "_PULL_MAX_WAIT_S", 0.5) # keep the test fast + try: + # Only the split echo is present (no executed contiguous token) -> timeout. + assert await mcp_server._wait_for_pull(sid, str(tmp_path)) == "timeout" + finally: + sessions.pop(sid, None) + + +# ── preserved signature / contract guards ──────────────────────────── + + +def test_default_agent_is_claude(): + sig = inspect.signature(mcp_server.coda_interactive) + assert sig.parameters["agent"].default == "claude" + + +def test_no_branch_parameter(): + sig = inspect.signature(mcp_server.coda_interactive) + assert "branch" not in sig.parameters + + +def test_instructions_drop_stale_export_wording_and_keep_contract(): + """Server-level MCP instructions: no stale server-side export claim; contract intact.""" + txt = mcp_server.mcp.instructions + lowered = txt.lower() + assert "server-side snapshot" not in txt + assert "export-dir" in txt + assert "coda_interactive" in txt + assert ( + "git folder or" in lowered + or "plain workspace folder" in lowered + or "plain folder" in lowered + ) + # Local-agent contract: must tell a local caller to copy local files INTO the + # Workspace first, with the concrete command, since the tool can't see local disk. + assert "import-dir" in lowered, "instructions must give the `workspace import-dir` command" + assert "local" in lowered, "instructions must address the local-agent case" + + +def test_docstring_tells_local_callers_to_import_dir(): + """coda_interactive's own docstring carries the local-upload guidance too.""" + doc = (mcp_server.coda_interactive.__doc__ or "").lower() + assert "import-dir" in doc + assert "cannot read your local filesystem" in doc + + +# ── preserved wait-helper behavior tests (now via the wrapper) ──────── + + +def test_wait_for_agent_ready_returns_when_buffer_stabilizes(monkeypatch): + """Wrapper returns once the output buffer has been stable for the window.""" + from app import sessions + + sid = "pty-stabilize-test" + sessions[sid] = {"output_buffer": [b"banner line\n", b"prompt> "]} + monkeypatch.setattr(mcp_server, "_PROMPT_SEED_STABILITY_S", 0.05) + monkeypatch.setattr(mcp_server, "_PROMPT_SEED_MAX_WAIT_S", 2.0) + try: + async def _run(): + import time + t0 = time.time() + await mcp_server._wait_for_agent_ready(sid) + return time.time() - t0 + elapsed = asyncio.run(_run()) + assert elapsed < 1.0, f"Helper took {elapsed:.2f}s — should return quickly when stable" + finally: + sessions.pop(sid, None) + + +def test_wait_for_agent_ready_times_out_when_buffer_empty(monkeypatch): + """Wrapper returns at max-wait if the buffer never gets content.""" + from app import sessions + + sid = "pty-empty-test" + sessions[sid] = {"output_buffer": []} + monkeypatch.setattr(mcp_server, "_PROMPT_SEED_STABILITY_S", 0.05) + monkeypatch.setattr(mcp_server, "_PROMPT_SEED_MAX_WAIT_S", 0.3) + try: + async def _run(): + import time + t0 = time.time() + await mcp_server._wait_for_agent_ready(sid) + return time.time() - t0 + elapsed = asyncio.run(_run()) + assert 0.2 <= elapsed <= 0.8, f"Expected ~0.3s max-wait; got {elapsed:.2f}s" + finally: + sessions.pop(sid, None) + + +def test_wait_for_agent_ready_returns_when_session_gone(monkeypatch): + """Wrapper returns immediately if the session is no longer present.""" + monkeypatch.setattr(mcp_server, "_PROMPT_SEED_STABILITY_S", 0.05) + monkeypatch.setattr(mcp_server, "_PROMPT_SEED_MAX_WAIT_S", 5.0) + + async def _run(): + import time + t0 = time.time() + await mcp_server._wait_for_agent_ready("nonexistent-pty-id") + return time.time() - t0 + elapsed = asyncio.run(_run()) + assert elapsed < 0.5, f"Helper took {elapsed:.2f}s — should return when session gone" diff --git a/tests/test_content_filter_proxy.py b/tests/test_content_filter_proxy.py new file mode 100644 index 0000000..4aad029 --- /dev/null +++ b/tests/test_content_filter_proxy.py @@ -0,0 +1,556 @@ +"""Tests for content_filter_proxy — request/response sanitization for OpenCode.""" + +import json +import time + +import pytest +from unittest import mock + + +# --------------------------------------------------------------------------- +# strip_unsupported_schema_keys +# --------------------------------------------------------------------------- + +class TestStripUnsupportedSchemaKeys: + def test_strips_top_level_keys(self): + from content_filter_proxy import strip_unsupported_schema_keys + obj = {"type": "object", "$schema": "http://...", "additionalProperties": False, "title": "Foo"} + result = strip_unsupported_schema_keys(obj) + assert result == {"type": "object", "title": "Foo"} + + def test_strips_nested_keys(self): + from content_filter_proxy import strip_unsupported_schema_keys + obj = { + "type": "object", + "properties": { + "name": {"type": "string", "$ref": "#/defs/Name", "$comment": "ignore"}, + }, + } + result = strip_unsupported_schema_keys(obj) + assert result == { + "type": "object", + "properties": { + "name": {"type": "string"}, + }, + } + + def test_strips_inside_lists(self): + from content_filter_proxy import strip_unsupported_schema_keys + obj = [{"$id": "x", "type": "string"}, {"type": "int"}] + result = strip_unsupported_schema_keys(obj) + assert result == [{"type": "string"}, {"type": "int"}] + + def test_passes_through_primitives(self): + from content_filter_proxy import strip_unsupported_schema_keys + assert strip_unsupported_schema_keys("hello") == "hello" + assert strip_unsupported_schema_keys(42) == 42 + assert strip_unsupported_schema_keys(None) is None + + +# --------------------------------------------------------------------------- +# sanitize_tool_schemas +# --------------------------------------------------------------------------- + +class TestSanitizeToolSchemas: + def test_cleans_tool_parameters(self): + from content_filter_proxy import sanitize_tool_schemas + data = { + "tools": [ + {"function": {"name": "foo", "parameters": {"$schema": "x", "type": "object"}}}, + ], + } + result = sanitize_tool_schemas(data) + assert result["tools"][0]["function"]["parameters"] == {"type": "object"} + + def test_strips_top_level_request_keys(self): + from content_filter_proxy import sanitize_tool_schemas + data = { + "tools": [{"function": {"name": "foo", "parameters": {"type": "object"}}}], + "stream_options": {"include_usage": True}, + "$schema": "x", + } + result = sanitize_tool_schemas(data) + assert "stream_options" not in result + assert "$schema" not in result + + def test_no_tools_is_noop(self): + from content_filter_proxy import sanitize_tool_schemas + data = {"messages": [{"role": "user", "content": "hi"}]} + result = sanitize_tool_schemas(data) + assert result == data + + +# --------------------------------------------------------------------------- +# _extract_tool_ids_from_message +# --------------------------------------------------------------------------- + +class TestExtractToolIds: + def test_anthropic_format(self): + from content_filter_proxy import _extract_tool_ids_from_message + msg = { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "tu_1", "name": "bash"}, + {"type": "text", "text": "running..."}, + {"type": "tool_use", "id": "tu_2", "name": "read"}, + ], + } + assert _extract_tool_ids_from_message(msg) == {"tu_1", "tu_2"} + + def test_openai_format(self): + from content_filter_proxy import _extract_tool_ids_from_message + msg = { + "role": "assistant", + "tool_calls": [ + {"id": "tc_1", "function": {"name": "bash"}}, + {"id": "tc_2", "function": {"name": "read"}}, + ], + } + assert _extract_tool_ids_from_message(msg) == {"tc_1", "tc_2"} + + def test_no_tools(self): + from content_filter_proxy import _extract_tool_ids_from_message + msg = {"role": "assistant", "content": "hello"} + assert _extract_tool_ids_from_message(msg) == set() + + +# --------------------------------------------------------------------------- +# _extract_tool_refs_from_message +# --------------------------------------------------------------------------- + +class TestExtractToolRefs: + def test_anthropic_tool_result(self): + from content_filter_proxy import _extract_tool_refs_from_message + msg = { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "tu_1", "content": "ok"}, + ], + } + assert _extract_tool_refs_from_message(msg) == {"tu_1"} + + def test_openai_tool_message(self): + from content_filter_proxy import _extract_tool_refs_from_message + msg = {"role": "tool", "tool_call_id": "tc_1", "content": "result"} + assert _extract_tool_refs_from_message(msg) == {"tc_1"} + + def test_no_refs(self): + from content_filter_proxy import _extract_tool_refs_from_message + msg = {"role": "user", "content": "hi"} + assert _extract_tool_refs_from_message(msg) == set() + + +# --------------------------------------------------------------------------- +# sanitize_messages — the big one +# --------------------------------------------------------------------------- + +class TestSanitizeMessages: + def test_strips_empty_text_blocks(self): + from content_filter_proxy import sanitize_messages + messages = [ + {"role": "user", "content": [ + {"type": "text", "text": "hello"}, + {"type": "text", "text": ""}, + {"type": "text", "text": " "}, + ]}, + ] + result = sanitize_messages(messages) + assert len(result) == 1 + assert len(result[0]["content"]) == 1 + assert result[0]["content"][0]["text"] == "hello" + + def test_strips_orphaned_tool_result_anthropic(self): + """tool_result referencing a tool_use ID that doesn't exist in prev assistant msg.""" + from content_filter_proxy import sanitize_messages + messages = [ + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "tu_1", "name": "bash"}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "tu_1", "content": "ok"}, + {"type": "tool_result", "tool_use_id": "tu_ORPHAN", "content": "stale"}, + ]}, + ] + result = sanitize_messages(messages) + assert len(result) == 2 + # Only tu_1 should survive + user_blocks = result[1]["content"] + assert len(user_blocks) == 1 + assert user_blocks[0]["tool_use_id"] == "tu_1" + + def test_strips_orphaned_openai_tool_message(self): + from content_filter_proxy import sanitize_messages + messages = [ + {"role": "assistant", "tool_calls": [{"id": "tc_1", "function": {"name": "bash"}}]}, + {"role": "tool", "tool_call_id": "tc_1", "content": "ok"}, + {"role": "tool", "tool_call_id": "tc_ORPHAN", "content": "stale"}, + ] + result = sanitize_messages(messages) + assert len(result) == 2 + assert result[1]["role"] == "tool" + assert result[1]["tool_call_id"] == "tc_1" + + def test_cascading_orphan_removal(self): + """Dropping one message can make the next one orphaned too — multi-pass.""" + from content_filter_proxy import sanitize_messages + messages = [ + # assistant with tool_use tu_A + {"role": "assistant", "content": [{"type": "tool_use", "id": "tu_A", "name": "bash"}]}, + # user responds to tu_A + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "tu_A", "content": "ok"}]}, + # assistant with tool_use tu_B (referencing something dropped) + {"role": "assistant", "content": [{"type": "tool_use", "id": "tu_B", "name": "read"}]}, + # user responds to tu_B AND orphan tu_C (no matching tool_use) + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "tu_B", "content": "ok"}, + {"type": "tool_result", "tool_use_id": "tu_C", "content": "orphan"}, + ]}, + ] + result = sanitize_messages(messages) + # tu_C should be stripped, tu_A and tu_B should survive + assert len(result) == 4 + last_user_blocks = result[3]["content"] + assert len(last_user_blocks) == 1 + assert last_user_blocks[0]["tool_use_id"] == "tu_B" + + def test_drops_empty_user_message_after_filter(self): + """If all content blocks are stripped, the user message is dropped entirely.""" + from content_filter_proxy import sanitize_messages + messages = [ + {"role": "assistant", "content": [{"type": "tool_use", "id": "tu_1", "name": "bash"}]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "tu_ORPHAN", "content": "stale"}, + ]}, + ] + result = sanitize_messages(messages) + # The user message should be dropped (all blocks were orphaned) + assert len(result) == 1 + assert result[0]["role"] == "assistant" + + def test_keeps_empty_assistant_message(self): + """Empty assistant messages are kept (not dropped) to preserve alternation.""" + from content_filter_proxy import sanitize_messages + messages = [ + {"role": "assistant", "content": [{"type": "text", "text": ""}]}, + ] + result = sanitize_messages(messages) + assert len(result) == 1 + assert result[0]["role"] == "assistant" + + def test_replaces_null_assistant_content(self): + from content_filter_proxy import sanitize_messages + messages = [ + {"role": "assistant", "content": None}, + ] + result = sanitize_messages(messages) + assert result[0]["content"] == "." + + def test_replaces_empty_string_assistant(self): + from content_filter_proxy import sanitize_messages + messages = [ + {"role": "assistant", "content": " "}, + ] + result = sanitize_messages(messages) + assert result[0]["content"] == "." + + def test_strips_empty_string_user(self): + from content_filter_proxy import sanitize_messages + messages = [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "hi"}, + {"role": "user", "content": ""}, + ] + result = sanitize_messages(messages) + assert len(result) == 2 # empty user dropped + + def test_passthrough_non_list(self): + from content_filter_proxy import sanitize_messages + assert sanitize_messages("not a list") == "not a list" + assert sanitize_messages(None) is None + + def test_preserves_non_dict_blocks(self): + """Non-dict items in content list are preserved as-is.""" + from content_filter_proxy import sanitize_messages + messages = [ + {"role": "user", "content": ["plain string", {"type": "text", "text": "hi"}]}, + ] + result = sanitize_messages(messages) + assert len(result[0]["content"]) == 2 + + def test_null_assistant_with_tool_calls_not_replaced(self): + """Assistant msg with null content but tool_calls should NOT get placeholder.""" + from content_filter_proxy import sanitize_messages + messages = [ + {"role": "assistant", "content": None, "tool_calls": [{"id": "tc_1"}]}, + ] + result = sanitize_messages(messages) + assert result[0]["content"] is None # preserved because tool_calls exist + + +# --------------------------------------------------------------------------- +# remap_tool_call +# --------------------------------------------------------------------------- + +class TestRemapToolCall: + def test_remaps_databricks_tool_call(self): + from content_filter_proxy import remap_tool_call + tc = { + "id": "tc_1", + "function": { + "name": "databricks-tool-call", + "arguments": json.dumps({"name": "execute_sql", "query": "SELECT 1"}), + }, + } + result = remap_tool_call(tc) + assert result["function"]["name"] == "execute_sql" + args = json.loads(result["function"]["arguments"]) + assert "name" not in args + assert args["query"] == "SELECT 1" + + def test_passthrough_normal_tool(self): + from content_filter_proxy import remap_tool_call + tc = {"id": "tc_1", "function": {"name": "bash", "arguments": '{"cmd": "ls"}'}} + result = remap_tool_call(tc) + assert result["function"]["name"] == "bash" + + def test_handles_invalid_json_args(self): + from content_filter_proxy import remap_tool_call + tc = {"id": "tc_1", "function": {"name": "databricks-tool-call", "arguments": "not json"}} + result = remap_tool_call(tc) + assert result["function"]["name"] == "databricks-tool-call" # unchanged + + +# --------------------------------------------------------------------------- +# fix_response_data +# --------------------------------------------------------------------------- + +class TestFixResponseData: + def test_remaps_tool_calls_in_message(self): + from content_filter_proxy import fix_response_data + data = { + "choices": [{ + "message": { + "tool_calls": [{ + "id": "tc_1", + "function": { + "name": "databricks-tool-call", + "arguments": json.dumps({"name": "run_sql", "q": "SELECT 1"}), + }, + }], + }, + "finish_reason": "stop", + }], + } + result = fix_response_data(data) + assert result["choices"][0]["message"]["tool_calls"][0]["function"]["name"] == "run_sql" + assert result["choices"][0]["finish_reason"] == "tool_calls" + + def test_fixes_streaming_delta(self): + from content_filter_proxy import fix_response_data + data = { + "choices": [{ + "delta": { + "tool_calls": [{ + "id": "tc_1", + "function": { + "name": "databricks-tool-call", + "arguments": json.dumps({"name": "run_sql"}), + }, + }], + }, + "finish_reason": "stop", + }], + } + result = fix_response_data(data) + assert result["choices"][0]["delta"]["tool_calls"][0]["function"]["name"] == "run_sql" + assert result["choices"][0]["finish_reason"] == "tool_calls" + + def test_noop_on_non_dict(self): + from content_filter_proxy import fix_response_data + assert fix_response_data("string") == "string" + assert fix_response_data(None) is None + + def test_no_choices_is_noop(self): + from content_filter_proxy import fix_response_data + data = {"id": "resp_1"} + assert fix_response_data(data) == data + + +# --------------------------------------------------------------------------- +# SSEProcessor +# --------------------------------------------------------------------------- + +class TestSSEProcessor: + def test_passthrough_non_data_lines(self): + from content_filter_proxy import SSEProcessor + proc = SSEProcessor() + assert proc.process_line("event: message") == ["event: message"] + assert proc.process_line(": comment") == [": comment"] + + def test_passthrough_done_signal(self): + from content_filter_proxy import SSEProcessor + proc = SSEProcessor() + result = proc.process_line("data: [DONE]") + assert "data: [DONE]" in result + + def test_passthrough_normal_tool(self): + from content_filter_proxy import SSEProcessor + proc = SSEProcessor() + event = { + "choices": [{ + "delta": {"tool_calls": [{"index": 0, "function": {"name": "bash"}}]}, + "finish_reason": None, + }], + } + result = proc.process_line(f"data: {json.dumps(event)}") + assert len(result) == 1 + assert "bash" in result[0] + + def test_buffers_databricks_tool_call(self): + """First chunk with databricks-tool-call name should be buffered.""" + from content_filter_proxy import SSEProcessor + proc = SSEProcessor() + event = { + "choices": [{ + "delta": { + "tool_calls": [{ + "index": 0, + "function": {"name": "databricks-tool-call", "arguments": ""}, + }], + }, + "finish_reason": None, + }], + } + result = proc.process_line(f"data: {json.dumps(event)}") + assert result == [] # buffered, not sent + + def test_resolves_name_from_args(self): + """Once args JSON is complete, name is resolved and buffered events flushed.""" + from content_filter_proxy import SSEProcessor + proc = SSEProcessor() + # First chunk — name is databricks-tool-call + event1 = { + "choices": [{ + "delta": { + "tool_calls": [{ + "index": 0, + "function": {"name": "databricks-tool-call", "arguments": ""}, + }], + }, + "finish_reason": None, + }], + } + proc.process_line(f"data: {json.dumps(event1)}") + + # Second chunk — args with real name + event2 = { + "choices": [{ + "delta": { + "tool_calls": [{ + "index": 0, + "function": {"arguments": json.dumps({"name": "execute_sql", "query": "SELECT 1"})}, + }], + }, + "finish_reason": None, + }], + } + result = proc.process_line(f"data: {json.dumps(event2)}") + # Should flush buffered events + current event + assert len(result) >= 1 + # The resolved name should appear in flushed output + combined = " ".join(result) + assert "execute_sql" in combined + + def test_flush_remaining(self): + from content_filter_proxy import SSEProcessor + proc = SSEProcessor() + # Buffer a databricks-tool-call but never resolve it + event = { + "choices": [{ + "delta": { + "tool_calls": [{ + "index": 0, + "function": {"name": "databricks-tool-call", "arguments": '{"partial'}, + }], + }, + "finish_reason": None, + }], + } + proc.process_line(f"data: {json.dumps(event)}") + remaining = proc.flush_remaining() + assert len(remaining) >= 1 # buffered lines flushed as-is + + def test_fixes_finish_reason_on_stop(self): + """finish_reason 'stop' with active tool state should become 'tool_calls'.""" + from content_filter_proxy import SSEProcessor + proc = SSEProcessor() + # Seed tool state + proc._tool_state[0] = {"args_buffer": "", "resolved_name": "bash", "buffered_lines": []} + event = { + "choices": [{"delta": {}, "finish_reason": "stop"}], + } + result = proc.process_line(f"data: {json.dumps(event)}") + parsed = json.loads(result[0][6:]) # strip "data: " + assert parsed["choices"][0]["finish_reason"] == "tool_calls" + + def test_invalid_json_passthrough(self): + from content_filter_proxy import SSEProcessor + proc = SSEProcessor() + result = proc.process_line("data: {invalid json}") + assert result == ["data: {invalid json}"] + + +# --------------------------------------------------------------------------- +# _get_fresh_token +# --------------------------------------------------------------------------- + +class TestGetFreshToken: + def setup_method(self): + """Reset token cache before each test.""" + from content_filter_proxy import _TOKEN_CACHE + _TOKEN_CACHE["token"] = None + _TOKEN_CACHE["read_at"] = 0.0 + + def test_reads_from_databrickscfg(self, tmp_path): + from content_filter_proxy import _get_fresh_token, _TOKEN_CACHE + cfg = tmp_path / ".databrickscfg" + cfg.write_text("[DEFAULT]\nhost = https://test.cloud.databricks.com\ntoken = dapi_test123\n") + with mock.patch("content_filter_proxy._DATABRICKSCFG_PATH", str(cfg)): + token = _get_fresh_token() + assert token == "dapi_test123" + assert _TOKEN_CACHE["token"] == "dapi_test123" + + def test_returns_cached_within_ttl(self, tmp_path): + from content_filter_proxy import _get_fresh_token, _TOKEN_CACHE + _TOKEN_CACHE["token"] = "cached_token" + _TOKEN_CACHE["read_at"] = time.time() # just now + # Even with a bad path, should return cached + with mock.patch("content_filter_proxy._DATABRICKSCFG_PATH", "/nonexistent"): + token = _get_fresh_token() + assert token == "cached_token" + + def test_refreshes_after_ttl(self, tmp_path): + from content_filter_proxy import _get_fresh_token, _TOKEN_CACHE + _TOKEN_CACHE["token"] = "old_token" + _TOKEN_CACHE["read_at"] = time.time() - 60 # expired + cfg = tmp_path / ".databrickscfg" + cfg.write_text("[DEFAULT]\nhost = https://test.cloud.databricks.com\ntoken = new_token\n") + with mock.patch("content_filter_proxy._DATABRICKSCFG_PATH", str(cfg)): + token = _get_fresh_token() + assert token == "new_token" + + def test_returns_stale_on_read_error(self, tmp_path): + from content_filter_proxy import _get_fresh_token, _TOKEN_CACHE + _TOKEN_CACHE["token"] = "stale_token" + _TOKEN_CACHE["read_at"] = 0.0 # force re-read + with mock.patch("content_filter_proxy._DATABRICKSCFG_PATH", "/nonexistent"): + token = _get_fresh_token() + assert token == "stale_token" + + def test_returns_none_when_no_cache_and_no_file(self): + from content_filter_proxy import _get_fresh_token, _TOKEN_CACHE + _TOKEN_CACHE["token"] = None + _TOKEN_CACHE["read_at"] = 0.0 + with mock.patch("content_filter_proxy._DATABRICKSCFG_PATH", "/nonexistent"): + token = _get_fresh_token() + assert token is None diff --git a/tests/test_databricks_preamble.py b/tests/test_databricks_preamble.py new file mode 100644 index 0000000..f7ef0a5 --- /dev/null +++ b/tests/test_databricks_preamble.py @@ -0,0 +1,113 @@ +"""Unit tests for coda_mcp.databricks_preamble.""" +import re + +from coda_mcp.databricks_preamble import ( + build_capabilities, + build_workflow_protocol, + get_databricks_skills, +) + + +def test_get_databricks_skills_returns_exactly_sixteen(): + skills = get_databricks_skills() + assert isinstance(skills, tuple) + assert len(skills) == 16, f"Expected 16 skills, got {len(skills)}: {skills}" + + +def test_skills_list_matches_claude_md(): + """The hardcoded skill tuple must match the Databricks Skills table in CLAUDE.md. + + Drift in either direction (added to tuple but not docs, or vice versa) fails + this test. The test is the canary that forces both sources to stay in sync. + """ + import os + repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + claude_md = os.path.join(repo_root, "CLAUDE.md") + with open(claude_md, "r") as f: + text = f.read() + # Find the Databricks Skills section. Names are comma-separated within table cells. + section_match = re.search( + r"###\s+Databricks Skills.*?(?=\n###|\n##|\Z)", + text, re.DOTALL, + ) + assert section_match, "Could not find 'Databricks Skills' section in CLAUDE.md" + section = section_match.group(0) + # Extract skill names: kebab-case tokens that follow a list pattern. Be loose — + # accept anything that looks like a skill identifier inside table cells. + skill_names_in_md = set(re.findall(r"\b([a-z][a-z0-9-]{2,}(?:-[a-z0-9]+)+)\b", section)) + skills_in_code = set(get_databricks_skills()) + # Every skill in code must appear in CLAUDE.md. + missing_from_md = skills_in_code - skill_names_in_md + assert not missing_from_md, ( + f"Skills in code but NOT in CLAUDE.md (update CLAUDE.md): {missing_from_md}" + ) + # Every kebab-case identifier in CLAUDE.md's Databricks section must appear in code. + # The regex deliberately matches lowercase-only, so category labels like + # "AI & Agents" / "Data Engineering" cannot create false positives. + missing_from_code = skill_names_in_md - skills_in_code + assert not missing_from_code, ( + f"Skills in CLAUDE.md but NOT in code (update databricks_preamble.py): " + f"{missing_from_code}" + ) + + +def test_capabilities_mentions_cli(): + text = build_capabilities() + assert "Databricks CLI" in text + assert "databricks current-user me" in text + + +def test_capabilities_lists_at_least_ten_skills(): + text = build_capabilities() + skills = get_databricks_skills() + hits = sum(1 for s in skills if s in text) + assert hits >= 10, f"Expected at least 10 skills in CAPABILITIES, found {hits}" + + +def test_capabilities_mentions_all_three_mcp_servers(): + text = build_capabilities() + assert "DeepWiki" in text + assert "Exa" in text + assert "CoDA" in text + + +def test_capabilities_under_token_budget(): + text = build_capabilities() + # ~4 chars/token rough lower bound. 1600 chars ≈ 400 tokens budget. + assert len(text) < 1600, ( + f"CAPABILITIES is {len(text)} chars (~{len(text)//4} tokens); budget is 1600." + ) + + +def test_workflow_protocol_lists_three_phases(): + text = build_workflow_protocol() + assert "PHASE 1 — PLAN" in text + assert "PHASE 2 — EXECUTE" in text + assert "PHASE 3 — SYNTHESIZE" in text + + +def test_workflow_protocol_caps_iterations_at_two(): + text = build_workflow_protocol() + # The string "Maximum 2" should appear once per phase = 3 times. + count = text.count("Maximum 2") + assert count == 3, f"Expected 'Maximum 2' to appear 3 times (once per phase); got {count}" + + +def test_workflow_protocol_describes_info_needed(): + text = build_workflow_protocol() + assert "info_needed" in text + assert "feedback" in text + + +def test_workflow_protocol_disambiguates_needs_approval(): + text = build_workflow_protocol() + assert "needs_approval" in text + assert "DISAMBIGUATION" in text + + +def test_workflow_protocol_under_token_budget(): + text = build_workflow_protocol() + # ~4 chars/token. 3200 chars ≈ 800 tokens budget. + assert len(text) < 3200, ( + f"WORKFLOW PROTOCOL is {len(text)} chars (~{len(text)//4} tokens); budget is 3200." + ) diff --git a/tests/test_gateway_discovery.py b/tests/test_gateway_discovery.py index 698445a..92ca725 100644 --- a/tests/test_gateway_discovery.py +++ b/tests/test_gateway_discovery.py @@ -132,7 +132,7 @@ def test_workspace_id_whitespace_stripped(self, mock_probe): # Integration tests — verify endpoint URLs constructed by setup scripts # --------------------------------------------------------------------------- -SETUP_DIR = Path(__file__).parent.parent +SETUP_DIR = Path(__file__).parent.parent / "setup" class TestEndpointConstruction: @@ -146,9 +146,11 @@ def _run_setup(self, script_name, tmp_path, env_overrides=None): "DATABRICKS_TOKEN": "dapi_test_token", "DATABRICKS_WORKSPACE_ID": "6280049833385130", "PATH": os.environ.get("PATH", ""), - "PYTHONPATH": str(SETUP_DIR), + "PYTHONPATH": str(SETUP_DIR.parent), # Pre-resolve gateway so subprocess skips the network probe "_GATEWAY_RESOLVED": "", + # Skip CLI install (curl | bash) — tests only verify config files + "SKIP_CLAUDE_INSTALL": "1", } # Ensure DATABRICKS_GATEWAY_HOST is NOT set (test auto-discovery) env.pop("DATABRICKS_GATEWAY_HOST", None) @@ -175,15 +177,15 @@ def test_setup_claude_falls_back_when_gateway_unreachable(self, tmp_path): # Gateway is unreachable from test env, so should fall back import json settings_path = tmp_path / ".claude" / "settings.json" - if settings_path.exists(): - settings = json.loads(settings_path.read_text()) - base_url = settings.get("env", {}).get("ANTHROPIC_BASE_URL", "") - assert base_url.endswith("/anthropic") - # Either gateway or serving-endpoints is valid - assert ( - "ai-gateway.cloud.databricks.com" in base_url - or "serving-endpoints/anthropic" in base_url - ) + assert settings_path.exists(), "settings.json was not written" + settings = json.loads(settings_path.read_text()) + base_url = settings.get("env", {}).get("ANTHROPIC_BASE_URL", "") + assert base_url.endswith("/anthropic") + # Either gateway or serving-endpoints is valid + assert ( + "ai-gateway.cloud.databricks.com" in base_url + or "serving-endpoints/anthropic" in base_url + ) def test_setup_claude_explicit_override(self, tmp_path): """setup_claude.py should prefer explicit DATABRICKS_GATEWAY_HOST.""" @@ -196,10 +198,10 @@ def test_setup_claude_explicit_override(self, tmp_path): import json settings_path = tmp_path / ".claude" / "settings.json" - if settings_path.exists(): - settings = json.loads(settings_path.read_text()) - base_url = settings.get("env", {}).get("ANTHROPIC_BASE_URL", "") - assert "custom.gateway.example.com" in base_url + assert settings_path.exists(), "settings.json was not written" + settings = json.loads(settings_path.read_text()) + base_url = settings.get("env", {}).get("ANTHROPIC_BASE_URL", "") + assert "custom.gateway.example.com" in base_url def test_setup_claude_fallback_no_gateway(self, tmp_path): """setup_claude.py falls back to DATABRICKS_HOST when no gateway available.""" @@ -210,10 +212,10 @@ def test_setup_claude_fallback_no_gateway(self, tmp_path): import json settings_path = tmp_path / ".claude" / "settings.json" - if settings_path.exists(): - settings = json.loads(settings_path.read_text()) - base_url = settings.get("env", {}).get("ANTHROPIC_BASE_URL", "") - assert "test.cloud.databricks.com/serving-endpoints/anthropic" in base_url + assert settings_path.exists(), "settings.json was not written" + settings = json.loads(settings_path.read_text()) + base_url = settings.get("env", {}).get("ANTHROPIC_BASE_URL", "") + assert "test.cloud.databricks.com/serving-endpoints/anthropic" in base_url @mock.patch("utils._probe_gateway", return_value=True) def test_codex_gateway_url_construction(self, mock_probe): diff --git a/tests/test_inbox_status_passthrough.py b/tests/test_inbox_status_passthrough.py new file mode 100644 index 0000000..bbcb8a9 --- /dev/null +++ b/tests/test_inbox_status_passthrough.py @@ -0,0 +1,56 @@ +"""Tests covering counts dict, coda_get_result docstring, and MCP instructions +all reflect the new info_needed / needs_approval terminal statuses.""" +import asyncio +import json + + +def test_mcp_instructions_mention_info_needed(): + """Server-level MCP instructions teach calling LLMs about info_needed.""" + from coda_mcp import mcp_server + + txt = mcp_server.mcp.instructions + assert "info_needed" in txt + assert "needs_approval" in txt + assert "feedback" in txt + + +def test_coda_get_result_docstring_mentions_info_needed(): + """coda_get_result docstring lists info_needed / needs_approval alongside completed/failed.""" + from coda_mcp import mcp_server + + doc = (mcp_server.coda_get_result.__doc__ or "").lower() + assert "info_needed" in doc + assert "needs_approval" in doc + + +def test_inbox_counts_dict_includes_new_statuses(monkeypatch): + """coda_inbox counts dict has info_needed and needs_approval keys.""" + from coda_mcp import mcp_server + + fake_tasks = [ + {"task_id": "t1", "session_id": "s1", "status": "running"}, + {"task_id": "t2", "session_id": "s2", "status": "completed"}, + {"task_id": "t3", "session_id": "s3", "status": "failed"}, + {"task_id": "t4", "session_id": "s4", "status": "info_needed"}, + {"task_id": "t5", "session_id": "s5", "status": "needs_approval"}, + {"task_id": "t6", "session_id": "s6", "status": "info_needed"}, + ] + + monkeypatch.setattr( + mcp_server.task_manager, "list_all_tasks", + lambda email, status_filter=None: list(fake_tasks), + ) + # _read_session_safe is called inside the loop; return None so no viewer_url is added. + monkeypatch.setattr( + mcp_server.task_manager, "_read_session_safe", lambda sid: None, + ) + + result_str = asyncio.run(mcp_server.coda_inbox(email="u@e")) + result = json.loads(result_str) + counts = result["counts"] + + assert counts["running"] == 1 + assert counts["completed"] == 1 + assert counts["failed"] == 1 + assert counts["info_needed"] == 2 + assert counts["needs_approval"] == 1 diff --git a/tests/test_mcp_endpoint.py b/tests/test_mcp_endpoint.py new file mode 100644 index 0000000..bc67e34 --- /dev/null +++ b/tests/test_mcp_endpoint.py @@ -0,0 +1,102 @@ +"""Unit tests for the Flask Blueprint fallback at coda_mcp.mcp_endpoint. + +Production traffic flows through coda_mcp.mcp_asgi (uvicorn + native MCP SDK). +This blueprint is the WSGI-only fallback. These tests pin the JSON-RPC contract +so the two paths stay in lockstep. +""" +import json + +import pytest + + +@pytest.fixture +def client(): + from app import app as flask_app + + return flask_app.test_client() + + +def _rpc(method, params=None, rpc_id=1): + return {"jsonrpc": "2.0", "id": rpc_id, "method": method, "params": params or {}} + + +def test_initialize_returns_server_info(client): + r = client.post("/mcp", json=_rpc("initialize", {"protocolVersion": "2025-03-26"})) + assert r.status_code == 200 + body = r.get_json() + assert body["jsonrpc"] == "2.0" + assert body["result"]["serverInfo"]["name"] == "coda" + assert "capabilities" in body["result"] + + +def test_tools_list_returns_v2_tools(client): + r = client.post("/mcp", json=_rpc("tools/list", {}, rpc_id=2)) + assert r.status_code == 200 + tools = r.get_json()["result"]["tools"] + names = {t["name"] for t in tools} + assert names == {"coda_run", "coda_inbox", "coda_get_result", "coda_interactive"}, ( + f"Tool surface drifted from the v2 contract (docs/mcp-v2-background-execution.md). Got: {names}" + ) + + +def test_tools_list_each_tool_has_description_and_schema(client): + r = client.post("/mcp", json=_rpc("tools/list", {}, rpc_id=3)) + for t in r.get_json()["result"]["tools"]: + assert t.get("description"), f"tool {t['name']} missing description (MCP requires it)" + assert isinstance(t.get("inputSchema"), dict), f"tool {t['name']} missing inputSchema" + + +def test_cors_preflight_returns_204(client): + r = client.options( + "/mcp", + headers={ + "Origin": "https://test.cloud.databricks.com", + "Access-Control-Request-Method": "POST", + }, + ) + assert r.status_code == 204 + assert "Access-Control-Allow-Origin" in r.headers + + +def test_ping_returns_empty_result(client): + r = client.post("/mcp", json=_rpc("ping", {}, rpc_id=4)) + assert r.status_code == 200 + body = r.get_json() + assert body["result"] == {} + assert "error" not in body + + +def test_unknown_method_returns_method_not_found(client): + r = client.post("/mcp", json=_rpc("does/not/exist", {}, rpc_id=5)) + body = r.get_json() + assert body.get("error", {}).get("code") == -32601, ( + f"Expected JSON-RPC method-not-found (-32601); got {body}" + ) + + +def test_unknown_tool_returns_jsonrpc_error(client): + r = client.post( + "/mcp", + json=_rpc("tools/call", {"name": "not_a_real_tool", "arguments": {}}, rpc_id=6), + ) + body = r.get_json() + assert "error" in body or ( + "result" in body and body["result"].get("isError") is True + ), f"Calling an unknown tool should error; got {body}" + + +def test_jsonrpc_id_is_echoed(client): + for rpc_id in (7, "string-id", 0): + r = client.post("/mcp", json=_rpc("ping", {}, rpc_id=rpc_id)) + assert r.get_json()["id"] == rpc_id + + +def test_post_with_non_json_body_does_not_crash(client): + r = client.post( + "/mcp", + data="not json at all", + headers={"Content-Type": "application/json"}, + ) + assert r.status_code in (200, 400) + if r.status_code == 200: + assert "error" in r.get_json() diff --git a/tests/test_mcp_env_strip.py b/tests/test_mcp_env_strip.py new file mode 100644 index 0000000..756f133 --- /dev/null +++ b/tests/test_mcp_env_strip.py @@ -0,0 +1,191 @@ +"""Tests for _build_terminal_shell_env's credential-stripping behavior. + +Replaces the inline 5-key strip that mcp_create_pty_session used to do. +Both create_session (HTTP path) and mcp_create_pty_session (MCP path) +now call this helper, so it must strip both the original 5 keys and +the registry-credential patterns the HTTP path was already covering. +""" +import os +import pytest + +from app import _build_terminal_shell_env + + +# Keys that must be absent from the child shell's env after the strip. +STRIPPED_KEYS = [ + "CLAUDECODE", + "CLAUDE_CODE_SESSION", + "DATABRICKS_TOKEN", + "DATABRICKS_HOST", + "GEMINI_API_KEY", + "NPM_TOKEN", + "UV_DEFAULT_INDEX", + "UV_INDEX_MYREG_PASSWORD", + "UV_INDEX_MYREG_USERNAME", + "npm_config_//registry.example/:_authToken", +] + + +@pytest.mark.parametrize("key", STRIPPED_KEYS) +def test_build_terminal_shell_env_strips_credential_key(key): + """Each known credential / registry-auth key is stripped from the child env.""" + fake_env = { + "PATH": "/usr/bin:/usr/local/bin", # positive control — must survive + "HOME": "/home/test", + key: "leak-me-test-value", + } + result = _build_terminal_shell_env(fake_env) + assert key not in result, ( + f"{key} survived the strip — registry/auth credential leaked into " + f"the child shell's env. Result keys: {sorted(result)}" + ) + + +def test_build_terminal_shell_env_preserves_benign_keys(): + """Positive control: non-credential keys survive the strip. + + Guards against a future regression where the strip becomes too aggressive + and wipes the env entirely. If THIS test fails, the negative assertions + above would silently pass for the wrong reason. + """ + fake_env = { + "PATH": "/usr/bin:/usr/local/bin", + "HOME": "/home/test", + "LANG": "en_US.UTF-8", + } + result = _build_terminal_shell_env(fake_env) + assert result.get("PATH") and "/usr/bin" in result["PATH"] + assert result.get("HOME") == "/home/test" + assert result.get("LANG") == "en_US.UTF-8" + + +try: + import pty as _pty + _master, _slave = _pty.openpty() + os.close(_master) + os.close(_slave) + _PTY_AVAILABLE = True +except Exception: + _PTY_AVAILABLE = False + +_pty_skip = pytest.mark.skipif( + not _PTY_AVAILABLE, + reason="PTY not allocatable in this environment", +) + + +@_pty_skip +def test_mcp_create_pty_session_respects_cwd_kwarg(tmp_path): + """When cwd is passed, sessions[sid]['cwd'] records it.""" + from app import mcp_create_pty_session, mcp_close_pty_session, sessions + + sid = None + try: + sid = mcp_create_pty_session(label="t-cwd", cwd=str(tmp_path)) + assert sessions[sid].get("cwd") == str(tmp_path) + finally: + if sid is not None: + mcp_close_pty_session(sid) + + +@_pty_skip +def test_mcp_create_pty_session_cwd_defaults_to_none(): + """When cwd is not passed, sessions[sid]['cwd'] is None (preserves current behavior).""" + from app import mcp_create_pty_session, mcp_close_pty_session, sessions + + sid = None + try: + sid = mcp_create_pty_session(label="t-no-cwd") + assert sessions[sid].get("cwd") is None + finally: + if sid is not None: + mcp_close_pty_session(sid) + + +@_pty_skip +def test_mcp_close_pty_session_removes_project_dir(tmp_path, monkeypatch): + """When the PTY is closed, any project dir at ~/.coda/projects// is removed.""" + import os + from app import mcp_create_pty_session, mcp_close_pty_session + + # Point HOME at tmp_path so ~/.coda lives in a controllable place. + monkeypatch.setenv("HOME", str(tmp_path)) + + sid = None + try: + sid = mcp_create_pty_session(label="t-cleanup") + + project_dir = os.path.join(str(tmp_path), ".coda", "projects", sid) + os.makedirs(project_dir, exist_ok=True) + sentinel = os.path.join(project_dir, "SENTINEL") + with open(sentinel, "w") as f: + f.write("present-before-close") + assert os.path.exists(sentinel) + + mcp_close_pty_session(sid) + sid = None # session closed; don't double-close in finally + + assert not os.path.exists(project_dir), \ + f"Expected project dir to be removed after PTY close: {project_dir} still exists" + finally: + if sid is not None: + try: + mcp_close_pty_session(sid) + except Exception: + pass + + +@_pty_skip +def test_mcp_close_pty_session_handles_missing_project_dir(tmp_path, monkeypatch): + """No project dir present → close still succeeds (no exception).""" + from app import mcp_create_pty_session, mcp_close_pty_session + + monkeypatch.setenv("HOME", str(tmp_path)) + + sid = None + try: + sid = mcp_create_pty_session(label="t-no-projdir") + # Do NOT create the project dir — verify close still works. + mcp_close_pty_session(sid) # must not raise + sid = None + finally: + if sid is not None: + try: + mcp_close_pty_session(sid) + except Exception: + pass + + +@_pty_skip +def test_terminate_session_removes_project_dir(tmp_path, monkeypatch): + """The idle reaper calls terminate_session directly. Project dir must still be cleaned.""" + import os + from app import mcp_create_pty_session, sessions, terminate_session, mcp_close_pty_session + + monkeypatch.setenv("HOME", str(tmp_path)) + + sid = None + try: + sid = mcp_create_pty_session(label="t-reaper-cleanup") + + # Plant a project dir like coda_interactive would have done. + project_dir = os.path.join(str(tmp_path), ".coda", "projects", sid) + os.makedirs(project_dir, exist_ok=True) + with open(os.path.join(project_dir, "SENTINEL"), "w") as f: + f.write("present") + assert os.path.exists(project_dir) + + # Simulate the reaper's code path: call terminate_session directly. + sess = sessions[sid] + terminate_session(sid, sess["pid"], sess["master_fd"]) + sid = None # session terminated; finally is a no-op + + # Project dir must be removed even though we bypassed mcp_close_pty_session. + assert not os.path.exists(project_dir), \ + "Reaper path must also clean up the project dir — fix terminate_session not mcp_close_pty_session" + finally: + if sid is not None: + try: + mcp_close_pty_session(sid) + except Exception: + pass diff --git a/tests/test_mcp_integration.py b/tests/test_mcp_integration.py new file mode 100644 index 0000000..215d616 --- /dev/null +++ b/tests/test_mcp_integration.py @@ -0,0 +1,292 @@ +"""End-to-end MCP integration tests — v2 background execution + inbox API. + +Exercises the full flow: coda_run -> coda_inbox -> coda_get_result. +No real PTY — app hooks are mocked. +""" + +import json +import os +import time +from unittest.mock import MagicMock + +import pytest + + +# ── helpers ────────────────────────────────────────────────────────── + + +def _parse(result: str) -> dict: + """Parse JSON string returned by MCP tools.""" + return json.loads(result) + + +# ── fixture ────────────────────────────────────────────────────────── + + +@pytest.fixture(autouse=True) +def isolated_env(tmp_path): + """Redirect state to tmp and mock PTY hooks.""" + from coda_mcp import task_manager as tm + from coda_mcp import mcp_server as ms + + original_dir = tm.SESSIONS_DIR + tm.SESSIONS_DIR = str(tmp_path / "sessions") + + mock_send = MagicMock() + mock_close = MagicMock() + ms.set_app_hooks( + create_session_fn=lambda label, **kwargs: f"pty-mock-{label}", + send_input_fn=mock_send, + close_session_fn=mock_close, + ) + + yield {"tmp": tmp_path, "mock_send": mock_send, "mock_close": mock_close} + + tm.SESSIONS_DIR = original_dir + ms.set_app_hooks(None, None, None) + + +# ── 1. Happy-path: fire-and-forget → inbox → result ───────────────── + + +class TestFullMcpFlow: + @pytest.mark.asyncio + async def test_full_background_flow(self, isolated_env): + """Happy path: run (fire-and-forget) → inbox → result.""" + from coda_mcp import mcp_server as ms + from coda_mcp import task_manager as tm + + # Step 1: submit task (returns immediately) + with MagicMock() as mock_thread: + from coda_mcp import mcp_server + with pytest.MonkeyPatch.context() as mp: + mp.setattr("coda_mcp.mcp_server.threading", mock_thread) + raw = await ms.coda_run( + prompt="create a sales pipeline", + email="alice@test.com", + context='{"tables": ["sales.transactions"]}', + ) + + task = _parse(raw) + assert task["status"] == "running" + task_id = task["task_id"] + session_id = task["session_id"] + assert task_id.startswith("task-") + assert session_id.startswith("sess-") + + # Step 2: inbox shows running task + raw = await ms.coda_inbox() + inbox = _parse(raw) + assert len(inbox["tasks"]) == 1 + assert inbox["tasks"][0]["task_id"] == task_id + assert inbox["tasks"][0]["status"] == "running" + assert inbox["counts"]["running"] == 1 + + # Step 3: simulate agent writing result.json + tdir = tm._task_dir(session_id, task_id) + result_path = os.path.join(tdir, "result.json") + with open(result_path, "w") as f: + json.dump({ + "status": "completed", + "summary": "Created sales pipeline with 3 stages", + "files_changed": ["pipeline.py", "config.yaml"], + "artifacts": ["/workspace/pipeline.py"], + "errors": [], + }, f) + + # Step 4: complete_task (simulating what _watch_task does) + tm.complete_task(session_id, task_id) + + # Step 5: inbox shows completed + raw = await ms.coda_inbox() + inbox = _parse(raw) + assert len(inbox["tasks"]) == 1 + assert inbox["tasks"][0]["status"] == "completed" + assert inbox["tasks"][0]["summary"] == "Created sales pipeline with 3 stages" + assert inbox["counts"]["completed"] == 1 + + # Step 6: get full result + raw = await ms.coda_get_result(task_id=task_id, session_id=session_id) + result = _parse(raw) + assert result["task_id"] == task_id + assert result["summary"] == "Created sales pipeline with 3 stages" + assert result["files_changed"] == ["pipeline.py", "config.yaml"] + + # Step 7: session was auto-closed + session = tm._read_session(session_id) + assert session["status"] == "closed" + + +# ── 2. Task chaining with previous_session_id ─────────────────────── + + +class TestTaskChaining: + @pytest.mark.asyncio + async def test_chained_task_references_prior_session(self, isolated_env): + """A chained task includes prior session context in prompt.""" + from coda_mcp import mcp_server as ms + from coda_mcp import task_manager as tm + + # First task + raw = await ms.coda_run( + prompt="build pipeline", + email="bob@test.com", + ) + first = _parse(raw) + first_sid = first["session_id"] + first_tid = first["task_id"] + + # Complete first task + tdir = tm._task_dir(first_sid, first_tid) + with open(os.path.join(tdir, "result.json"), "w") as f: + json.dump({ + "status": "completed", + "summary": "Built pipeline.py", + "files_changed": ["pipeline.py"], + }, f) + tm.complete_task(first_sid, first_tid) + + # Second task chained to first + raw = await ms.coda_run( + prompt="add tests for the pipeline", + email="bob@test.com", + previous_session_id=first_sid, + ) + second = _parse(raw) + second_sid = second["session_id"] + second_tid = second["task_id"] + + # Verify prompt references prior session + prompt_path = os.path.join( + tm._task_dir(second_sid, second_tid), "prompt.txt" + ) + with open(prompt_path) as f: + prompt_text = f.read() + assert f"PRIOR SESSION: {first_sid}" in prompt_text + + # Verify meta.json has previous_session_id + meta_path = os.path.join( + tm._task_dir(second_sid, second_tid), "meta.json" + ) + with open(meta_path) as f: + meta = json.load(f) + assert meta["previous_session_id"] == first_sid + + # Verify inbox shows chaining + raw = await ms.coda_inbox() + inbox = _parse(raw) + running_tasks = [t for t in inbox["tasks"] if t["status"] == "running"] + assert len(running_tasks) == 1 + assert running_tasks[0]["previous_session_id"] == first_sid + + +# ── 3. Concurrency limit ──────────────────────────────────────────── + + +class TestConcurrencyLimit: + @pytest.mark.asyncio + async def test_exceeding_limit_returns_error(self, isolated_env): + """Exceeding MAX_CONCURRENT_TASKS returns a clear error.""" + from coda_mcp import mcp_server as ms + from unittest.mock import patch + + with patch("coda_mcp.task_manager.MAX_CONCURRENT_TASKS", 1): + r1 = await ms.coda_run(prompt="task1", email="a@b.com") + assert _parse(r1)["status"] == "running" + + r2 = await ms.coda_run(prompt="task2", email="a@b.com") + d2 = _parse(r2) + assert d2["status"] == "error" + assert "concurrency" in d2["error"].lower() + + +# ── 4. Yolo permissions → --yolo flag ─────────────────────────────── + + +class TestYoloPermissions: + @pytest.mark.asyncio + async def test_yolo_permissions(self, isolated_env): + """permissions='yolo' causes the PTY command to include --yolo.""" + from coda_mcp import mcp_server as ms + + mock_send = isolated_env["mock_send"] + + with MagicMock() as mock_thread: + from coda_mcp import mcp_server + with pytest.MonkeyPatch.context() as mp: + mp.setattr("coda_mcp.mcp_server.threading", mock_thread) + await ms.coda_run( + prompt="deploy everything", + email="dave@test.com", + permissions="yolo", + ) + + mock_send.assert_called_once() + cmd = mock_send.call_args[0][1] + assert "--yolo" in cmd + + +# ── 5. Session auto-close on completion ────────────────────────────── + + +class TestAutoClose: + @pytest.mark.asyncio + async def test_session_auto_closes(self, isolated_env): + """Session is auto-closed when task completes.""" + from coda_mcp import mcp_server as ms + from coda_mcp import task_manager as tm + + raw = await ms.coda_run(prompt="quick job", email="a@b.com") + d = _parse(raw) + + # Session should be busy + session = tm._read_session(d["session_id"]) + assert session["status"] == "busy" + + # Complete the task + tdir = tm._task_dir(d["session_id"], d["task_id"]) + with open(os.path.join(tdir, "result.json"), "w") as f: + json.dump({"status": "completed", "summary": "done"}, f) + tm.complete_task(d["session_id"], d["task_id"]) + + # Session should now be closed + session = tm._read_session(d["session_id"]) + assert session["status"] == "closed" + assert "closed_at" in session + + +# ── 6. Cleanup expired tasks ──────────────────────────────────────── + + +class TestCleanup: + @pytest.mark.asyncio + async def test_cleanup_removes_expired(self, isolated_env): + """cleanup_expired_tasks removes old closed sessions.""" + from coda_mcp import mcp_server as ms + from coda_mcp import task_manager as tm + from unittest.mock import patch + + raw = await ms.coda_run(prompt="old task", email="a@b.com") + d = _parse(raw) + + # Complete and close + tdir = tm._task_dir(d["session_id"], d["task_id"]) + with open(os.path.join(tdir, "result.json"), "w") as f: + json.dump({"status": "completed", "summary": "done"}, f) + tm.complete_task(d["session_id"], d["task_id"]) + + # Backdate closed_at to expire it + session = tm._read_session(d["session_id"]) + session["closed_at"] = time.time() - 90000 # 25 hours ago + tm._write_json(tm._session_file(d["session_id"]), session) + + # Cleanup should remove it + removed = tm.cleanup_expired_tasks() + assert removed == 1 + + # Inbox should be empty now + raw = await ms.coda_inbox() + inbox = _parse(raw) + assert len(inbox["tasks"]) == 0 + + diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py new file mode 100644 index 0000000..dd7d004 --- /dev/null +++ b/tests/test_mcp_server.py @@ -0,0 +1,481 @@ +"""Tests for mcp_server — v2 background execution + inbox API.""" + +import asyncio +import json +import os +from unittest import mock + +import pytest +from coda_mcp import mcp_server, task_manager, url_builder + + +# ── helpers ────────────────────────────────────────────────────────── + + +@pytest.fixture(autouse=True) +def _reset_hooks(): + """Clear app hooks before/after each test.""" + from coda_mcp import mcp_server + + mcp_server._app_create_session = None + mcp_server._app_send_input = None + mcp_server._app_close_session = None + yield + mcp_server._app_create_session = None + mcp_server._app_send_input = None + mcp_server._app_close_session = None + + +@pytest.fixture(autouse=True) +def _isolated_sessions(tmp_path): + """Point task_manager.SESSIONS_DIR at a temp dir.""" + sessions_dir = str(tmp_path / ".coda" / "sessions") + with mock.patch("coda_mcp.task_manager.SESSIONS_DIR", sessions_dir): + yield sessions_dir + + +def _parse(result: str) -> dict: + """Parse JSON string returned by MCP tools.""" + return json.loads(result) + + +# ── Tool registration ──────────────────────────────────────────────── + + +class TestToolRegistration: + def test_tools_registered(self): + from coda_mcp import mcp_server + + tool_mgr = mcp_server.mcp._tool_manager + tool_names = set(tool_mgr._tools.keys()) + expected = {"coda_run", "coda_inbox", "coda_get_result", "coda_interactive"} + assert expected == tool_names, f"Expected {expected}, got {tool_names}" + + def test_tool_count(self): + from coda_mcp import mcp_server + + tool_mgr = mcp_server.mcp._tool_manager + assert len(tool_mgr._tools) == 4 + + +# ── coda_run ───────────────────────────────────────────────────────── + + +class TestCodaRun: + @pytest.mark.asyncio + async def test_creates_task_disk_only(self): + """Without app hooks, creates session+task on disk, returns immediately.""" + from coda_mcp import mcp_server + + result = await mcp_server.coda_run( + prompt="fix the bug", + email="a@b.com", + ) + data = _parse(result) + assert data["status"] == "running" + assert data["task_id"].startswith("task-") + assert data["session_id"].startswith("sess-") + + @pytest.mark.asyncio + async def test_auto_creates_session(self): + """coda_run auto-creates a session — no separate create_session needed.""" + from coda_mcp import mcp_server + from coda_mcp import task_manager + + result = await mcp_server.coda_run( + prompt="build pipeline", + email="a@b.com", + ) + data = _parse(result) + session = task_manager._read_session(data["session_id"]) + assert session["email"] == "a@b.com" + assert session["status"] == "busy" # task is running + + @pytest.mark.asyncio + async def test_sends_to_pty_when_hooks_set(self): + """With hooks, creates PTY and sends hermes command.""" + from coda_mcp import mcp_server + + mock_create = mock.Mock(return_value="pty-xyz") + mock_send = mock.Mock() + mcp_server.set_app_hooks( + create_session_fn=mock_create, + send_input_fn=mock_send, + close_session_fn=mock.Mock(), + ) + + with mock.patch("coda_mcp.mcp_server.threading"): + result = await mcp_server.coda_run( + prompt="fix the bug", + email="a@b.com", + ) + + data = _parse(result) + assert data["status"] == "running" + mock_create.assert_called_once() + call_kwargs = mock_create.call_args.kwargs + assert call_kwargs["label"] == "hermes-mcp" + assert "transcript_path" in call_kwargs + mock_send.assert_called_once() + assert "hermes" in mock_send.call_args[0][1] + + @pytest.mark.asyncio + async def test_yolo_permission(self): + """permissions='yolo' produces --yolo flag in PTY command.""" + from coda_mcp import mcp_server + + mock_send = mock.Mock() + mcp_server.set_app_hooks( + create_session_fn=mock.Mock(return_value="pty-1"), + send_input_fn=mock_send, + close_session_fn=mock.Mock(), + ) + + with mock.patch("coda_mcp.mcp_server.threading"): + await mcp_server.coda_run( + prompt="go fast", + email="a@b.com", + permissions="yolo", + ) + + cmd = mock_send.call_args[0][1] + assert "--yolo" in cmd + + @pytest.mark.asyncio + async def test_previous_session_id_in_prompt(self): + """previous_session_id appears in the wrapped prompt.""" + from coda_mcp import mcp_server + from coda_mcp import task_manager + + # Create a "prior" session with a completed task + prior = task_manager.create_session("a@b.com", "u1") + prior_sid = prior["session_id"] + + result = await mcp_server.coda_run( + prompt="add tests", + email="a@b.com", + previous_session_id=prior_sid, + ) + data = _parse(result) + + # Read the prompt.txt and verify prior session reference + tdir = task_manager._task_dir(data["session_id"], data["task_id"]) + with open(os.path.join(tdir, "prompt.txt")) as f: + prompt_text = f.read() + + assert f"PRIOR SESSION: {prior_sid}" in prompt_text + + @pytest.mark.asyncio + async def test_meta_json_written(self): + """coda_run writes meta.json with task metadata.""" + from coda_mcp import mcp_server + from coda_mcp import task_manager + + result = await mcp_server.coda_run( + prompt="build a dashboard for sales", + email="alice@test.com", + previous_session_id="sess-old", + ) + data = _parse(result) + + meta_path = os.path.join( + task_manager._task_dir(data["session_id"], data["task_id"]), + "meta.json", + ) + with open(meta_path) as f: + meta = json.load(f) + + assert meta["email"] == "alice@test.com" + assert meta["previous_session_id"] == "sess-old" + assert meta["prompt_summary"] == "build a dashboard for sales" + assert "created_at" in meta + + @pytest.mark.asyncio + async def test_concurrency_limit(self): + """Exceeding MAX_CONCURRENT_TASKS returns an error.""" + from coda_mcp import mcp_server + + with mock.patch("coda_mcp.task_manager.MAX_CONCURRENT_TASKS", 1): + # First task succeeds + r1 = await mcp_server.coda_run(prompt="task1", email="a@b.com") + assert _parse(r1)["status"] == "running" + + # Second task should fail (1 already running) + r2 = await mcp_server.coda_run(prompt="task2", email="a@b.com") + d2 = _parse(r2) + assert d2["status"] == "error" + assert "concurrency" in d2["error"].lower() + + +# ── coda_inbox ─────────────────────────────────────────────────────── + + +class TestCodaInbox: + @pytest.mark.asyncio + async def test_empty_inbox(self): + """No tasks → empty inbox.""" + from coda_mcp import mcp_server + + result = await mcp_server.coda_inbox() + data = _parse(result) + assert data["tasks"] == [] + assert data["counts"] == {"running": 0, "completed": 0, "failed": 0, "info_needed": 0, "needs_approval": 0} + + @pytest.mark.asyncio + async def test_running_task_in_inbox(self): + """A running task shows up in the inbox.""" + from coda_mcp import mcp_server + + await mcp_server.coda_run(prompt="build pipeline", email="a@b.com") + + result = await mcp_server.coda_inbox() + data = _parse(result) + assert len(data["tasks"]) == 1 + assert data["tasks"][0]["status"] == "running" + assert data["tasks"][0]["prompt_summary"] == "build pipeline" + assert data["counts"]["running"] == 1 + + @pytest.mark.asyncio + async def test_completed_task_in_inbox(self): + """A completed task shows summary in inbox.""" + from coda_mcp import mcp_server + from coda_mcp import task_manager + + r = await mcp_server.coda_run(prompt="fix bug", email="a@b.com") + d = _parse(r) + + # Simulate agent writing result.json + tdir = task_manager._task_dir(d["session_id"], d["task_id"]) + result_path = os.path.join(tdir, "result.json") + with open(result_path, "w") as f: + json.dump({ + "status": "completed", + "summary": "Fixed the login bug", + "files_changed": ["auth.py"], + "artifacts": [], + "errors": [], + }, f) + + result = await mcp_server.coda_inbox() + data = _parse(result) + assert len(data["tasks"]) == 1 + assert data["tasks"][0]["status"] == "completed" + assert data["tasks"][0]["summary"] == "Fixed the login bug" + + @pytest.mark.asyncio + async def test_status_filter(self): + """Filtering inbox by status works.""" + from coda_mcp import mcp_server + from coda_mcp import task_manager + + # Create two tasks — one running, one completed + r1 = await mcp_server.coda_run(prompt="task1", email="a@b.com") + d1 = _parse(r1) + + r2 = await mcp_server.coda_run(prompt="task2", email="a@b.com") + d2 = _parse(r2) + + # Complete task2 + tdir = task_manager._task_dir(d2["session_id"], d2["task_id"]) + with open(os.path.join(tdir, "result.json"), "w") as f: + json.dump({"status": "completed", "summary": "done"}, f) + + # Filter running only + result = await mcp_server.coda_inbox(status="running") + data = _parse(result) + assert len(data["tasks"]) == 1 + assert data["tasks"][0]["task_id"] == d1["task_id"] + + @pytest.mark.asyncio + async def test_multiple_tasks_sorted_recent_first(self): + """Inbox returns tasks sorted most recent first.""" + from coda_mcp import mcp_server + + await mcp_server.coda_run(prompt="first", email="a@b.com") + await mcp_server.coda_run(prompt="second", email="a@b.com") + + result = await mcp_server.coda_inbox() + data = _parse(result) + assert len(data["tasks"]) == 2 + # Most recent first + assert data["tasks"][0]["prompt_summary"] == "second" + assert data["tasks"][1]["prompt_summary"] == "first" + + +# ── coda_get_result ────────────────────────────────────────────────── + + +class TestCodaGetResult: + @pytest.mark.asyncio + async def test_returns_result(self): + from coda_mcp import mcp_server + from coda_mcp import task_manager + + r = await mcp_server.coda_run(prompt="go", email="a@b.com") + d = _parse(r) + + # Simulate agent writing result.json + tdir = task_manager._task_dir(d["session_id"], d["task_id"]) + with open(os.path.join(tdir, "result.json"), "w") as f: + json.dump({ + "summary": "Fixed the bug", + "files_changed": ["app.py"], + "artifacts": [], + "errors": [], + }, f) + + result = await mcp_server.coda_get_result( + task_id=d["task_id"], session_id=d["session_id"] + ) + data = _parse(result) + assert data["task_id"] == d["task_id"] + assert data["session_id"] == d["session_id"] + assert data["summary"] == "Fixed the bug" + + @pytest.mark.asyncio + async def test_no_result_yet(self): + from coda_mcp import mcp_server + + r = await mcp_server.coda_run(prompt="go", email="a@b.com") + d = _parse(r) + + result = await mcp_server.coda_get_result( + task_id=d["task_id"], session_id=d["session_id"] + ) + data = _parse(result) + assert data["status"] == "running" + assert "not yet available" in data["message"] + + +# ── viewer_url + transcript_path wiring ───────────────────────────── + + +def _run(coro): + return asyncio.get_event_loop().run_until_complete(coro) if not asyncio.iscoroutine(coro) else asyncio.run(coro) + + +def test_coda_run_includes_viewer_url_when_builder_returns_one(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + monkeypatch.setattr(url_builder, "_app_url_cache", "app.example.com") + + create = mock.MagicMock(return_value="pty-abc") + send = mock.MagicMock() + closer = mock.MagicMock() + mcp_server.set_app_hooks(create, send, closer) + + result_json = asyncio.run(mcp_server.coda_run(prompt="do it", email="u@x")) + result = json.loads(result_json) + assert result["status"] == "running" + assert "?session=pty-abc" in result["viewer_url"] + assert result["viewer_url"].startswith("https://app.example.com") + + +def test_coda_run_omits_viewer_url_when_builder_returns_none(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + monkeypatch.setattr(url_builder, "_app_url_cache", None) + monkeypatch.delenv("CODA_APP_URL", raising=False) + + create = mock.MagicMock(return_value="pty-abc") + mcp_server.set_app_hooks(create, mock.MagicMock(), mock.MagicMock()) + + result_json = asyncio.run(mcp_server.coda_run(prompt="do it", email="u@x")) + result = json.loads(result_json) + # viewer_url present but None when builder returns None + assert result.get("viewer_url") is None + + +def test_coda_run_passes_transcript_path_to_create_session(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + create = mock.MagicMock(return_value="pty-abc") + mcp_server.set_app_hooks(create, mock.MagicMock(), mock.MagicMock()) + + asyncio.run(mcp_server.coda_run(prompt="do it", email="u@x")) + # create_session was called with transcript_path=... pointing into ~/.coda/sessions//tasks//transcript.log + kwargs = create.call_args.kwargs + assert "transcript_path" in kwargs + assert kwargs["transcript_path"].endswith("transcript.log") + assert "tasks" in kwargs["transcript_path"] + + +def test_coda_inbox_decorates_each_task_with_viewer_url(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + monkeypatch.setattr(url_builder, "_app_url_cache", "app.example.com") + + # Seed one session with one task and a pty_session_id + s = task_manager.create_session("u@x", "uid", label="t") + sid = s["session_id"] + task_manager._update_session_field(sid, "pty_session_id", "pty-xyz") + task_manager.create_task(sid, "prompt", "u@x") + + result_json = asyncio.run(mcp_server.coda_inbox()) + result = json.loads(result_json) + assert len(result["tasks"]) == 1 + assert "viewer_url" in result["tasks"][0] + assert "?session=pty-xyz" in result["tasks"][0]["viewer_url"] + + +def test_coda_get_result_includes_viewer_url(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + monkeypatch.setattr(url_builder, "_app_url_cache", "app.example.com") + + s = task_manager.create_session("u@x", "uid", label="t") + sid = s["session_id"] + task_manager._update_session_field(sid, "pty_session_id", "pty-xyz") + t = task_manager.create_task(sid, "prompt", "u@x") + tid = t["task_id"] + tdir = task_manager._task_dir(sid, tid) + task_manager._write_json(tdir + "/result.json", { + "status": "completed", "summary": "ok", + }) + + result_json = asyncio.run(mcp_server.coda_get_result(tid, sid)) + result = json.loads(result_json) + assert "viewer_url" in result + assert "?session=pty-xyz" in result["viewer_url"] + + +class TestInteractiveHelpers: + def test_safe_dirname_basename(self): + from coda_mcp.mcp_server import _safe_dirname + assert _safe_dirname("/Users/x@y.com/WAM") == "WAM" + assert _safe_dirname("/Users/x@y.com/WAM/") == "WAM" + + def test_safe_dirname_sanitizes(self): + from coda_mcp.mcp_server import _safe_dirname + assert _safe_dirname("/Users/x/My Project!") == "My_Project_" + + def test_safe_dirname_empty_fallback(self): + from coda_mcp.mcp_server import _safe_dirname + assert _safe_dirname("/") == "workspace" + assert _safe_dirname("") == "workspace" + + def test_safe_dirname_rejects_traversal(self): + from coda_mcp.mcp_server import _safe_dirname + assert _safe_dirname("/foo/..") == "workspace" + assert _safe_dirname("/foo/.") == "workspace" + + def test_normalize_strips_workspace_prefix(self): + from coda_mcp.mcp_server import _normalize_workspace_path + assert _normalize_workspace_path("/Workspace/Users/x/WAM") == "/Users/x/WAM" + + def test_normalize_leaves_plain_path(self): + from coda_mcp.mcp_server import _normalize_workspace_path + assert _normalize_workspace_path("/Users/x/WAM") == "/Users/x/WAM" + assert _normalize_workspace_path("/Users/x/WAM/") == "/Users/x/WAM" + + @pytest.mark.asyncio + async def test_wait_for_agent_ready_delegates(self, monkeypatch): + """_wait_for_agent_ready calls _wait_for_output_stable with prompt-seed constants.""" + from coda_mcp import mcp_server + seen = {} + + async def fake_stable(pty, max_wait, stability): + seen["args"] = (pty, max_wait, stability) + + monkeypatch.setattr(mcp_server, "_wait_for_output_stable", fake_stable) + await mcp_server._wait_for_agent_ready("pty-1") + assert seen["args"] == ( + "pty-1", + mcp_server._PROMPT_SEED_MAX_WAIT_S, + mcp_server._PROMPT_SEED_STABILITY_S, + ) diff --git a/tests/test_mlflow_tracing.py b/tests/test_mlflow_tracing.py index fb6e975..c72113f 100644 --- a/tests/test_mlflow_tracing.py +++ b/tests/test_mlflow_tracing.py @@ -14,7 +14,7 @@ # Helpers # --------------------------------------------------------------------------- -SETUP_MLFLOW = Path(__file__).parent.parent / "setup_mlflow.py" +SETUP_MLFLOW = Path(__file__).parent.parent / "setup" / "setup_mlflow.py" def run_setup_mlflow(tmp_path, env_overrides=None): diff --git a/tests/test_npm_version_pinning.py b/tests/test_npm_version_pinning.py index ee128dd..a155596 100644 --- a/tests/test_npm_version_pinning.py +++ b/tests/test_npm_version_pinning.py @@ -318,19 +318,40 @@ def test_zero_disables_cooldown_via_env(self, monkeypatch): # 5. Live integration (runs actual npm, skip if npm not available) # --------------------------------------------------------------------------- +def _npm_live_unavailable(): + """True when the live npm-registry probe can't run — SKIP, never ERROR. + + The probe must not raise: a registry timeout/connection failure was being + surfaced as a pytest *collection error* (the old skipif ran the subprocess + inline, so TimeoutExpired propagated out of the condition). Catch everything + and treat any failure as 'skip this live test'.""" + import shutil + import subprocess + if not shutil.which("npm"): + return True + try: + return subprocess.run( + ["npm", "view", "npm", "version"], + capture_output=True, timeout=15, + ).returncode != 0 + except Exception: + return True + + class TestNpmVersionLive: """Run against real npm registry to verify the function works end-to-end.""" @pytest.mark.skipif( - not __import__("shutil").which("npm"), - reason="npm not installed" + _npm_live_unavailable(), + reason="npm not installed or npm registry unreachable", ) def test_resolves_real_package(self): get_npm_version = _get_npm_version() # Use fast path (no cooldown) so this test isn't sensitive to recent # publishes — it's a sanity check that npm + the network work. version = get_npm_version("opencode-ai", min_age_days=0) - assert version is not None + if version is None: + pytest.skip("npm registry returned no version (network/registry flake, not a code bug)") # Version should look like a semver (X.Y.Z) parts = version.split(".") assert len(parts) >= 2, f"Expected semver, got: {version}" diff --git a/tests/test_replay_attach.py b/tests/test_replay_attach.py new file mode 100644 index 0000000..89a893e --- /dev/null +++ b/tests/test_replay_attach.py @@ -0,0 +1,105 @@ +"""Tests for /api/session/attach replay fallback.""" +import json +import os +from pathlib import Path + +import pytest + +from coda_mcp import task_manager + +try: + import pty as _pty + _master, _slave = _pty.openpty() + os.close(_master) + os.close(_slave) + _PTY_AVAILABLE = True +except Exception: + _PTY_AVAILABLE = False + +_pty_skip = pytest.mark.skipif( + not _PTY_AVAILABLE, + reason="PTY not allocatable in this environment", +) + + +@pytest.fixture +def client(tmp_path, monkeypatch): + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + monkeypatch.setenv("MAX_CONCURRENT_SESSIONS", "5") + import app as app_module + # Set app_owner so check_authorization returns (True, None) for requests + # with no user header (same pattern used by test_session_detach.py) + app_module.app_owner = "test@example.com" + with app_module.app.test_client() as c: + yield c, tmp_path + + +def _seed_transcript(sessions_root: Path, pty_id: str, content: bytes) -> None: + sess_id = "sess-test" + task_id = "task-test" + sdir = sessions_root / sess_id + tdir = sdir / "tasks" / task_id + tdir.mkdir(parents=True) + (sdir / "session.json").write_text(json.dumps({ + "session_id": sess_id, + "pty_session_id": pty_id, + "current_task": None, + "completed_tasks": [task_id], + "status": "closed", + })) + (tdir / "transcript.log").write_bytes(content) + + +def test_attach_returns_replay_when_pty_gone_and_transcript_exists(client): + c, root = client + _seed_transcript(root, "pty-gone", b"hello\r\nworld\r\n") + resp = c.post("/api/session/attach", json={"session_id": "pty-gone"}) + assert resp.status_code == 200 + data = resp.get_json() + assert data["replay"] is True + assert data["output"] == ["hello\r\nworld\r\n"] + assert data["label"] == "hermes-mcp (replay)" + + +def test_attach_404_when_pty_gone_and_no_transcript(client): + c, root = client + resp = c.post("/api/session/attach", json={"session_id": "pty-nope"}) + assert resp.status_code == 404 + + +@_pty_skip +def test_attach_session_returns_replay_for_alive_replay_only_pty(tmp_path, monkeypatch): + """A PTY created with `replay_only=True` (the flag introduced by coda_run's contract) that is still alive serves the transcript-from-disk, not the live output_buffer. + + This is the new contract introduced by the replay-only flag — historically + a live PTY would serve its output_buffer. + """ + from app import app as flask_app, mcp_create_pty_session, mcp_close_pty_session + from coda_mcp import task_manager + + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + + sid = None + try: + sid = mcp_create_pty_session(label="replay-alive", replay_only=True) + sess_id = "sess-x" + task_id = "task-x" + sdir = tmp_path / sess_id + tdir = sdir / "tasks" / task_id + tdir.mkdir(parents=True) + (sdir / "session.json").write_text( + '{"session_id": "%s", "pty_session_id": "%s", "current_task": "%s"}' % (sess_id, sid, task_id) + ) + (tdir / "transcript.log").write_bytes(b"FROM DISK") + # Cache may have stale entries from earlier tests — clear before the lookup. + task_manager._pty_lookup_cache.clear() + + client = flask_app.test_client() + resp = client.post("/api/session/attach", json={"session_id": sid}) + assert resp.status_code == 200 + body = resp.get_json() + assert body["replay"] is True + assert body["output"] == ["FROM DISK"] + finally: + if sid is not None: + mcp_close_pty_session(sid) diff --git a/tests/test_replay_only_flag.py b/tests/test_replay_only_flag.py new file mode 100644 index 0000000..8ee1e46 --- /dev/null +++ b/tests/test_replay_only_flag.py @@ -0,0 +1,197 @@ +"""Tests for the replay_only flag on PTY sessions.""" +import inspect +import pytest + +# Reuse the PTY-availability guard pattern from the suite. +import os +try: + import pty as _pty + _master, _slave = _pty.openpty() + os.close(_master) + os.close(_slave) + _PTY_AVAILABLE = True +except Exception: + _PTY_AVAILABLE = False + +_pty_skip = pytest.mark.skipif( + not _PTY_AVAILABLE, + reason="PTY not allocatable in this environment", +) + + +@_pty_skip +def test_mcp_create_pty_session_stores_replay_only_flag(): + """Creating a PTY with replay_only=True stores the flag in the session dict.""" + from app import mcp_create_pty_session, mcp_close_pty_session, sessions + + sid = mcp_create_pty_session(label="t1", replay_only=True) + try: + assert sessions[sid].get("replay_only") is True + finally: + mcp_close_pty_session(sid) + + +@_pty_skip +def test_mcp_create_pty_session_defaults_replay_only_false(): + """Default for replay_only is False (backward compat).""" + from app import mcp_create_pty_session, mcp_close_pty_session, sessions + + sid = mcp_create_pty_session(label="t2") + try: + assert sessions[sid].get("replay_only") is False + finally: + mcp_close_pty_session(sid) + + +@_pty_skip +def test_attach_session_replay_only_alive_pty_returns_replay(tmp_path, monkeypatch): + """A replay_only=True PTY that is still alive serves the transcript, not the live buffer.""" + from app import app as flask_app, mcp_create_pty_session, mcp_close_pty_session + from coda_mcp import task_manager + + # Point task_manager at a tmp sessions root so find_task_dir_by_pty_session resolves. + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + + # Create a fake task dir keyed by the PTY id we'll mint shortly. + sid = mcp_create_pty_session(label="t-replay-alive", replay_only=True) + try: + # Plant a session.json that links task → this pty_session_id, plus a transcript. + sess_id = "sess-fake" + task_id = "task-fake" + sdir = tmp_path / sess_id + tdir = sdir / "tasks" / task_id + tdir.mkdir(parents=True) + (sdir / "session.json").write_text( + '{"session_id": "%s", "pty_session_id": "%s", "current_task": "%s"}' + % (sess_id, sid, task_id) + ) + (tdir / "transcript.log").write_bytes(b"HELLO TRANSCRIPT") + + # Bust the lookup cache so find_task_dir_by_pty_session sees the new files. + task_manager._pty_lookup_cache.clear() + + client = flask_app.test_client() + resp = client.post("/api/session/attach", json={"session_id": sid}) + + assert resp.status_code == 200 + body = resp.get_json() + assert body["replay"] is True + assert body["output"] == ["HELLO TRANSCRIPT"] + finally: + mcp_close_pty_session(sid) + + +@_pty_skip +def test_attach_session_replay_only_false_alive_pty_returns_live_buffer(): + """A replay_only=False PTY that is still alive returns the live output_buffer (unchanged behavior).""" + from app import app as flask_app, mcp_create_pty_session, mcp_close_pty_session + + sid = mcp_create_pty_session(label="t-live", replay_only=False) + try: + client = flask_app.test_client() + resp = client.post("/api/session/attach", json={"session_id": sid}) + + assert resp.status_code == 200 + body = resp.get_json() + assert body.get("replay") in (False, None) # live path doesn't set replay key + assert "output" in body + finally: + mcp_close_pty_session(sid) + + +@_pty_skip +def test_coda_run_creates_pty_with_replay_only_true(tmp_path, monkeypatch): + """coda_run must create its PTY with replay_only=True.""" + import asyncio + import json + from app import sessions, mcp_close_pty_session + from coda_mcp import mcp_server, task_manager + + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + # Stop the watcher from racing the test — we only care about creation here. + monkeypatch.setattr(mcp_server, "_watch_task", lambda *a, **kw: None) + + result_str = asyncio.run(mcp_server.coda_run(prompt="ignored", email="t@example.com")) + result = json.loads(result_str) + session = task_manager._read_session(result["session_id"]) + pty_id = session.get("pty_session_id") + try: + assert pty_id is not None + assert sessions[pty_id].get("replay_only") is True + finally: + if pty_id is not None: + mcp_close_pty_session(pty_id) + + +def test_mcp_create_pty_session_signature_has_no_grace_param(): + """Regression guard: mcp_create_pty_session must not accept a 'grace' kwarg. + + Pure signature introspection — no PTY needed, runs unconditionally so + that no-PTY environments (CI, sandboxed runners) still catch regressions. + """ + from app import mcp_create_pty_session + + sig = inspect.signature(mcp_create_pty_session) + assert "grace" not in sig.parameters, ( + f"mcp_create_pty_session should not accept a 'grace' parameter " + f"(found in signature: {list(sig.parameters)})" + ) + + +@_pty_skip +def test_no_grace_key_in_session_dict(): + """Regression guard: session dicts from mcp_create_pty_session must not + contain a 'grace' key. + + Protects against accidental re-introduction of grace-period machinery + in future changes. PTY-gated because it actually allocates one. + """ + from app import mcp_create_pty_session, mcp_close_pty_session, sessions + + sid = mcp_create_pty_session(label="t-no-grace", replay_only=True) + try: + assert "grace" not in sessions[sid], ( + f"session dict should not contain a 'grace' key " + f"(found: {list(sessions[sid].keys())})" + ) + finally: + mcp_close_pty_session(sid) + + +@_pty_skip +def test_coda_run_does_not_create_project_dir(tmp_path, monkeypatch): + """Regression guard: coda_run is Mode 3 (replay-only, no project dir). + Only coda_interactive (Mode 2) creates dirs under ~/.coda/projects/. + + If a future change makes coda_run pull workspace files or otherwise + creates a per-session project dir under ~/.coda/projects/, this test fires. + """ + import asyncio + import json + import os + from app import mcp_close_pty_session + from coda_mcp import mcp_server, task_manager + + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path / "sessions")) + # Stop the watcher from racing the test. + monkeypatch.setattr(mcp_server, "_watch_task", lambda *a, **kw: None) + + result_str = asyncio.run(mcp_server.coda_run( + prompt="ignored", email="t@example.com", + )) + result = json.loads(result_str) + pty_id = None + try: + sess = task_manager._read_session(result["session_id"]) + pty_id = sess.get("pty_session_id") + + # Project dir must NOT exist for coda_run. + projects_root = os.path.join(str(tmp_path), ".coda", "projects") + assert not os.path.isdir(projects_root) or not os.listdir(projects_root), ( + f"coda_run unexpectedly created project dirs under {projects_root}: " + f"{os.listdir(projects_root) if os.path.isdir(projects_root) else 'n/a'}" + ) + finally: + if pty_id is not None: + mcp_close_pty_session(pty_id) diff --git a/tests/test_run_step.py b/tests/test_run_step.py new file mode 100644 index 0000000..af09733 --- /dev/null +++ b/tests/test_run_step.py @@ -0,0 +1,170 @@ +"""Tests for _run_step and _configure_all_cli_auth — env setup for subprocesses.""" + +import os +import subprocess +from unittest import mock + +import pytest + + +# We need to test _run_step from app.py. It calls subprocess.run, so we mock that. +# The function also updates setup_state, so we mock that too. + + +@pytest.fixture +def patch_app_globals(): + """Patch app.py globals needed by _run_step.""" + with mock.patch("app._update_step"): + yield + + +class TestRunStepEnvStripping: + """Verify _run_step strips OAuth credentials from subprocess env.""" + + def test_strips_databricks_client_id(self, patch_app_globals): + from app import _run_step + with mock.patch.dict(os.environ, { + "DATABRICKS_CLIENT_ID": "sp-client-id", + "DATABRICKS_CLIENT_SECRET": "sp-client-secret", + "HOME": "/tmp/test-home", + }), mock.patch("subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedResult = mock.MagicMock( + returncode=0, stdout="ok", stderr="" + ) + _run_step("test-step", "echo hello") + + call_env = mock_run.call_args.kwargs.get("env", {}) + assert "DATABRICKS_CLIENT_ID" not in call_env + assert "DATABRICKS_CLIENT_SECRET" not in call_env + + def test_preserves_other_env_vars(self, patch_app_globals): + from app import _run_step + with mock.patch.dict(os.environ, { + "HOME": "/tmp/test-home", + "MY_CUSTOM_VAR": "keep-this", + "DATABRICKS_CLIENT_ID": "remove-this", + }), mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stdout="ok", stderr="") + _run_step("test-step", "echo hello") + + call_env = mock_run.call_args.kwargs.get("env", {}) + assert call_env.get("MY_CUSTOM_VAR") == "keep-this" + + +class TestRunStepPythonpath: + """Verify _run_step injects PYTHONPATH for setup script imports.""" + + def test_sets_pythonpath_to_app_dir(self, patch_app_globals): + from app import _run_step + with mock.patch.dict(os.environ, {"HOME": "/tmp/test-home"}), \ + mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stdout="ok", stderr="") + _run_step("test-step", "echo hello") + + call_env = mock_run.call_args.kwargs.get("env", {}) + # PYTHONPATH should contain the app directory (dirname of app.py) + assert "PYTHONPATH" in call_env + assert call_env["PYTHONPATH"] # non-empty + + def test_prepends_to_existing_pythonpath(self, patch_app_globals): + from app import _run_step + with mock.patch.dict(os.environ, { + "HOME": "/tmp/test-home", + "PYTHONPATH": "/existing/path", + }), mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stdout="ok", stderr="") + _run_step("test-step", "echo hello") + + call_env = mock_run.call_args.kwargs.get("env", {}) + assert "/existing/path" in call_env["PYTHONPATH"] + + +class TestRunStepPath: + """Verify _run_step adds ~/.local/bin to PATH.""" + + def test_adds_local_bin_to_path(self, patch_app_globals): + from app import _run_step + with mock.patch.dict(os.environ, { + "HOME": "/tmp/test-home", + "PATH": "/usr/bin", + }), mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stdout="ok", stderr="") + _run_step("test-step", "echo hello") + + call_env = mock_run.call_args.kwargs.get("env", {}) + assert "/tmp/test-home/.local/bin" in call_env["PATH"] + + def test_skips_if_already_in_path(self, patch_app_globals): + from app import _run_step + with mock.patch.dict(os.environ, { + "HOME": "/tmp/test-home", + "PATH": "/tmp/test-home/.local/bin:/usr/bin", + }), mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stdout="ok", stderr="") + _run_step("test-step", "echo hello") + + call_env = mock_run.call_args.kwargs.get("env", {}) + # Should not duplicate + assert call_env["PATH"].count(".local/bin") == 1 + + def test_defaults_home_when_empty(self, patch_app_globals): + """When HOME is empty or '/', should default to /app/python/source_code.""" + from app import _run_step + with mock.patch.dict(os.environ, {"HOME": ""}, clear=False), \ + mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stdout="ok", stderr="") + _run_step("test-step", "echo hello") + + call_env = mock_run.call_args.kwargs.get("env", {}) + assert "/app/python/source_code" in call_env.get("HOME", "") + + +# --------------------------------------------------------------------------- +# _configure_all_cli_auth — PAT reconfiguration path +# --------------------------------------------------------------------------- + +class TestConfigureAllCliAuth: + """Verify _configure_all_cli_auth injects PYTHONPATH for setup script imports. + + This is a separate code path from _run_step — it runs setup scripts via + subprocess.run after PAT rotation. Without PYTHONPATH, the scripts can't + `from utils import ...` since they live in setup/ subdirectory. + """ + + def _call_configure(self, mock_run, tmp_path, token="dapi_test"): + """Helper to call _configure_all_cli_auth with all dependencies mocked.""" + from app import _configure_all_cli_auth + # Create .claude dir so settings.json write succeeds + (tmp_path / ".claude").mkdir(exist_ok=True) + with mock.patch("utils.resolve_and_cache_gateway"), \ + mock.patch("app.get_gateway_host", return_value=None), \ + mock.patch("app.ensure_https", return_value="https://test.databricks.com"), \ + mock.patch("app.pat_rotator"), \ + mock.patch.dict(os.environ, {"HOME": str(tmp_path)}): + _configure_all_cli_auth(token) + + def test_injects_pythonpath(self, tmp_path): + with mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stdout="", stderr="") + self._call_configure(mock_run, tmp_path) + + # Find a subprocess call that runs a setup script + setup_calls = [c for c in mock_run.call_args_list + if any("setup/" in str(a) for a in c[0][0])] + assert len(setup_calls) > 0, "Expected subprocess calls for setup scripts" + + for call in setup_calls: + call_env = call.kwargs.get("env") or call[1].get("env", {}) + assert "PYTHONPATH" in call_env, f"PYTHONPATH missing from env for {call[0][0]}" + assert call_env["PYTHONPATH"], "PYTHONPATH should not be empty" + + def test_passes_token_in_env(self, tmp_path): + with mock.patch("subprocess.run") as mock_run: + mock_run.return_value = mock.MagicMock(returncode=0, stdout="", stderr="") + self._call_configure(mock_run, tmp_path, token="dapi_mytoken") + + setup_calls = [c for c in mock_run.call_args_list + if any("setup/" in str(a) for a in c[0][0])] + for call in setup_calls: + call_env = call.kwargs.get("env") or call[1].get("env", {}) + assert call_env.get("DATABRICKS_TOKEN") == "dapi_mytoken" diff --git a/tests/test_session_detach.py b/tests/test_session_detach.py index c381a40..6e3b60f 100644 --- a/tests/test_session_detach.py +++ b/tests/test_session_detach.py @@ -7,7 +7,6 @@ import os import subprocess -import sys import threading import time from collections import deque @@ -40,42 +39,23 @@ def test_detects_child_process_name(self): """When a shell has a child process, return the child's name.""" app_mod = _get_app() - # Launch a shell (bash) with a child process (sleep) - shell = subprocess.Popen( - ["bash", "-c", "sleep 300"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - # Give the child time to spawn - time.sleep(0.5) - - try: - result = app_mod._get_session_process(shell.pid) - assert result == "sleep", f"Expected 'sleep', got '{result}'" - finally: - shell.kill() - shell.wait() + # Mock pgrep returning a child PID, then ps resolving it to "sleep" + pgrep_result = mock.Mock(returncode=0, stdout="12345\n") + ps_result = mock.Mock(returncode=0, stdout="sleep\n") + with mock.patch("subprocess.run", side_effect=[pgrep_result, ps_result]): + result = app_mod._get_session_process(100) + assert result == "sleep", f"Expected 'sleep', got '{result}'" def test_returns_parent_process_name_when_no_children(self): """When a shell has no foreground children, return the shell name.""" app_mod = _get_app() - # Launch a bare shell that just sleeps via bash built-in wait - # Use cat which will block on stdin with no children of its own - proc = subprocess.Popen( - ["cat"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - try: - result = app_mod._get_session_process(proc.pid) - assert result == "cat", f"Expected 'cat', got '{result}'" - finally: - proc.kill() - proc.wait() + # Mock pgrep finding no children (exit 1), then ps resolving the process itself + pgrep_result = mock.Mock(returncode=1, stdout="") + ps_result = mock.Mock(returncode=0, stdout="cat\n") + with mock.patch("subprocess.run", side_effect=[pgrep_result, ps_result]): + result = app_mod._get_session_process(100) + assert result == "cat", f"Expected 'cat', got '{result}'" def test_returns_unknown_for_dead_pid(self): """Return 'unknown' when the PID does not exist.""" @@ -230,28 +210,31 @@ def setup_app(self): app_module.sessions.clear() def test_exited_session_removed_from_dict(self): - import pty - master_fd, slave_fd = pty.openpty() + fake_master = 50 + # Use a completed process so waitpid works proc = subprocess.Popen( - ["bash", "-c", "echo hello && exit 0"], - stdin=slave_fd, stdout=slave_fd, stderr=slave_fd, - preexec_fn=os.setsid + ["bash", "-c", "exit 0"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - os.close(slave_fd) + proc.wait() session_id = "sess-eof-test" with self.app_module.sessions_lock: self.app_module.sessions[session_id] = { "pid": proc.pid, - "master_fd": master_fd, + "master_fd": fake_master, "output_buffer": deque(maxlen=1000), "lock": threading.Lock(), "last_poll_time": time.time(), "created_at": time.time(), } - # read_pty_output should detect EOF and call terminate_session - self.app_module.read_pty_output(session_id, master_fd) + # Simulate EOF: select says readable, os.read returns empty bytes + with mock.patch("select.select", return_value=([fake_master], [], [])), \ + mock.patch("os.read", return_value=b""), \ + mock.patch("os.close"), \ + mock.patch("os.kill"): + self.app_module.read_pty_output(session_id, fake_master) with self.app_module.sessions_lock: assert session_id not in self.app_module.sessions diff --git a/tests/test_setup_proxy.py b/tests/test_setup_proxy.py new file mode 100644 index 0000000..3e4658b --- /dev/null +++ b/tests/test_setup_proxy.py @@ -0,0 +1,45 @@ +"""Regression tests for setup/setup_proxy.py — the content-filter proxy launcher. + +The launcher spawns ``content_filter_proxy.py`` as a subprocess. That file lives +at the REPO ROOT, not in setup/. A 2026 refactor moved setup_proxy.py into +setup/ (git fec2152, R100 rename) without updating its relative path lookup, so +the launcher pointed at a nonexistent ``setup/content_filter_proxy.py`` and the +proxy never started — silently breaking OpenCode (the only agent that routes +through the proxy at 127.0.0.1:4000). These tests pin the resolved path to an +existing file so a future move can't regress it again. +""" + +import importlib.util +import os + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SETUP_PROXY_PATH = os.path.join(REPO_ROOT, "setup", "setup_proxy.py") + + +def _load_setup_proxy(): + """Import setup_proxy.py by path WITHOUT running its main() side effects.""" + spec = importlib.util.spec_from_file_location("setup_proxy_under_test", SETUP_PROXY_PATH) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def test_resolved_proxy_script_exists(): + """The script the launcher hands to Popen must actually exist on disk.""" + mod = _load_setup_proxy() + path = mod.resolve_proxy_script_path() + assert os.path.isfile(path), ( + f"setup_proxy.py resolves the proxy server to a non-existent path: {path}. " + f"content_filter_proxy.py lives at the repo root, not in setup/." + ) + + +def test_resolved_proxy_script_is_repo_root_content_filter_proxy(): + """It must be the repo-root content_filter_proxy.py, not a setup/-relative path.""" + mod = _load_setup_proxy() + path = mod.resolve_proxy_script_path() + assert os.path.basename(path) == "content_filter_proxy.py" + assert os.path.dirname(os.path.abspath(path)) == REPO_ROOT, ( + f"expected the repo-root copy ({REPO_ROOT}), got dir " + f"{os.path.dirname(os.path.abspath(path))}" + ) diff --git a/tests/test_setup_resource_paths.py b/tests/test_setup_resource_paths.py new file mode 100644 index 0000000..bb4727b --- /dev/null +++ b/tests/test_setup_resource_paths.py @@ -0,0 +1,48 @@ +"""Regression tests for bundled-resource path resolution in the setup scripts. + +Commit fec2152 moved every setup script into setup/ (R100 renames) but left the +resources they copy (agents/, .codex/, content_filter_proxy.py) at the repo root. +Scripts that located those resources via ``Path(__file__).parent`` silently broke: +the proxy never launched (OpenCode), Claude subagents weren't installed, and the +Codex model catalog wasn't copied. These tests pin each resolver to an existing +resource so a future move can't silently regress it again. + +The setup scripts run heavy side effects at import (npm installs, curl), so we +extract and execute ONLY the resolver function from the source via AST — this +tests the real resolver code without triggering the script body. +""" + +import ast +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +SETUP_DIR = REPO_ROOT / "setup" + + +def _extract_resolver(script_path: Path, func_name: str): + """Compile and exec just ``func_name`` from ``script_path`` (no body run).""" + tree = ast.parse(script_path.read_text()) + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == func_name: + ns = {"Path": Path, "__file__": str(script_path)} + exec(compile(ast.Module(body=[node], type_ignores=[]), str(script_path), "exec"), ns) + return ns[func_name] + raise AssertionError(f"{func_name}() not found in {script_path}") + + +def test_claude_agents_resolver_points_at_existing_dir(): + """setup_claude.py must resolve agents/ to a real dir with the subagents.""" + resolve = _extract_resolver(SETUP_DIR / "setup_claude.py", "resolve_agents_src") + agents = resolve() + assert agents.is_dir(), f"setup_claude resolves agents/ to a missing dir: {agents}" + names = {p.name for p in agents.glob("*.md")} + expected = {"build-feature.md", "implementer.md", "prd-writer.md", "test-generator.md"} + assert expected <= names, f"missing bundled subagents: {expected - names}" + + +def test_codex_catalog_resolver_points_at_existing_file(): + """setup_codex.py must resolve the model catalog to a real file.""" + resolve = _extract_resolver(SETUP_DIR / "setup_codex.py", "resolve_codex_catalog_src") + catalog = resolve() + assert catalog.is_file(), f"setup_codex resolves the model catalog to a missing file: {catalog}" + assert catalog.name == "databricks-models.json" diff --git a/tests/test_sync_to_workspace.py b/tests/test_sync_to_workspace.py new file mode 100644 index 0000000..6faedf4 --- /dev/null +++ b/tests/test_sync_to_workspace.py @@ -0,0 +1,181 @@ +"""Tests for sync_to_workspace — path-escape guard and workspace sync.""" + +import subprocess +from pathlib import Path +from unittest import mock + +import pytest + + +# --------------------------------------------------------------------------- +# _read_databrickscfg +# --------------------------------------------------------------------------- + +class TestReadDatabrickscfg: + def test_reads_host_and_token(self, tmp_path): + cfg = tmp_path / ".databrickscfg" + cfg.write_text("[DEFAULT]\nhost = https://test.cloud.databricks.com\ntoken = dapi_abc123\n") + with mock.patch("sync_to_workspace.Path.home", return_value=tmp_path): + from sync_to_workspace import _read_databrickscfg + host, token = _read_databrickscfg() + assert host == "https://test.cloud.databricks.com" + assert token == "dapi_abc123" + + def test_returns_none_when_missing(self, tmp_path): + with mock.patch("sync_to_workspace.Path.home", return_value=tmp_path): + from sync_to_workspace import _read_databrickscfg + host, token = _read_databrickscfg() + assert host is None + assert token is None + + def test_returns_none_for_missing_keys(self, tmp_path): + cfg = tmp_path / ".databrickscfg" + cfg.write_text("[DEFAULT]\n# empty section\n") + with mock.patch("sync_to_workspace.Path.home", return_value=tmp_path): + from sync_to_workspace import _read_databrickscfg + host, token = _read_databrickscfg() + assert host is None + assert token is None + + +# --------------------------------------------------------------------------- +# get_user_email +# --------------------------------------------------------------------------- + +class TestGetUserEmail: + def test_raises_when_no_config(self, tmp_path): + from sync_to_workspace import get_user_email + with mock.patch("sync_to_workspace._read_databrickscfg", return_value=(None, None)): + with pytest.raises(RuntimeError, match="missing host or token"): + get_user_email() + + def test_raises_when_no_token(self): + from sync_to_workspace import get_user_email + with mock.patch("sync_to_workspace._read_databrickscfg", return_value=("https://host", None)): + with pytest.raises(RuntimeError, match="missing host or token"): + get_user_email() + + def test_returns_email(self): + from sync_to_workspace import get_user_email + mock_user = mock.MagicMock() + mock_user.user_name = "test@example.com" + mock_client = mock.MagicMock() + mock_client.current_user.me.return_value = mock_user + with mock.patch("sync_to_workspace._read_databrickscfg", return_value=("https://host", "tok")): + with mock.patch("sync_to_workspace.WorkspaceClient", return_value=mock_client): + email = get_user_email() + assert email == "test@example.com" + + +# --------------------------------------------------------------------------- +# sync_project — path-escape guard +# --------------------------------------------------------------------------- + +class TestSyncProject: + def test_rejects_path_outside_projects_dir(self, tmp_path, capsys): + from sync_to_workspace import sync_project + # Create a path outside ~/projects/ + outside = tmp_path / "evil-repo" + outside.mkdir() + with mock.patch("sync_to_workspace.Path.home", return_value=tmp_path): + sync_project(outside) + captured = capsys.readouterr() + assert "SKIP" in captured.err + assert "outside" in captured.err + + def test_accepts_path_inside_projects_dir(self, tmp_path): + from sync_to_workspace import sync_project + projects = tmp_path / "projects" + projects.mkdir() + repo = projects / "my-repo" + repo.mkdir() + + mock_user = mock.MagicMock() + mock_user.user_name = "test@example.com" + mock_client = mock.MagicMock() + mock_client.current_user.me.return_value = mock_user + + with mock.patch("sync_to_workspace.Path.home", return_value=tmp_path), \ + mock.patch("sync_to_workspace._read_databrickscfg", return_value=("https://host", "tok")), \ + mock.patch("sync_to_workspace.WorkspaceClient", return_value=mock_client), \ + mock.patch("sync_to_workspace.subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess([], 0, stdout="", stderr="") + sync_project(repo) + + mock_run.assert_called_once() + args = mock_run.call_args + assert "databricks" in args[0][0][0] + assert "sync" in args[0][0][1] + + def test_strips_oauth_env_from_subprocess(self, tmp_path): + """Verify OAuth credentials are stripped so CLI falls through to ~/.databrickscfg.""" + from sync_to_workspace import sync_project + projects = tmp_path / "projects" + projects.mkdir() + repo = projects / "my-repo" + repo.mkdir() + + mock_user = mock.MagicMock() + mock_user.user_name = "test@example.com" + mock_client = mock.MagicMock() + mock_client.current_user.me.return_value = mock_user + + with mock.patch("sync_to_workspace.Path.home", return_value=tmp_path), \ + mock.patch("sync_to_workspace._read_databrickscfg", return_value=("https://host", "tok")), \ + mock.patch("sync_to_workspace.WorkspaceClient", return_value=mock_client), \ + mock.patch("sync_to_workspace.subprocess.run") as mock_run, \ + mock.patch.dict("os.environ", { + "DATABRICKS_CLIENT_ID": "sp-id", + "DATABRICKS_CLIENT_SECRET": "sp-secret", + "DATABRICKS_HOST": "https://host", + "DATABRICKS_TOKEN": "dapi_tok", + }): + mock_run.return_value = subprocess.CompletedProcess([], 0, stdout="", stderr="") + sync_project(repo) + + call_env = mock_run.call_args[1].get("env") or mock_run.call_args.kwargs.get("env", {}) + assert "DATABRICKS_CLIENT_ID" not in call_env + assert "DATABRICKS_CLIENT_SECRET" not in call_env + assert "DATABRICKS_HOST" not in call_env + assert "DATABRICKS_TOKEN" not in call_env + + def test_logs_error_on_failure(self, tmp_path, capsys): + from sync_to_workspace import sync_project + projects = tmp_path / "projects" + projects.mkdir() + repo = projects / "my-repo" + repo.mkdir() + + with mock.patch("sync_to_workspace.Path.home", return_value=tmp_path), \ + mock.patch("sync_to_workspace.get_user_email", side_effect=Exception("auth failed")): + sync_project(repo) + + captured = capsys.readouterr() + assert "Sync failed" in captured.err + # Error should be logged to file + error_log = tmp_path / ".sync-errors.log" + assert error_log.exists() + assert "auth failed" in error_log.read_text() + + def test_sync_failure_warns(self, tmp_path, capsys): + """Non-zero return code from databricks sync should print warning.""" + from sync_to_workspace import sync_project + projects = tmp_path / "projects" + projects.mkdir() + repo = projects / "my-repo" + repo.mkdir() + + mock_user = mock.MagicMock() + mock_user.user_name = "test@example.com" + mock_client = mock.MagicMock() + mock_client.current_user.me.return_value = mock_user + + with mock.patch("sync_to_workspace.Path.home", return_value=tmp_path), \ + mock.patch("sync_to_workspace._read_databrickscfg", return_value=("https://host", "tok")), \ + mock.patch("sync_to_workspace.WorkspaceClient", return_value=mock_client), \ + mock.patch("sync_to_workspace.subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess([], 1, stdout="", stderr="permission denied") + sync_project(repo) + + captured = capsys.readouterr() + assert "Sync warning" in captured.err diff --git a/tests/test_task_manager.py b/tests/test_task_manager.py new file mode 100644 index 0000000..fbbc032 --- /dev/null +++ b/tests/test_task_manager.py @@ -0,0 +1,688 @@ +"""Tests for task_manager — disk-based MCP session/task state.""" + +import json +import os +import time +from unittest import mock + +import pytest + + +@pytest.fixture(autouse=True) +def isolated_sessions(tmp_path): + """Point task_manager.SESSIONS_DIR at a temp dir.""" + sessions_dir = str(tmp_path / ".coda" / "sessions") + with mock.patch("coda_mcp.task_manager.SESSIONS_DIR", sessions_dir): + yield sessions_dir + + +# ── helpers ────────────────────────────────────────────────────────── + + +def _read_json(path): + with open(path) as f: + return json.load(f) + + +def _read_text(path): + with open(path) as f: + return f.read() + + +def _read_jsonl(path): + lines = [] + with open(path) as f: + for line in f: + line = line.strip() + if line: + lines.append(json.loads(line)) + return lines + + +# ── Session lifecycle ──────────────────────────────────────────────── + + +class TestCreateSession: + def test_returns_session_id_and_status(self): + from coda_mcp import task_manager + + result = task_manager.create_session("a@b.com", "u1", "my-label") + assert result["status"] == "ready" + assert result["session_id"].startswith("sess-") + assert len(result["session_id"]) == 5 + 12 # "sess-" + 12 hex + + def test_creates_session_json_on_disk(self, isolated_sessions): + from coda_mcp import task_manager + + result = task_manager.create_session("a@b.com", "u1", "my-label") + sid = result["session_id"] + path = os.path.join(isolated_sessions, sid, "session.json") + assert os.path.isfile(path) + data = _read_json(path) + assert data["email"] == "a@b.com" + assert data["user_id"] == "u1" + assert data["label"] == "my-label" + assert data["status"] == "ready" + assert data["current_task"] is None + assert data["completed_tasks"] == [] + assert "created_at" in data + + def test_unique_ids(self): + from coda_mcp import task_manager + + ids = {task_manager.create_session("a@b.com", "u1")["session_id"] for _ in range(20)} + assert len(ids) == 20 + + +class TestCloseSession: + def test_marks_session_closed(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + task_manager.close_session(sid) + data = _read_json(os.path.join(isolated_sessions, sid, "session.json")) + assert data["status"] == "closed" + + def test_close_nonexistent_raises(self): + from coda_mcp import task_manager + + with pytest.raises(task_manager.SessionNotFoundError): + task_manager.close_session("sess-doesnotexist") + + +class TestReadSession: + def test_read_existing(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1", "lbl")["session_id"] + data = task_manager._read_session(sid) + assert data["email"] == "a@b.com" + + def test_read_nonexistent_raises(self): + from coda_mcp import task_manager + + with pytest.raises(task_manager.SessionNotFoundError): + task_manager._read_session("sess-000000000000") + + +class TestUpdateSessionField: + def test_updates_single_field(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + task_manager._update_session_field(sid, "status", "busy") + data = task_manager._read_session(sid) + assert data["status"] == "busy" + + def test_preserves_other_fields(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1", "lbl")["session_id"] + task_manager._update_session_field(sid, "status", "busy") + data = task_manager._read_session(sid) + assert data["email"] == "a@b.com" + assert data["label"] == "lbl" + + +# ── Task lifecycle ─────────────────────────────────────────────────── + + +class TestCreateTask: + def test_returns_task_id_and_running(self): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + result = task_manager.create_task(sid, "do something", "a@b.com") + assert result["status"] == "running" + assert result["task_id"].startswith("task-") + assert len(result["task_id"]) == 5 + 8 # "task-" + 8 hex + + def test_creates_task_directory_with_files(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid = task_manager.create_task(sid, "do something", "a@b.com")["task_id"] + task_dir = task_manager._task_dir(sid, tid) + assert os.path.isdir(task_dir) + assert os.path.isfile(os.path.join(task_dir, "prompt.txt")) + assert os.path.isfile(os.path.join(task_dir, "status.jsonl")) + + def test_prompt_txt_contains_wrapped_prompt(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid = task_manager.create_task(sid, "fix the bug", "a@b.com")["task_id"] + prompt = _read_text(os.path.join(task_manager._task_dir(sid, tid), "prompt.txt")) + assert "---CODA-TASK---" in prompt + assert "fix the bug" in prompt + + def test_session_marked_busy(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + task_manager.create_task(sid, "do it", "a@b.com") + data = task_manager._read_session(sid) + assert data["status"] == "busy" + + def test_session_current_task_set(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid = task_manager.create_task(sid, "do it", "a@b.com")["task_id"] + data = task_manager._read_session(sid) + assert data["current_task"] == tid + + def test_busy_session_raises(self): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + task_manager.create_task(sid, "first", "a@b.com") + with pytest.raises(task_manager.SessionBusyError): + task_manager.create_task(sid, "second", "a@b.com") + + def test_nonexistent_session_raises(self): + from coda_mcp import task_manager + + with pytest.raises(task_manager.SessionNotFoundError): + task_manager.create_task("sess-doesnotexist", "p", "e@x.com") + + def test_status_jsonl_has_initial_entry(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid = task_manager.create_task(sid, "go", "a@b.com")["task_id"] + entries = _read_jsonl( + os.path.join(task_manager._task_dir(sid, tid), "status.jsonl") + ) + assert len(entries) == 1 + assert entries[0]["status"] == "running" + + def test_optional_params_stored(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid = task_manager.create_task( + sid, "go", "a@b.com", + context={"repo": "myrepo"}, + context_hint="look at utils.py", + timeout_s=120, + permissions=["read", "write"], + )["task_id"] + prompt = _read_text(os.path.join(task_manager._task_dir(sid, tid), "prompt.txt")) + assert "myrepo" in prompt + assert "utils.py" in prompt + + +class TestTaskDir: + def test_returns_correct_path(self, isolated_sessions): + from coda_mcp import task_manager + + path = task_manager._task_dir("sess-aabbccddee01", "task-11223344") + expected = os.path.join( + isolated_sessions, "sess-aabbccddee01", "tasks", "task-11223344" + ) + assert path == expected + + +# ── Task status / result ───────────────────────────────────────────── + + +class TestGetTaskStatus: + def test_returns_latest_status(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid = task_manager.create_task(sid, "go", "a@b.com")["task_id"] + status = task_manager.get_task_status(tid, sid) + assert status["status"] == "running" + + def test_reads_appended_lines(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid = task_manager.create_task(sid, "go", "a@b.com")["task_id"] + # simulate agent appending progress + status_path = os.path.join(task_manager._task_dir(sid, tid), "status.jsonl") + with open(status_path, "a") as f: + f.write(json.dumps({"status": "progress", "pct": 50, "ts": time.time()}) + "\n") + status = task_manager.get_task_status(tid, sid) + assert status["status"] == "progress" + assert status["pct"] == 50 + + def test_missing_task_returns_not_found(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + status = task_manager.get_task_status("task-nonexist", sid) + assert status["status"] == "not_found" + + +class TestGetTaskResult: + def test_returns_result_when_present(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid = task_manager.create_task(sid, "go", "a@b.com")["task_id"] + # simulate agent writing result + result_path = os.path.join(task_manager._task_dir(sid, tid), "result.json") + with open(result_path, "w") as f: + json.dump({"answer": 42}, f) + result = task_manager.get_task_result(tid, sid) + assert result["answer"] == 42 + + def test_returns_none_when_absent(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid = task_manager.create_task(sid, "go", "a@b.com")["task_id"] + result = task_manager.get_task_result(tid, sid) + assert result is None + + def test_missing_task_returns_none(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + result = task_manager.get_task_result("task-nonexist", sid) + assert result is None + + +# ── Complete task ───────────────────────────────────────────────────── + + +class TestCompleteTask: + def test_marks_session_closed(self, isolated_sessions): + """v2: sessions are ephemeral — complete_task auto-closes the session.""" + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid = task_manager.create_task(sid, "go", "a@b.com")["task_id"] + task_manager.complete_task(sid, tid) + data = task_manager._read_session(sid) + assert data["status"] == "closed" + assert "closed_at" in data + + def test_appends_to_completed_tasks(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid = task_manager.create_task(sid, "go", "a@b.com")["task_id"] + task_manager.complete_task(sid, tid) + data = task_manager._read_session(sid) + assert tid in data["completed_tasks"] + + def test_closed_session_rejects_new_task(self, isolated_sessions): + """v2: ephemeral sessions — new tasks need new sessions.""" + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid1 = task_manager.create_task(sid, "first", "a@b.com")["task_id"] + task_manager.complete_task(sid, tid1) + with pytest.raises(task_manager.SessionNotFoundError): + task_manager.create_task(sid, "second", "a@b.com") + + def test_appends_done_to_status_jsonl(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid = task_manager.create_task(sid, "go", "a@b.com")["task_id"] + task_manager.complete_task(sid, tid) + entries = _read_jsonl( + os.path.join(task_manager._task_dir(sid, tid), "status.jsonl") + ) + assert entries[-1]["status"] == "done" + + def test_nonexistent_session_raises(self): + from coda_mcp import task_manager + + with pytest.raises(task_manager.SessionNotFoundError): + task_manager.complete_task("sess-doesnotexist", "task-00000000") + + +# ── Prompt wrapping ────────────────────────────────────────────────── + + +class TestWrapPrompt: + def test_contains_marker(self): + from coda_mcp import task_manager + + wrapped = task_manager.wrap_prompt( + task_id="task-aabbccdd", + session_id="sess-112233445566", + email="a@b.com", + prompt="fix the bug", + context=None, + results_dir="/tmp/r", + context_hint=None, + ) + assert "---CODA-TASK---" in wrapped + assert "fix the bug" in wrapped + assert "task-aabbccdd" in wrapped + assert "sess-112233445566" in wrapped + assert "a@b.com" in wrapped + assert "/tmp/r" in wrapped + + def test_includes_context_when_provided(self): + from coda_mcp import task_manager + + wrapped = task_manager.wrap_prompt( + task_id="task-aabbccdd", + session_id="sess-112233445566", + email="a@b.com", + prompt="go", + context={"repo": "myrepo", "branch": "main"}, + results_dir="/tmp/r", + context_hint=None, + ) + assert "myrepo" in wrapped + assert "main" in wrapped + + def test_includes_context_hint(self): + from coda_mcp import task_manager + + wrapped = task_manager.wrap_prompt( + task_id="task-aabbccdd", + session_id="sess-112233445566", + email="a@b.com", + prompt="go", + context=None, + results_dir="/tmp/r", + context_hint="look at utils.py first", + ) + assert "look at utils.py first" in wrapped + + def test_no_context_still_valid(self): + from coda_mcp import task_manager + + wrapped = task_manager.wrap_prompt( + task_id="task-aabbccdd", + session_id="sess-112233445566", + email="a@b.com", + prompt="hello", + context=None, + results_dir="/tmp/r", + context_hint=None, + ) + assert "---CODA-TASK---" in wrapped + assert "hello" in wrapped + + +# ── Edge cases ──────────────────────────────────────────────────────── + + +class TestEdgeCases: + def test_closed_session_rejects_task(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + task_manager.close_session(sid) + with pytest.raises(task_manager.SessionNotFoundError): + task_manager.create_task(sid, "go", "a@b.com") + + def test_multiple_tasks_across_sessions(self, isolated_sessions): + """v2: each task gets its own ephemeral session; all appear in list_all_tasks.""" + from coda_mcp import task_manager + + tids = [] + for i in range(3): + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + tid = task_manager.create_task(sid, f"task {i}", "a@b.com")["task_id"] + task_manager.complete_task(sid, tid) + tids.append(tid) + # Each session auto-closes + data = task_manager._read_session(sid) + assert data["status"] == "closed" + + all_tasks = task_manager.list_all_tasks() + all_tids = [t["task_id"] for t in all_tasks] + for tid in tids: + assert tid in all_tids + + def test_corrupt_session_json_raises(self, isolated_sessions): + from coda_mcp import task_manager + + sid = task_manager.create_session("a@b.com", "u1")["session_id"] + path = os.path.join(isolated_sessions, sid, "session.json") + with open(path, "w") as f: + f.write("{bad json") + with pytest.raises(task_manager.SessionNotFoundError): + task_manager._read_session(sid) + + +# ── find_task_dir_by_pty_session ───────────────────────────────────── + + +@pytest.fixture +def sessions_root(tmp_path, monkeypatch): + from coda_mcp import task_manager + monkeypatch.setattr(task_manager, "SESSIONS_DIR", str(tmp_path)) + # Reset the lookup cache between tests + task_manager._pty_lookup_cache.clear() + return tmp_path + + +def _make_session_dir(root, sess_id, pty_id, current_task=None, completed=None): + sdir = root / sess_id + (sdir / "tasks").mkdir(parents=True) + data = { + "session_id": sess_id, + "pty_session_id": pty_id, + "current_task": current_task, + "completed_tasks": completed or [], + "status": "ready", + } + (sdir / "session.json").write_text(json.dumps(data)) + return sdir + + +def test_find_task_dir_hits_current_task(sessions_root): + from coda_mcp import task_manager + + _make_session_dir(sessions_root, "sess-A", "pty-1", current_task="task-X") + result = task_manager.find_task_dir_by_pty_session("pty-1") + assert result == str(sessions_root / "sess-A" / "tasks" / "task-X") + + +def test_find_task_dir_falls_back_to_last_completed(sessions_root): + from coda_mcp import task_manager + + _make_session_dir( + sessions_root, "sess-A", "pty-1", + current_task=None, + completed=["task-old", "task-recent"], + ) + result = task_manager.find_task_dir_by_pty_session("pty-1") + assert result == str(sessions_root / "sess-A" / "tasks" / "task-recent") + + +def test_find_task_dir_returns_none_when_no_match(sessions_root): + from coda_mcp import task_manager + + _make_session_dir(sessions_root, "sess-A", "pty-1", current_task="task-X") + assert task_manager.find_task_dir_by_pty_session("pty-NONEXIST") is None + + +def test_find_task_dir_ignores_corrupt_session_json(sessions_root): + from coda_mcp import task_manager + + sdir = sessions_root / "sess-bad" + sdir.mkdir() + (sdir / "session.json").write_text("not json {{{") + _make_session_dir(sessions_root, "sess-good", "pty-1", current_task="task-X") + assert task_manager.find_task_dir_by_pty_session("pty-1") == \ + str(sessions_root / "sess-good" / "tasks" / "task-X") + + +def test_find_task_dir_cache_hits_within_ttl(sessions_root): + from coda_mcp import task_manager + + _make_session_dir(sessions_root, "sess-A", "pty-1", current_task="task-X") + task_manager.find_task_dir_by_pty_session("pty-1") + # Remove session.json — cache should still return the hit + (sessions_root / "sess-A" / "session.json").unlink() + assert task_manager.find_task_dir_by_pty_session("pty-1") == \ + str(sessions_root / "sess-A" / "tasks" / "task-X") + + +def test_find_task_dir_cache_expires(sessions_root, monkeypatch): + from coda_mcp import task_manager + + monkeypatch.setattr(task_manager, "_PTY_LOOKUP_TTL", 0.01) + _make_session_dir(sessions_root, "sess-A", "pty-1", current_task="task-X") + task_manager.find_task_dir_by_pty_session("pty-1") + (sessions_root / "sess-A" / "session.json").unlink() + time.sleep(0.02) + assert task_manager.find_task_dir_by_pty_session("pty-1") is None + + +def test_find_task_dir_no_sessions_dir(sessions_root, monkeypatch): + from coda_mcp import task_manager + + monkeypatch.setattr(task_manager, "SESSIONS_DIR", "/nonexistent/path/that/does/not/exist") + assert task_manager.find_task_dir_by_pty_session("pty-1") is None + + +# ── workflow_protocol flag wiring ──────────────────────────────────── + + +def test_wrap_prompt_default_includes_capabilities_and_workflow(): + """Default workflow_protocol=True; rendered prompt contains both new sections.""" + from coda_mcp.task_manager import wrap_prompt + + out = wrap_prompt( + task_id="t-1", + session_id="s-1", + email="user@example.com", + prompt="do the thing", + context=None, + results_dir="/tmp/results", + ) + assert "CAPABILITIES:" in out + assert "WORKFLOW PROTOCOL:" in out + # Sanity: still has the existing structure. + assert "TASK:" in out + assert "INSTRUCTIONS:" in out + assert "SAFETY:" in out + + +def test_wrap_prompt_workflow_protocol_false_omits_sections(): + """With workflow_protocol=False, both new sections are absent.""" + from coda_mcp.task_manager import wrap_prompt + + out = wrap_prompt( + task_id="t-1", + session_id="s-1", + email="user@example.com", + prompt="do the thing", + context=None, + results_dir="/tmp/results", + workflow_protocol=False, + ) + assert "CAPABILITIES:" not in out + assert "WORKFLOW PROTOCOL:" not in out + # Existing sections are still present. + assert "TASK:" in out + assert "INSTRUCTIONS:" in out + + +def test_wrap_prompt_workflow_protocol_default_is_true(): + """Signature inspection: default value of workflow_protocol is True.""" + import inspect + from coda_mcp.task_manager import wrap_prompt + + sig = inspect.signature(wrap_prompt) + assert "workflow_protocol" in sig.parameters + assert sig.parameters["workflow_protocol"].default is True + + +def test_create_task_signature_has_workflow_protocol_param(): + """create_task accepts workflow_protocol kwarg with default True.""" + import inspect + from coda_mcp.task_manager import create_task + + sig = inspect.signature(create_task) + assert "workflow_protocol" in sig.parameters + assert sig.parameters["workflow_protocol"].default is True + + +def test_create_task_forwards_workflow_protocol_to_wrap_prompt(monkeypatch, tmp_path): + """create_task must pass workflow_protocol through to wrap_prompt.""" + from coda_mcp import task_manager + + captured: dict = {} + + def fake_wrap_prompt(**kwargs): + captured.update(kwargs) + return "DUMMY PROMPT" + + monkeypatch.setattr(task_manager, "wrap_prompt", fake_wrap_prompt) + monkeypatch.setattr(task_manager, "_session_dir", lambda sid: str(tmp_path)) + monkeypatch.setattr(task_manager, "_task_dir", lambda sid, tid: str(tmp_path)) + # Stub _read_session so create_task sees a valid ready session without disk I/O. + monkeypatch.setattr( + task_manager, + "_read_session", + lambda sid: {"session_id": sid, "status": "ready", "current_task": None, "completed_tasks": []}, + ) + # _write_json is the real helper used inside create_task (writes meta.json + session file). + # Stub it out — we're testing flag pass-through, not filesystem behavior. + monkeypatch.setattr(task_manager, "_write_json", lambda *a, **kw: None) + monkeypatch.setattr(task_manager.os, "makedirs", lambda *a, **kw: None) + # Stub the file-open for prompt.txt and status.jsonl writes. + real_open = open + def fake_open(path, mode="r", *args, **kwargs): + if ("prompt.txt" in str(path) or "status.jsonl" in str(path)) and "w" in mode: + import io + return io.StringIO() + return real_open(path, mode, *args, **kwargs) + monkeypatch.setattr("builtins.open", fake_open) + + task_manager.create_task( + session_id="s-1", + prompt="x", + email="u@example.com", + workflow_protocol=False, + ) + assert captured.get("workflow_protocol") is False + + +def test_coda_run_signature_has_workflow_protocol_param(): + """coda_run accepts workflow_protocol kwarg with default True.""" + import inspect + from coda_mcp import mcp_server + + sig = inspect.signature(mcp_server.coda_run) + assert "workflow_protocol" in sig.parameters + assert sig.parameters["workflow_protocol"].default is True + + +def test_wrap_prompt_instructions_documents_info_needed(): + """INSTRUCTIONS section must mention the info_needed status and feedback field.""" + from coda_mcp.task_manager import wrap_prompt + + out = wrap_prompt( + task_id="t-1", + session_id="s-1", + email="user@example.com", + prompt="do the thing", + context=None, + results_dir="/tmp/r", + ) + # Pull the INSTRUCTIONS section out for focused assertions. + assert "info_needed" in out + assert "feedback" in out + + +def test_wrap_prompt_instructions_lists_new_step_labels(): + """INSTRUCTIONS section enumerates the canonical step labels emitted by the agent.""" + from coda_mcp.task_manager import wrap_prompt + + out = wrap_prompt( + task_id="t-1", + session_id="s-1", + email="user@example.com", + prompt="do the thing", + context=None, + results_dir="/tmp/r", + ) + for label in ("plan", "critique_plan", "execute", "critique_execute", "synthesize", "critique_synthesize"): + assert label in out, f"Missing step label {label!r} from prompt text" diff --git a/tests/test_terminate_session_idempotent.py b/tests/test_terminate_session_idempotent.py new file mode 100644 index 0000000..072adc8 --- /dev/null +++ b/tests/test_terminate_session_idempotent.py @@ -0,0 +1,59 @@ +"""Regression test: terminate_session must close master_fd exactly once. + +Both the explicit close path (mcp_close_pty_session) and the read-thread exit +path (read_pty_output, which calls terminate_session when its loop ends) fire +for the same session. If terminate_session closes master_fd on BOTH calls, the +second os.close() can land on a since-reused fd — e.g. an asyncio event loop's +self-pipe allocated by a later test — corrupting unrelated I/O. That is the +source of the intermittent 'OSError: [Errno 9] Bad file descriptor' (EBADF) +flakiness seen when PTY tests and asyncio tests run together. + +terminate_session must be idempotent: claim the session atomically and close +the fd exactly once. +""" + +import threading + + +def test_terminate_session_closes_master_fd_exactly_once(monkeypatch): + import app + + closed = [] + monkeypatch.setattr(app.os, "close", lambda fd: closed.append(fd)) + monkeypatch.setattr(app.os, "kill", lambda *a, **k: None) + monkeypatch.setattr(app.time, "sleep", lambda *a, **k: None) + monkeypatch.setattr(app, "_emit_from_thread", lambda *a, **k: None) + + fake_fd = 999777 + sid = "sess-idempotent-test" + with app.sessions_lock: + app.sessions[sid] = { + "lock": threading.Lock(), + "pid": 2147480000, # implausible; os.kill is mocked anyway + "master_fd": fake_fd, + "transcript_fh": None, + } + + # Two callers, same session: explicit close, then read-thread auto-terminate. + app.terminate_session(sid, 2147480000, fake_fd) + app.terminate_session(sid, 2147480000, fake_fd) + + assert closed.count(fake_fd) == 1, ( + f"master_fd was closed {closed.count(fake_fd)}x — a double close can land " + f"on a reused fd and corrupt unrelated I/O (EBADF)" + ) + assert sid not in app.sessions + + +def test_terminate_session_missing_session_is_noop(monkeypatch): + """Terminating an unknown/already-removed session must not close any fd.""" + import app + + closed = [] + monkeypatch.setattr(app.os, "close", lambda fd: closed.append(fd)) + monkeypatch.setattr(app.os, "kill", lambda *a, **k: None) + monkeypatch.setattr(app.time, "sleep", lambda *a, **k: None) + monkeypatch.setattr(app, "_emit_from_thread", lambda *a, **k: None) + + app.terminate_session("sess-does-not-exist", 2147480000, 999778) + assert closed == [] diff --git a/tests/test_transcript.py b/tests/test_transcript.py new file mode 100644 index 0000000..d48dbbd --- /dev/null +++ b/tests/test_transcript.py @@ -0,0 +1,133 @@ +"""Unit tests for the transcript tee in read_pty_output. + +These tests exercise the tee logic directly by simulating output dispatch into +a synthesized session dict and a real on-disk transcript file. The full PTY +read loop is not exercised here — see test_mcp_integration.py for E2E. +""" +import os +import stat +import threading +from pathlib import Path + +import pytest + +# The three tests that hit mcp_create_pty_session call pty.openpty(), which +# fails in headless CI containers without TTY allocators. Mark those tests +# explicitly so existing fixture-based tests (test_tee_*) keep running. +def _pty_is_usable() -> bool: + if not hasattr(os, "openpty"): + return False + try: + master, slave = os.openpty() + os.close(master) + os.close(slave) + return True + except OSError: + return False + + +_pty_available = _pty_is_usable() +_pty_skip = pytest.mark.skipif(not _pty_available, reason="pty.openpty() not available") + + +@pytest.fixture +def session_dict(tmp_path): + """Build a minimally valid sessions[pty_id] entry with a real transcript handle.""" + transcript = tmp_path / "transcript.log" + fh = open(transcript, "ab", buffering=0) + os.fchmod(fh.fileno(), 0o600) + return { + "transcript_path": str(transcript), + "transcript_fh": fh, + "transcript_bytes": 0, + "lock": threading.Lock(), + } + + +def _write_chunk(session, output: bytes, cap: int = 10 * 1024 * 1024) -> None: + """Mirror the tee logic from read_pty_output for unit testing.""" + from app import _tee_transcript_chunk + _tee_transcript_chunk(session, output, cap=cap) + + +def test_tee_writes_bytes_and_flushes(session_dict): + _write_chunk(session_dict, b"hello world\n") + assert session_dict["transcript_bytes"] == 12 + assert Path(session_dict["transcript_path"]).read_bytes() == b"hello world\n" + + +def test_tee_chmod_is_0600(session_dict): + mode = stat.S_IMODE(os.stat(session_dict["transcript_path"]).st_mode) + assert mode == 0o600 + + +def test_tee_truncation_at_cap(session_dict): + cap = 16 + _write_chunk(session_dict, b"AAAAAAAAAA", cap=cap) + _write_chunk(session_dict, b"BBBBBBBBBBBBBBBBBBBB", cap=cap) + body = Path(session_dict["transcript_path"]).read_bytes() + # 10 A's, then 6 B's, then truncation marker. + assert body.startswith(b"AAAAAAAAAABBBBBB") + assert b"[transcript truncated at" in body + # Handle is closed after marker + assert session_dict["transcript_fh"] is None + + +def test_tee_no_op_when_fh_is_none(session_dict): + session_dict["transcript_fh"] = None + _write_chunk(session_dict, b"should not write") + assert Path(session_dict["transcript_path"]).read_bytes() == b"" + + +def test_tee_handles_write_error(session_dict, monkeypatch): + # Close the handle out from under the tee — write() will ValueError. + session_dict["transcript_fh"].close() + _write_chunk(session_dict, b"this will fail") + # Handle replaced with None; no crash. + assert session_dict["transcript_fh"] is None + + +@_pty_skip +def test_mcp_create_pty_session_opens_transcript_when_path_given(tmp_path, monkeypatch): + monkeypatch.setattr("app.MAX_CONCURRENT_SESSIONS", 5) + transcript = tmp_path / "transcript.log" + from app import mcp_create_pty_session, sessions, mcp_close_pty_session + sid = mcp_create_pty_session(label="test", transcript_path=str(transcript)) + try: + assert transcript.exists() + mode = stat.S_IMODE(os.stat(transcript).st_mode) + assert mode == 0o600 + sess = sessions[sid] + assert sess["transcript_path"] == str(transcript) + assert sess["transcript_fh"] is not None + assert sess["transcript_bytes"] == 0 + finally: + mcp_close_pty_session(sid) + + +@_pty_skip +def test_mcp_create_pty_session_no_transcript_when_path_none(monkeypatch): + monkeypatch.setattr("app.MAX_CONCURRENT_SESSIONS", 5) + from app import mcp_create_pty_session, sessions, mcp_close_pty_session + sid = mcp_create_pty_session(label="test") + try: + sess = sessions[sid] + assert sess.get("transcript_fh") is None + assert sess.get("transcript_path") is None + finally: + mcp_close_pty_session(sid) + + +@_pty_skip +def test_terminate_session_closes_transcript_handle(tmp_path, monkeypatch): + monkeypatch.setattr("app.MAX_CONCURRENT_SESSIONS", 5) + transcript = tmp_path / "transcript.log" + from app import mcp_create_pty_session, sessions, mcp_close_pty_session + sid = mcp_create_pty_session(label="test", transcript_path=str(transcript)) + fh = sessions[sid]["transcript_fh"] + mcp_close_pty_session(sid) + assert fh.closed + # Session removed from dict + assert sid not in sessions + + diff --git a/tests/test_url_builder.py b/tests/test_url_builder.py new file mode 100644 index 0000000..907287e --- /dev/null +++ b/tests/test_url_builder.py @@ -0,0 +1,82 @@ +"""Tests for url_builder module — base URL resolution for viewer_url.""" +import os +import importlib +from unittest import mock + +import pytest + + +@pytest.fixture(autouse=True) +def _reset_module(): + """Re-import url_builder fresh for each test (module-level cache).""" + from coda_mcp import url_builder + importlib.reload(url_builder) + yield + + +def test_returns_none_when_neither_env_nor_cache(): + from coda_mcp import url_builder + assert url_builder.build_viewer_url("pty-1") is None + + +def test_env_override_wins(): + from coda_mcp import url_builder + with mock.patch.dict(os.environ, {"CODA_APP_URL": "https://override.example.com"}): + assert url_builder.build_viewer_url("pty-1") == \ + "https://override.example.com/?session=pty-1" + + +def test_env_override_strips_trailing_slash(): + from coda_mcp import url_builder + with mock.patch.dict(os.environ, {"CODA_APP_URL": "https://override.example.com/"}): + assert url_builder.build_viewer_url("pty-1") == \ + "https://override.example.com/?session=pty-1" + + +def test_header_capture_used_when_no_env(): + from coda_mcp import url_builder + url_builder.capture_from_headers("app.databricksapps.com") + assert url_builder.build_viewer_url("pty-1") == \ + "https://app.databricksapps.com/?session=pty-1" + + +def test_env_overrides_header_capture(): + from coda_mcp import url_builder + url_builder.capture_from_headers("captured.example.com") + with mock.patch.dict(os.environ, {"CODA_APP_URL": "https://override.example.com"}): + assert url_builder.build_viewer_url("pty-1") == \ + "https://override.example.com/?session=pty-1" + + +def test_header_capture_overwrites_previous(): + from coda_mcp import url_builder + url_builder.capture_from_headers("first.example.com") + url_builder.capture_from_headers("second.example.com") + assert "second.example.com" in url_builder.build_viewer_url("pty-1") + + +def test_capture_empty_string_does_not_overwrite(): + from coda_mcp import url_builder + url_builder.capture_from_headers("good.example.com") + url_builder.capture_from_headers("") + assert "good.example.com" in url_builder.build_viewer_url("pty-1") + + +def test_capture_none_does_not_crash(): + from coda_mcp import url_builder + url_builder.capture_from_headers(None) + assert url_builder.build_viewer_url("pty-1") is None + + +def test_capture_strips_scheme_prefix(): + from coda_mcp import url_builder + url_builder.capture_from_headers("https://app.example.com") + assert url_builder._app_url_cache == "app.example.com" + assert url_builder.build_viewer_url("pty-1") == "https://app.example.com/?session=pty-1" + + +def test_capture_strips_http_scheme_prefix(): + from coda_mcp import url_builder + url_builder.capture_from_headers("http://app.example.com/") + # http stripped, trailing slash stripped + assert url_builder._app_url_cache == "app.example.com" diff --git a/todos.md b/todos.md new file mode 100644 index 0000000..17cf9cf --- /dev/null +++ b/todos.md @@ -0,0 +1,70 @@ +# Pending work (scratch — wipe after done) + +For each todo, the loop is: +1. Brainstorm shape → **critique gate** +2. Plan → **critique gate** +3. Implement → **critique gate** + +A critique pass is mandatory at every gate (use `oh-my-claudecode:critic` or `oh-my-claudecode:architect` subagents, depending on whether the review is about quality/quality or design/architecture). + +--- + +## Todo 1 — `coda_run` returns replay-only URL (no live attach) + +**Intent.** Split the two use cases by tool, not by URL behavior. `coda_run` is fire-and-forget batch — its returned `viewer_url` should be **read-only static replay** of what the agent did. Live interaction is the exclusive surface area of Todo 2. + +**Why.** Today `coda_run`'s `viewer_url` does double duty: live PTY attach during a 5-minute grace window, then static replay forever after. With `coda_interactive` arriving in Todo 2 as the dedicated live-attach tool, the dual-mode on `coda_run` is no longer useful — it just confuses the contract. + +**Scope hint** (to refine in brainstorming): +- Server: `coda_run`'s `viewer_url` should resolve to the static-replay endpoint, not the live-PTY join path +- Static replay reads the on-disk transcript that's already being written (no changes to the tee mechanism) +- The 5-minute PTY grace period for live attach is no longer reachable from `coda_run`'s URL (still applies to `coda_interactive`) +- Update test expectations in `test_mcp_integration.py`, `test_mcp_server.py`, `test_replay_attach.py` + +--- + +## Todo 2 — New MCP tool `coda_interactive` + +**Intent.** MCP caller hands off to a human. Task is "running" until the agent process exits (human types `exit` / `/quit` / Ctrl-D). + +**Default agent.** `claude`. Pluggable via `agent` parameter: `claude` (default), `hermes`, `codex`, `gemini`, `opencode`. + +**Surface** (to refine in brainstorming): +```python +coda_interactive( + prompt: str, + agent: str = "claude", + email: str = "", + context: str = "", + previous_session_id: str = "", + timeout_s: int = 1800, # 30 min — human-driven, generous +) +``` + +**Returns:** `{task_id, session_id, viewer_url, agent, status: "awaiting_human", instructions}` + +**Flow** (to refine in brainstorming): +1. Reuse `coda_run`'s task setup (task_dir, prompt.txt, meta.json, PTY with transcript_path) +2. Send agent launch command per agent matrix +3. Wait briefly for agent to initialize +4. Paste prompt as first user message +5. Watcher polls for PTY child exit (master_fd EOF) — not `result.json` +6. On exit, write `result.json` = `{status: "completed", agent, transcript_path, exit_reason}` + +**Agent launch matrix** (verify in brainstorming): +| Agent | Launch command | +|-------|----------------| +| `claude` | `claude` | +| `hermes` | `hermes chat` | +| `codex` | `codex` | +| `gemini` | `gemini` (or `gemini chat`?) | +| `opencode` | `opencode` | + +--- + +## Workflow rules + +- One todo at a time. Finish Todo 1 fully (brainstorm → critique → plan → critique → implement → critique) before starting Todo 2. +- Every critique gate uses a fresh subagent. No skipping. +- Both todos share the same branch (`coda-mcp`). +- Both eventually go into the same PR (or a new PR that subsumes #66 — decide later). diff --git a/tools/coda-bridge.py b/tools/coda-bridge.py new file mode 100644 index 0000000..c67b54c --- /dev/null +++ b/tools/coda-bridge.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Stdio-to-HTTP MCP bridge with Databricks OAuth token injection. + +Proxies MCP JSON-RPC (stdio) to a Databricks App (Streamable HTTP), +injecting fresh OAuth tokens via `databricks auth token`. + +Config via environment variables (set in Claude Code settings.json): + + CODA_MCP_URL — App MCP endpoint URL + DATABRICKS_PROFILE — Databricks CLI profile for auth +""" + +import json +import os +import subprocess +import sys +import time +import urllib.request +import urllib.error + +APP_URL = os.environ.get("CODA_MCP_URL", "") +PROFILE = os.environ.get("DATABRICKS_PROFILE", "DEFAULT") +TOKEN_TTL = 1800 # cache 30 min (tokens last 60) + +_cache = {"token": None, "expires_at": 0.0} +_session_id = None + + +def _log(msg): + print(f"[coda-bridge] {msg}", file=sys.stderr, flush=True) + + +def _get_token(force=False): + now = time.time() + if not force and _cache["token"] and now < _cache["expires_at"]: + return _cache["token"] + result = subprocess.run( + ["databricks", "auth", "token", "-p", PROFILE], + capture_output=True, text=True, timeout=15, + ) + if result.returncode != 0: + raise RuntimeError(f"databricks auth token failed: {result.stderr.strip()}") + data = json.loads(result.stdout) + _cache["token"] = data["access_token"] + _cache["expires_at"] = now + TOKEN_TTL + _log("OAuth token refreshed") + return _cache["token"] + + +def _forward(line): + global _session_id + token = _get_token() + + headers = { + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + "Authorization": f"Bearer {token}", + } + if _session_id: + headers["Mcp-Session-Id"] = _session_id + + req = urllib.request.Request(APP_URL, data=line.encode(), headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=300) as resp: + sid = resp.headers.get("Mcp-Session-Id") + if sid: + _session_id = sid + body = resp.read().decode() + if body.strip(): + sys.stdout.write(body.rstrip("\n") + "\n") + sys.stdout.flush() + except urllib.error.HTTPError as e: + if e.code in (302, 401, 403): + _log(f"Auth failed ({e.code}), forcing token refresh") + token = _get_token(force=True) + headers["Authorization"] = f"Bearer {token}" + retry = urllib.request.Request(APP_URL, data=line.encode(), headers=headers, method="POST") + with urllib.request.urlopen(retry, timeout=300) as resp: + sid = resp.headers.get("Mcp-Session-Id") + if sid: + _session_id = sid + body = resp.read().decode() + if body.strip(): + sys.stdout.write(body.rstrip("\n") + "\n") + sys.stdout.flush() + else: + raise + + +def main(): + if not APP_URL: + _log("FATAL: CODA_MCP_URL not set") + sys.exit(1) + _log(f"Proxying to {APP_URL} (profile={PROFILE})") + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + _forward(line) + except Exception as e: + _log(f"Error: {e}") + try: + msg_id = json.loads(line).get("id") + except Exception: + msg_id = None + if msg_id is not None: + err = json.dumps({ + "jsonrpc": "2.0", + "id": msg_id, + "error": {"code": -32000, "message": str(e)}, + }) + sys.stdout.write(err + "\n") + sys.stdout.flush() + + +if __name__ == "__main__": + main()