diff --git a/CONTEXT.md b/CONTEXT.md index bd686fd..eb16c69 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -409,7 +409,7 @@ under `agent_memory/profile/` (soul.md, agent.md) and `user_memory/profile/` (us `HEARTBEAT.md` / `TOOLS.md` stay at the Workspace root. **Onboarding** (`raven onboard` → `run_wizard`): -The first-run wizard (LLM provider → sandbox → channel → EverOS memory → deep_research) that also seeds the +The first-run wizard (LLM provider → sandbox → channel → EverOS memory → deep_research → cold-start import) that also seeds the Workspace via `sync_workspace_templates()`; gated at startup by `ensure_configured_or_onboard()`. **Bootstrap Files**: diff --git a/docs/superpowers/plans/2026-07-12-raven-upgrade-command.md b/docs/superpowers/plans/2026-07-12-raven-upgrade-command.md deleted file mode 100644 index 9d6953a..0000000 --- a/docs/superpowers/plans/2026-07-12-raven-upgrade-command.md +++ /dev/null @@ -1,664 +0,0 @@ -# Raven Upgrade Command Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a safe `raven upgrade` command that checks and installs the latest published stable Raven Release without requiring users to rerun the public installer. - -**Architecture:** A focused top-level CLI module resolves and validates GitHub Release metadata, compares strict Raven semantic versions, and protects editable or malformed installations. For mutation, a standard-library-only helper runs on the external base Python: POSIX replaces the current process synchronously, while Windows starts an external helper that waits for the uv trampoline to exit before replacing the locked entrypoint. uv's trampoline job silently releases child processes, so no explicit breakaway flag is needed. The helper restores the active uv tool/bin directories and performs the installer-compatible fallback only after the Raven executable is no longer active. TUI dispatch remains unable to run self-upgrade, while dormant update copy points to the real command. - -**Tech Stack:** Python 3.12, Typer, Rich, httpx, importlib.metadata, uv, pytest, React/Ink, TypeScript, Vitest. - -## Global Constraints - -- Only `GET https://api.github.com/repos/EverMind-AI/Raven/releases/latest` defines an available stable update. -- Never advertise unpublished commits, tags without Releases, draft Releases, or prereleases. -- Do not add background or silent automatic updates. -- Do not overwrite editable source checkouts or unsupported package-manager installations. -- Do not modify Raven state under `~/.raven`. -- Use uv for all Python dependency and command execution; never use pip. -- Run Python tests with `uv run pytest`. -- Keep repository comments necessary and English-only. -- Do not add report assets, standalone web artifacts, or files over 1 MiB. -- Use Conventional Commits with ASCII-only English messages. - -## File map - -- Create `raven/cli/upgrade_commands.py`: release lookup, validation, install-mode guards, uv execution, and Typer registration. -- Create `tests/test_cli_upgrade_commands.py`: focused unit and command tests with no real network or tool mutation. -- Create `tests/integration/test_cli_upgrade_real_uv.py`: bounded uv self-replacement test using temporary custom directories. -- Modify `.github/workflows/ci.yml`: run the real self-replacement test on Windows. -- Modify `raven/cli/commands.py`: register the new top-level command. -- Modify `tests/test_cli_smoke.py`: pin `upgrade` in the root command surface. -- Modify `raven/tui_rpc/methods/cli_dispatch.py`: reject upgrades from an active TUI RPC process. -- Modify `tests/test_tui_rpc_cli_dispatch.py`: pin the expanded dispatch blacklist. -- Modify `ui-tui/src/components/branding.tsx`: replace the nonexistent fallback command. -- Modify `ui-tui/src/demo/gallery.tsx`: keep demo data aligned with production copy. -- Modify `ui-tui/src/__tests__/branding.test.tsx`: render and verify the fallback upgrade hint. -- Modify `README.md` and `README.zh-CN.md`: document checks, upgrades, state preservation, and source-install behavior. - ---- - -### Task 1: Release discovery and version comparison - -**Files:** -- Create: `raven/cli/upgrade_commands.py` -- Create: `tests/test_cli_upgrade_commands.py` - -**Interfaces:** -- Consumes: `httpx.Client`, `importlib.metadata.version("raven")`. -- Produces: `ReleaseInfo(version: str, wheel_url: str)`, `_version_key(value: str) -> tuple[int, int, int]`, `_parse_release_payload(payload: object) -> ReleaseInfo`, `_fetch_latest_release() -> ReleaseInfo`, `_current_version() -> str`. - -- [ ] **Step 1: Write failing tests for strict Raven versions and release metadata** - -Add tests with exact stable and invalid examples: - -```python -import pytest - -from raven.cli import upgrade_commands - - -def test_version_key_accepts_documented_stable_versions() -> None: - assert upgrade_commands._version_key("0.1.3") == (0, 1, 3) - assert upgrade_commands._version_key("v2.10.4") == (2, 10, 4) - - -@pytest.mark.parametrize("value", ["0.1", "0.1.3-rc1", "latest", "01.2.3"]) -def test_version_key_rejects_nonstable_versions(value: str) -> None: - with pytest.raises(upgrade_commands.UpgradeError): - upgrade_commands._version_key(value) - - -def test_parse_release_payload_selects_exact_release_wheel() -> None: - release = upgrade_commands._parse_release_payload( - { - "tag_name": "v0.1.4", - "draft": False, - "prerelease": False, - "assets": [ - { - "name": "raven-0.1.4-py3-none-any.whl", - "browser_download_url": "https://github.com/EverMind-AI/Raven/releases/download/v0.1.4/raven-0.1.4-py3-none-any.whl", - } - ], - } - ) - assert release.version == "0.1.4" -``` - -Also cover draft, prerelease, malformed payload, wrong wheel filename, duplicate exact wheels, HTTP URL, wrong host, and wrong repository path. - -- [ ] **Step 2: Run the focused tests and confirm the red state** - -Run: - -```bash -uv run pytest tests/test_cli_upgrade_commands.py -q -``` - -Expected: collection or import failure because `raven.cli.upgrade_commands` does not exist. - -- [ ] **Step 3: Implement the immutable release record and strict parsing** - -Create the module with these boundaries: - -```python -from __future__ import annotations - -import re -from dataclasses import dataclass -from importlib import metadata -from urllib.parse import urlparse - -import httpx - -LATEST_RELEASE_API = "https://api.github.com/repos/EverMind-AI/Raven/releases/latest" -_VERSION_RE = re.compile(r"^v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$") - - -class UpgradeError(RuntimeError): - pass - - -@dataclass(frozen=True) -class ReleaseInfo: - version: str - wheel_url: str - - -def _version_key(value: str) -> tuple[int, int, int]: - match = _VERSION_RE.fullmatch(value) - if match is None: - raise UpgradeError(f"Unsupported Raven version: {value}") - major, minor, patch = match.groups() - return int(major), int(minor), int(patch) - - -def _current_version() -> str: - return metadata.version("raven") -``` - -Implement `_parse_release_payload` so it validates booleans, the exact `raven-X.Y.Z-py3-none-any.whl` asset, HTTPS, host `github.com`, and path `/EverMind-AI/Raven/releases/download/vX.Y.Z/` before returning `ReleaseInfo`. - -- [ ] **Step 4: Implement the HTTP boundary with deterministic errors** - -Use an injectable client and the public API headers: - -```python -def _fetch_latest_release(client: httpx.Client | None = None) -> ReleaseInfo: - headers = { - "Accept": "application/vnd.github+json", - "User-Agent": f"raven/{_current_version()}", - "X-GitHub-Api-Version": "2022-11-28", - } - if client is not None: - response = client.get(LATEST_RELEASE_API, headers=headers) - response.raise_for_status() - return _parse_release_payload(response.json()) - with httpx.Client(timeout=10.0, follow_redirects=True) as owned_client: - response = owned_client.get(LATEST_RELEASE_API, headers=headers) - response.raise_for_status() - return _parse_release_payload(response.json()) -``` - -Extend tests with `httpx.MockTransport` for success, timeout, non-2xx, and invalid JSON. The command layer will translate `httpx.HTTPError`, `ValueError`, and `UpgradeError` into user-facing failures. - -- [ ] **Step 5: Run focused tests and commit the release-discovery unit** - -Run: - -```bash -uv run pytest tests/test_cli_upgrade_commands.py -q -uv run ruff check raven/cli/upgrade_commands.py tests/test_cli_upgrade_commands.py -uv run ruff format --check raven/cli/upgrade_commands.py tests/test_cli_upgrade_commands.py -``` - -Expected: all Task 1 tests pass and lint exits zero. - -Commit: - -```bash -git add raven/cli/upgrade_commands.py tests/test_cli_upgrade_commands.py -git commit -m "feat(cli): resolve raven release upgrades" -``` - -### Task 2: Installation guards and upgrade command - -**Files:** -- Modify: `raven/cli/upgrade_commands.py` -- Modify: `tests/test_cli_upgrade_commands.py` -- Modify: `raven/cli/commands.py:104-118` -- Modify: `tests/test_cli_smoke.py:44-73,154-181` - -**Interfaces:** -- Consumes: `ReleaseInfo`, `_version_key`, `_fetch_latest_release`, `_current_version` from Task 1. -- Produces: `_is_editable_install() -> bool`, `_uv_tool_target() -> ToolInstallTarget | None`, `_handoff_upgrade(release, current_version, target) -> NoReturn`, an inline standard-library helper, and `register(app: typer.Typer) -> None`. - -- [ ] **Step 1: Write failing command and install-mode tests** - -Add CLI tests that monkeypatch all external boundaries: - -```python -from unittest.mock import Mock - -import pytest -from typer.testing import CliRunner - -from raven.cli import upgrade_commands -from raven.cli.commands import app - -WHEEL_URL = "https://github.com/EverMind-AI/Raven/releases/download/v0.1.4/raven-0.1.4-py3-none-any.whl" -runner = CliRunner() - - -def test_upgrade_check_reports_available_without_install(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(upgrade_commands, "_current_version", lambda: "0.1.3") - monkeypatch.setattr( - upgrade_commands, - "_fetch_latest_release", - lambda: upgrade_commands.ReleaseInfo("0.1.4", WHEEL_URL), - ) - handoff = Mock() - monkeypatch.setattr(upgrade_commands, "_handoff_upgrade", handoff) - - result = runner.invoke(app, ["upgrade", "--check"]) - - assert result.exit_code == 0 - assert "0.1.3 -> 0.1.4" in result.stdout - assert "raven upgrade" in result.stdout - handoff.assert_not_called() -``` - -Cover equal versions, a newer local version with no downgrade, editable refusal, unsupported install refusal, missing uv, successful channel install, channel failure followed by base success, both attempts failing, and network/release errors returning exit code 1. - -For exact uv behavior, capture these calls: - -```python -assert calls == [ - ("/usr/bin/uv", f"raven[channels] @ {WHEEL_URL}"), - ("/usr/bin/uv", WHEEL_URL), -] -``` - -- [ ] **Step 2: Run tests and confirm command registration is red** - -Run: - -```bash -uv run pytest tests/test_cli_upgrade_commands.py tests/test_cli_smoke.py -q -``` - -Expected: failures because `upgrade` is not registered and install helpers do not exist. - -- [ ] **Step 3: Implement strict PEP 610 and uv-receipt guards** - -Distinguish an absent `direct_url.json` from a malformed present file. Require -a nonempty URL and exactly one valid `archive_info`, `dir_info`, or `vcs_info` -record. Treat `dir_info.editable` as optional with a false default, but require -it to be boolean when present; require the PEP 610 VCS fields. - -Parse `sys.prefix/uv-receipt.toml` into an immutable `ToolInstallTarget`. -Require a Raven requirement and exactly one Raven entrypoint with an absolute -`install-path`. Derive `UV_TOOL_DIR` from `Path(sys.prefix).parent` and -`UV_TOOL_BIN_DIR` from the entrypoint parent. Missing receipts remain an -unsupported install; present malformed receipts fail closed. - -- [ ] **Step 4: Implement platform-specific post-exit uv execution** - -Resolve uv and `sys._base_executable` to files outside the active Raven tool -prefix. Encode the multiline helper source into a whitespace-free bootstrap, -override `UV_TOOL_DIR` and `UV_TOOL_BIN_DIR` in a copied environment, and build: - -```python -argv = [base_python, "-I", "-c", bootstrap, uv_path, wheel_url, current, latest] -``` - -On POSIX, flush inherited output and call `os.execve(base_python, argv, env)`. -On Windows, append `os.getppid()` to `argv`, start the helper with -`subprocess.Popen(argv, env=env)`, and return after printing that the user must -wait for the final completion message. uv configures its trampoline job with -`JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK`, so this child is not terminated when -the trampoline exits. The helper opens the trampoline process with -`SYNCHRONIZE`, waits for it with a bounded `WaitForSingleObject`, and only then -invokes uv. - -The helper must use only the standard library and trusted argument arrays. It -runs the channels requirement first, falls back to the base wheel, warns only -when that fallback succeeds, prints the final success itself, catches uv -execution `OSError`, and returns the final uv status when both attempts fail. -Because the helper is inline, no temporary artifact needs cleanup. Unit tests -must cover bootstrap transport, Windows process scheduling, parent waiting, and -spawn failures. The real integration test must use uv tool/bin paths containing -spaces and verify the installed version after the helper finishes. - -- [ ] **Step 5: Register and orchestrate the top-level command** - -Add `upgrade_commands` to the import and registration list in `raven/cli/commands.py`. The Typer callback must: - -1. Fetch and validate the latest Release. -2. Compare current and latest version keys. -3. Exit zero for equal or newer-local versions. -4. Print `current -> latest` and exit zero for `--check`. -5. Refuse editable, malformed, and non-uv installs before invoking `_handoff_upgrade`. -6. Do not print success in the parent; the post-exit helper owns final status. -7. Catch `UpgradeError`, `httpx.HTTPError`, JSON errors, and metadata errors, print one actionable red error, and raise `typer.Exit(1)`. - -Register `upgrade` in both `TOP_LEVEL_COMMANDS` and `REGISTERED_COMMAND_NAMES` in `tests/test_cli_smoke.py`. - -Add `tests/integration/test_cli_upgrade_real_uv.py` to install an old temporary -uv tool in custom directories, invoke its own handoff, and verify the same -entrypoint reports the new version. Run this bounded test in a dedicated -Windows CI job to cover executable locking. - -- [ ] **Step 6: Run focused tests and commit the working CLI** - -Run: - -```bash -uv run pytest tests/test_cli_upgrade_commands.py tests/test_cli_smoke.py -q -uv run ruff check raven/cli/upgrade_commands.py raven/cli/commands.py tests/test_cli_upgrade_commands.py tests/test_cli_smoke.py -uv run ruff format --check raven/cli/upgrade_commands.py raven/cli/commands.py tests/test_cli_upgrade_commands.py tests/test_cli_smoke.py -``` - -Expected: all focused tests and lint pass. - -Commit: - -```bash -git add raven/cli/upgrade_commands.py raven/cli/commands.py tests/test_cli_upgrade_commands.py tests/test_cli_smoke.py -git commit -m "feat(cli): add raven upgrade command" -``` - -### Task 3: TUI upgrade safety and accurate hint - -**Files:** -- Modify: `raven/tui_rpc/methods/cli_dispatch.py:66-103` -- Modify: `tests/test_tui_rpc_cli_dispatch.py:232-257` -- Modify: `ui-tui/src/components/branding.tsx:415-435` -- Modify: `ui-tui/src/demo/gallery.tsx:102-104` -- Modify: `ui-tui/src/__tests__/branding.test.tsx` - -**Interfaces:** -- Consumes: the registered `upgrade` command from Task 2. -- Produces: `("upgrade",)` in `_DISPATCH_BLACKLIST`; TUI fallback text `raven upgrade`. - -- [ ] **Step 1: Write failing Python and TypeScript safety tests** - -Extend the Python blacklist expectation and probe: - -```python -expected_entries = { - ("gateway",), - ("provider", "login"), - ("channels", "login", "weixin"), - ("channels", "login", "whatsapp"), - ("sandbox", "shell"), - ("tui",), - ("onboard",), - ("upgrade",), -} -assert _is_dispatch_compatible(["upgrade"]) is False -assert _is_dispatch_compatible(["upgrade", "--check"]) is False -``` - -Render the real session panel in `branding.test.tsx`: - -```tsx -it('recommends the real raven upgrade command', () => { - const info: SessionInfo = { - model: 'anthropic/claude-sonnet-4-6', - skills: {}, - tools: {}, - update_behind: 1 - } - const { lastFrame } = render() - expect(lastFrame()).toContain('raven upgrade') - expect(lastFrame()).not.toContain('raven update') -}) -``` - -Import `SessionPanel` and `SessionInfo` explicitly. - -- [ ] **Step 2: Run both tests and confirm the red state** - -Run: - -```bash -uv run pytest tests/test_tui_rpc_cli_dispatch.py -q -npm test --prefix ui-tui -- src/__tests__/branding.test.tsx -``` - -Expected: Python blacklist mismatch and TypeScript fallback text assertion failure. - -- [ ] **Step 3: Add the TUI blacklist entry and correct both literals** - -Add `("upgrade",)` with an English why-comment stating that replacing the active Raven process is terminal-only. Update blacklist count comments and assertions from seven to eight entries. - -Change both dormant literals: - -```tsx -{info.update_command || 'raven upgrade'} -``` - -```tsx -update_command: 'raven upgrade' -``` - -- [ ] **Step 4: Run TUI and RPC tests and commit** - -Run: - -```bash -uv run pytest tests/test_tui_rpc_cli_dispatch.py tests/test_tui_rpc_commands_catalog.py -q -npm test --prefix ui-tui -- src/__tests__/branding.test.tsx -npm run lint --prefix ui-tui -npm run type-check --prefix ui-tui -``` - -Expected: all commands exit zero. - -Commit: - -```bash -git add raven/tui_rpc/methods/cli_dispatch.py tests/test_tui_rpc_cli_dispatch.py ui-tui/src/components/branding.tsx ui-tui/src/demo/gallery.tsx ui-tui/src/__tests__/branding.test.tsx -git commit -m "fix(cli): keep upgrades outside active tui" -``` - -### Task 4: User documentation - -**Files:** -- Modify: `README.md:52-90` -- Modify: `README.zh-CN.md:56-94` - -**Interfaces:** -- Consumes: the final CLI behavior from Task 2. -- Produces: matching English and Chinese upgrade instructions. - -- [ ] **Step 1: Add an existing-install upgrade section to both READMEs** - -The English section must include these exact commands and boundaries: - -```markdown -### Upgrade an existing installation - -Check for the latest published stable release: - - raven upgrade --check - -Upgrade Raven without resetting configuration, sessions, or memory: - - raven upgrade - -Raven upgrades are user-triggered, not automatic. Editable source installs are -not overwritten; update the checkout and rerun its development setup instead. -``` - -Add the equivalent Chinese section with the same commands and meaning. Also add `raven upgrade --check` and `raven upgrade` to each useful-command table. - -- [ ] **Step 2: Verify documentation scope and commit** - -Run: - -```bash -rg -n "raven upgrade" README.md README.zh-CN.md -git diff --check -PYTHONPATH=. uv run --extra dev python scripts/check_large_files.py origin/main -``` - -Expected: both READMEs contain check and upgrade guidance; checks exit zero. - -Commit: - -```bash -git add README.md README.zh-CN.md -git commit -m "docs: explain raven upgrade workflow" -``` - -### Task 5: Correct Windows handoff semantics - -**Files:** -- Modify: `raven/cli/upgrade_commands.py` -- Modify: `tests/test_cli_upgrade_commands.py` -- Modify: `tests/integration/test_cli_upgrade_real_uv.py` -- Modify: `README.md` -- Modify: `README.zh-CN.md` - -**Interfaces:** -- Consumes: `_UPGRADE_HELPER_SOURCE`, `_handoff_upgrade()`, and `ToolInstallTarget` from Task 2. -- Produces: `_upgrade_helper_bootstrap() -> str`; POSIX synchronous handoff; Windows breakaway handoff that waits for the uv trampoline before mutation. - -- [ ] **Step 1: Add failing Windows transport and scheduling tests** - -Extend `tests/test_cli_upgrade_commands.py` so a simulated Windows handoff uses -paths containing spaces, calls `subprocess.Popen` with the external base Python, -appends a fixed `os.getppid()` value, and never calls `os.execve`. Assert that -the `-c` bootstrap contains no whitespace and that a spawn `OSError` becomes -`UpgradeError`. - -- [ ] **Step 2: Add failing helper parent-wait tests** - -Load `_UPGRADE_HELPER_SOURCE`, replace its `wait_for_parent` global with a mock, -and call `main()` with a fifth parent-PID argument. Assert that the parent wait -completes before the first uv subprocess call and that a nonzero wait result -prevents uv from running. - -- [ ] **Step 3: Run the new unit tests and verify RED** - -Run: - -```bash -uv run pytest tests/test_cli_upgrade_commands.py -q -``` - -Expected: the new Windows tests fail because `_handoff_upgrade()` still calls -`os.execve` with the multiline source and the helper has no parent wait. - -- [ ] **Step 4: Implement the minimal cross-platform handoff** - -Add `base64` and `subprocess` imports and generate a one-line bootstrap -equivalent to: - -```python -exec(compile(__import__("base64").b64decode(encoded), "", "exec")) -``` - -Keep `os.execve` for POSIX. On Windows, append `str(os.getppid())`, call -`subprocess.Popen(argv, env=env)`, and print -`Raven upgrade started. Wait for the completion message before running Raven again.` - -Inside `_UPGRADE_HELPER_SOURCE`, accept the optional fifth PID argument. Open -that process with Windows `SYNCHRONIZE`, wait at most 30 seconds with -`WaitForSingleObject`, close the handle, and refuse to invoke uv when opening or -waiting fails. - -- [ ] **Step 5: Strengthen the real integration test and documentation** - -Copy the discovered uv executable into `external tools`, use `custom tools` and -`custom bin` for the uv installation, and keep the success-output and final -`--version == 2.0.0` assertions. Explain in both READMEs that POSIX completion -is synchronous while Windows users must wait for the helper's completion -message. - -- [ ] **Step 6: Run focused tests and verify GREEN** - -Run: - -```bash -uv run pytest tests/test_cli_upgrade_commands.py tests/integration/test_cli_upgrade_real_uv.py -q -uv run ruff check raven/cli/upgrade_commands.py tests/test_cli_upgrade_commands.py tests/integration/test_cli_upgrade_real_uv.py -uv run ruff format --check raven/cli/upgrade_commands.py tests/test_cli_upgrade_commands.py tests/integration/test_cli_upgrade_real_uv.py -``` - -Expected: all focused tests and lint pass. - -### Task 6: Full verification and pull request - -**Files:** -- Verify all files changed by Tasks 1-4. -- Create: `/tmp/raven-upgrade-pr.md` outside the repository. - -**Interfaces:** -- Consumes: all implementation commits. -- Produces: pushed branch and draft pull request closing issue #111. - -- [ ] **Step 1: Run the complete relevant verification matrix** - -Run: - -```bash -uv run pytest tests/test_cli_upgrade_commands.py tests/test_cli_smoke.py tests/test_tui_rpc_cli_dispatch.py tests/test_tui_rpc_commands_catalog.py -q -uv run pytest -q -uv run ruff check raven/cli/upgrade_commands.py raven/cli/commands.py raven/tui_rpc/methods/cli_dispatch.py tests/test_cli_upgrade_commands.py tests/test_cli_smoke.py tests/test_tui_rpc_cli_dispatch.py -uv run ruff format --check raven/cli/upgrade_commands.py raven/cli/commands.py raven/tui_rpc/methods/cli_dispatch.py tests/test_cli_upgrade_commands.py tests/test_cli_smoke.py tests/test_tui_rpc_cli_dispatch.py -npm test --prefix ui-tui -npm run lint --prefix ui-tui -npm run lint:rpc --prefix ui-tui -npm run type-check --prefix ui-tui -make check-large-files -``` - -Expected: every command exits zero with no failures. - -- [ ] **Step 2: Rebase onto the latest main and rerun focused tests** - -Run: - -```bash -git fetch origin main -git merge-tree --write-tree HEAD origin/main -git rebase origin/main -uv run pytest tests/test_cli_upgrade_commands.py tests/test_cli_smoke.py tests/test_tui_rpc_cli_dispatch.py tests/test_tui_rpc_commands_catalog.py -q -npm test --prefix ui-tui -- src/__tests__/branding.test.tsx -``` - -Expected: the merge-tree and rebase are clean, then focused tests pass. - -- [ ] **Step 3: Run repository message and title lint** - -Run: - -```bash -make check-commits -PR_TITLE='feat(cli): add raven upgrade command' make check-pr-title -``` - -Expected: both lint gates pass. - -- [ ] **Step 4: Draft and validate the PR description** - -Write `/tmp/raven-upgrade-pr.md` with the repository template and these facts: - -```markdown -## Change description - -> Add `raven upgrade --check` and `raven upgrade` using the latest published stable GitHub Release wheel. Protect editable and unsupported installs, preserve `~/.raven` state, keep self-upgrade outside the active TUI process, and document the user workflow. - -Closes #111 - -## Type of change -- [ ] Bug fix -- [x] New feature -- [x] Document -- [ ] Others - -## Related issues (if there is) - -> Closes #111 - -## Checklists - -### Development - -- [x] Lint rules pass locally -- [x] Application changes have been tested thoroughly -- [x] Automated tests covering modified code pass - -### Security - -- [x] Security impact of change has been considered -- [x] Code follows security best practices and guidelines - -### Code review - -- [x] Pull request has a descriptive title and context useful to a reviewer. Screenshots or screencasts are attached as necessary -``` - -Validate: - -```bash -if LC_ALL=C rg -n '[^\x00-\x7F]' /tmp/raven-upgrade-pr.md; then exit 1; fi -cat /tmp/raven-upgrade-pr.md -``` - -Expected: no non-ASCII matches and the full body matches verified work. - -- [ ] **Step 5: Push and open the draft PR** - -Run: - -```bash -git push -u origin feat/raven_upgrade_command -gh pr create --repo EverMind-AI/Raven --base main --head feat/raven_upgrade_command --draft --title 'feat(cli): add raven upgrade command' --body-file /tmp/raven-upgrade-pr.md -``` - -Expected: GitHub prints the new draft PR URL. diff --git a/docs/superpowers/specs/2026-07-12-raven-upgrade-command-design.md b/docs/superpowers/specs/2026-07-12-raven-upgrade-command-design.md deleted file mode 100644 index 6be96c0..0000000 --- a/docs/superpowers/specs/2026-07-12-raven-upgrade-command-design.md +++ /dev/null @@ -1,190 +0,0 @@ -# Raven Upgrade Command Design - -## Status - -Approved for implementation on 2026-07-12. Tracks GitHub issue #111. - -## Context - -Raven's public installers resolve the wheel attached to the latest published -GitHub Release and install it as a global uv tool. Raven does not currently -expose a user-facing update command, so existing users must rerun the original -installer to receive a new release. The TUI also retains dormant Hermes-era -fields that refer to commit counts and a nonexistent `raven update` command. - -## Goals - -- Add `raven upgrade --check` for a read-only release check. -- Add `raven upgrade` for upgrading a supported uv-tool installation. -- Use the latest published stable GitHub Release as the only update source. -- Preserve Raven state under `~/.raven`. -- Refuse to overwrite editable or unsupported installations. -- Keep upgrade execution out of the running TUI process. -- Document the upgrade workflow for all supported operating systems. - -## Non-goals - -- Background or silent automatic updates. -- Updating from an unpublished commit, tag, draft Release, or prerelease. -- Updating a developer's source checkout automatically. -- Enabling PyPI distribution. -- Adding a synchronous network check to TUI startup. - -## Considered approaches - -### Native Python updater (selected) - -The CLI queries GitHub's latest-release endpoint, validates the release wheel, -compares versions, and replaces itself with a standard-library-only helper -running on the tool environment's external base Python. This path is testable, -does not execute downloaded shell code, and releases the active Raven -executable before uv replaces the installation on Windows. - -### Installer wrapper - -The CLI could rerun `install.sh` or `install.ps1`. This would reuse the current -installer but would execute remote scripts, repeat first-install runtime checks, -depend on platform shells, and be harder to test reliably. - -### Check-only assistant - -The CLI could report an available version and print the existing installer -command. This is the smallest change, but it leaves the manual reinstall step -that the feature is intended to remove. - -## Architecture - -### CLI module - -Add `raven/cli/upgrade_commands.py` with the same `register(app)` boundary used -by the other top-level command modules. Keep orchestration in the command -callback and isolate network, metadata, version, and subprocess behavior behind -small functions that unit tests can replace. - -The module will expose a small immutable release record containing the stable -version and wheel URL. Version comparison will accept Raven's documented -`MAJOR.MINOR.PATCH` format and will not treat commit distance as a release. - -### Release resolution - -Use `GET https://api.github.com/repos/EverMind-AI/Raven/releases/latest` through -the existing `httpx` runtime dependency. Require: - -- a stable `vMAJOR.MINOR.PATCH` tag; -- a non-draft, non-prerelease response; -- exactly one Raven `.whl` asset for that version; -- an HTTPS download URL under the Raven GitHub Release path. - -Malformed responses, timeouts, rate limits, missing assets, and non-success -responses must produce actionable errors and a nonzero exit code. - -### Installation-mode protection - -Read the installed distribution's PEP 610 `direct_url.json` metadata. When the -file is present, require a nonempty URL and exactly one valid archive, -directory, or VCS origin record. An editable installation may run -`raven upgrade --check`, but `raven upgrade` must stop and explain that the -source checkout should be pulled and rebuilt. - -Before mutation, require the uv tool receipt in the active environment. Derive -the active `UV_TOOL_DIR` from `sys.prefix` and `UV_TOOL_BIN_DIR` from the Raven -entrypoint's absolute `install-path`; malformed or ambiguous targets fail -closed. A non-editable installation that is not managed by uv must receive the -official installer guidance instead of being overwritten. - -### Upgrade execution - -When a newer release exists, locate `uv` on `PATH` and require both uv and -`sys._base_executable` to be outside the active Raven tool environment. The -standard-library-only helper receives explicit `UV_TOOL_DIR` and -`UV_TOOL_BIN_DIR` values through a copied environment. Its source is encoded -into a single whitespace-free `python -I -c` bootstrap, so Windows argument -marshalling cannot split the multiline program. - -On POSIX, replace the current process with the external base Python via -`os.execve`, preserving synchronous completion and the helper's final status. -Windows cannot provide the same exec semantics: the uv-generated `raven.exe` -trampoline remains locked until it exits. Launch the external helper with -`subprocess.Popen`, pass the trampoline PID, return only after the helper has -been scheduled, and have the helper wait for that parent process to exit before -invoking uv. uv's trampoline job is configured for silent child breakaway, so -an explicit Windows breakaway creation flag is unnecessary and could conflict -with an enclosing job. The helper inherits the console so it can print its -final result. There is no temporary helper file to clean up. - -Only after the active Raven entrypoint is no longer running does the helper -mirror the supported installer flow: - -1. Run `uv tool install --force "raven[channels] @ "`. -2. If optional channel dependencies fail, retry the base wheel. -3. If the base fallback succeeds, warn that channel adapters may be - unavailable. -4. If both attempts fail, print the final uv failure status with recovery - guidance. On POSIX that status is returned synchronously; on Windows the - original command reports only whether the detached handoff was scheduled. - Raven state under `~/.raven` remains untouched. - -All process calls receive trusted argument arrays without a shell. The command -does not modify `~/.raven`, so configuration, sessions, memory, and runtime -state remain intact. The helper prints the final success or failure after uv -finishes. Windows users must wait for that completion message before running -Raven again. - -### TUI boundary - -Add `upgrade` to the TUI RPC dispatch blacklist because replacing Raven from an -active TUI process is unsafe. Correct dormant fallback/demo text from -`raven update` to `raven upgrade`, but do not add a startup network request or a -new RPC version contract in this PR. - -A future TUI update notice should use release-oriented fields such as -`update_available` and `latest_version`, populated asynchronously or from a -cache, rather than the current `update_behind` commit count. - -## User-visible behavior - -- Current version equals latest: report that Raven is up to date and exit zero. -- Current version is newer than latest: report a development/newer build and - exit zero without downgrading. -- New stable release with `--check`: report `current -> latest`, print - `Run raven upgrade`, and exit zero without a subprocess. -- New stable release without `--check`: run the uv installation flow. -- Editable or unsupported install with `--check`: perform the release comparison - without changing the environment. -- Editable or unsupported install without `--check`: explain the correct update - path and exit nonzero without attempting uv installation. - -## Tests - -Add `tests/test_cli_upgrade_commands.py`, a bounded real-uv integration test, -and update the pinned CLI smoke surface. Cover: - -- command help and root registration; -- up-to-date, newer-local, and update-available comparisons; -- `--check` never invoking uv; -- exact helper argument arrays, custom uv tool/bin targets, and platform-specific - process handoff; -- whitespace-safe helper transport and Windows trampoline waiting; -- the channel-to-base fallback warning and final uv exit status; -- missing uv and both installation attempts failing; -- editable, malformed, and unsupported installation refusal; -- HTTP, malformed metadata, invalid URL, and missing-wheel failures; -- TUI dispatch blacklist and corrected fallback command text. - -The integration test installs a temporary old uv tool in custom directories -whose paths contain spaces, runs its own upgrade handoff, and verifies that the -same entrypoint reports the new version. A dedicated Windows CI job runs this -test to protect the argument-marshalling and executable-locking scenarios that -motivate the external helper. - -Run the focused Python suite, CLI/TUI RPC tests, TUI type/lint/tests, repository -lint, commit-message lint, PR-title lint, and the large-file check before the PR. - -## Documentation - -Update both README files so existing users understand that: - -- `raven upgrade --check` checks the latest stable Release; -- `raven upgrade` replaces the installed Raven tool without resetting state; -- the command is manual, not a background auto-update service; -- source installations follow the developer update workflow. diff --git a/pyproject.toml b/pyproject.toml index c851c4e..6eecad7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "tiktoken>=0.7.0,<1.0.0", "questionary>=2.0.0,<3.0.0", "watchfiles>=0.21,<2.0", - "everos[multimodal]==1.1.2", + "everos[multimodal]==1.1.3", "tomli-w>=1.2.0", # OAuth provider login (Codex / GitHub Copilot) imports this during onboard, # so it must be a runtime dep -- otherwise `uv tool install raven` users hit diff --git a/raven/channels/adapters/mochat/parsing.py b/raven/channels/adapters/mochat/parsing.py index 4e8cc8f..64f100f 100644 --- a/raven/channels/adapters/mochat/parsing.py +++ b/raven/channels/adapters/mochat/parsing.py @@ -200,9 +200,8 @@ def mention_gate(config: MochatConfig, target_kind: str, target_id: str, group_i def parse_timestamp(value: Any) -> int | None: """ISO-8601 string -> epoch milliseconds, or None.""" - if not isinstance(value, str) or not value.strip(): - return None - try: - return int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() * 1000) - except ValueError: + if not isinstance(value, str): return None + from raven.utils.text import parse_iso_ts_ms + + return parse_iso_ts_ms(value) diff --git a/raven/cli/commands.py b/raven/cli/commands.py index 14c3989..848b39a 100644 --- a/raven/cli/commands.py +++ b/raven/cli/commands.py @@ -18,6 +18,7 @@ - ``sandbox`` → ``raven/cli/sandbox_commands.py`` - ``sentinel`` → ``raven/cli/sentinel_commands.py`` - ``sessions`` → ``raven/cli/session_commands.py`` + - ``import`` → ``raven/cli/import_commands.py`` - ``skill`` → ``raven/cli/skill_commands.py`` Shared helpers used across multiple command modules live in @@ -141,6 +142,10 @@ def main( app.add_typer(session_app, name="sessions") +from raven.cli.import_commands import import_app + +app.add_typer(import_app, name="import") + def run() -> None: """Console-script entry point. diff --git a/raven/cli/import_commands.py b/raven/cli/import_commands.py new file mode 100644 index 0000000..25979d2 --- /dev/null +++ b/raven/cli/import_commands.py @@ -0,0 +1,495 @@ +"""Cold-start import CLI commands: scan, run, status.""" + +from __future__ import annotations + +import asyncio +import json +import sys +from collections.abc import Callable +from pathlib import Path +from typing import Any, Optional + +import typer +from rich.console import Console +from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn +from rich.table import Table + +from raven.cli._plugin_stack import build_plugin_registry, maybe_build_memory_backend +from raven.config.loader import load_config +from raven.importer.orchestrator import ImportSummary, ProgressEvent, run_import +from raven.importer.state import ImportState +from raven.importer.types import Platform, Scanner, ScanResult, SourceKind, Tier, filter_by_tier + +console = Console() + +import_app = typer.Typer( + help="Cold-start import from other AI tools", + invoke_without_command=True, + no_args_is_help=True, +) + + +PLATFORM_DISPLAY_NAMES: dict[str, str] = { + Platform.CLAUDE_CODE: "Claude Code", + Platform.CODEX: "Codex", + Platform.KIMICODE: "Kimi Code", + Platform.HERMES: "Hermes", + Platform.OPENCLAW: "OpenClaw", +} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _default_state() -> ImportState: + return ImportState() + + +def _format_size(size_bytes: int) -> str: + if size_bytes < 1024: + return f"{size_bytes} B" + if size_bytes < 1024 * 1024: + return f"{size_bytes / 1024:.0f} KB" + return f"{size_bytes / (1024 * 1024):.1f} MB" + + +def _platform_option(value: Optional[str]) -> Platform | None: + if value is None: + return None + try: + return Platform(value) + except ValueError: + raise typer.BadParameter(f"Unknown platform {value!r}. Available: {', '.join(p.value for p in Platform)}") + + +async def _build_and_run( + items: list[tuple[Scanner, ScanResult]], + state: ImportState, + *, + on_progress: Callable[[ProgressEvent], None] | None = None, + cancel_path: Path | None = None, +) -> ImportSummary: + from raven.config.raven import load_raven_config + + workspace = load_config().workspace_path + ec_config = load_raven_config() + registry = build_plugin_registry(ec_config) + backend = maybe_build_memory_backend(workspace, ec_config, registry=registry) + if backend is None: + console.print( + "[red]No memory backend configured. Run `raven onboard` first.[/red]", + ) + raise typer.Exit(1) + + try: + await backend.start() + except Exception as e: + console.print(f"[red]Failed to start EverOS memory server: {e}[/red]") + console.print("[dim]Check the server log: ~/.raven/logs/everos-server.log[/dim]") + console.print("[dim]Retry: raven import run[/dim]") + raise typer.Exit(1) + try: + return await run_import(items, backend, state, on_progress=on_progress, cancel_path=cancel_path) + finally: + await backend.stop() + + +# --------------------------------------------------------------------------- +# scan +# --------------------------------------------------------------------------- + + +@import_app.command("scan") +def scan_cmd( + platform: Optional[str] = typer.Option(None, "--platform", help="Filter to a specific platform"), +) -> None: + """Preview importable data from other AI tools.""" + from loguru import logger as _logger + + _logger.disable("raven") + platform_filter = _platform_option(platform) + + async def _do() -> list[ScanResult]: + from raven.importer.scanners import scan_all + + return await scan_all(platform_filter=platform_filter) + + try: + results = asyncio.run(_do()) + finally: + _logger.enable("raven") + + if not results: + console.print("No importable data found.") + console.print(f"Supported platforms: {', '.join(PLATFORM_DISPLAY_NAMES.values())}") + return + + table = Table(title="Cold-Start Import -- Available Sources") + table.add_column("Platform") + table.add_column("Kind") + table.add_column("Source Key") + table.add_column("Files", justify="right") + table.add_column("Size", justify="right") + + for r in sorted(results, key=lambda x: (x.platform, x.kind, x.source_key)): + table.add_row( + PLATFORM_DISPLAY_NAMES.get(r.platform.value, r.platform.value), + r.kind.value, + r.source_key, + str(len(r.file_paths)), + _format_size(r.estimated_size), + ) + + console.print(table) + mem = sum(1 for r in results if r.kind == SourceKind.MEMORY_FILE) + conv = sum(1 for r in results if r.kind == SourceKind.CONVERSATION) + console.print(f"\nTotal: {len(results)} items ({mem} memory files, {conv} conversations)") + + +# --------------------------------------------------------------------------- +# status +# --------------------------------------------------------------------------- + + +@import_app.command("status") +def status_cmd( + output_json: bool = typer.Option(False, "--json", help="Output raw JSON"), +) -> None: + """Show cold-start import progress.""" + import time + from collections import Counter + + from rich.progress_bar import ProgressBar + + from raven.config.paths import get_logs_dir + + state = _default_state() + progress = state.get_progress() + entries = {k: v for k, v in progress.get("entries", {}).items() if ":" in k} + meta = progress.get("meta", {}) + total = meta.get("total", len(entries)) + + if not total and not entries: + if output_json: + console.print(json.dumps({"total": 0, "submitted": 0, "failed": 0, "skipped": 0, "status": "none"})) + else: + console.print("No import in progress. Run `raven import run` to start.") + return + + # Compute counts + status_counts = Counter(v.get("status") for v in entries.values()) + submitted = status_counts.get("submitted", 0) + failed = status_counts.get("failed", 0) + done = submitted + failed + remaining = max(0, total - done) + + # Per-platform breakdown + platform_stats: dict[str, dict[str, int]] = {} + failed_items: list[tuple[str, str]] = [] + timestamps: list[float] = [] + for key, entry in entries.items(): + platform = key.split(":", 1)[0] if ":" in key else "unknown" + if platform not in platform_stats: + platform_stats[platform] = {"submitted": 0, "failed": 0, "total": 0} + platform_stats[platform]["total"] += 1 + platform_stats[platform][entry.get("status", "unknown")] = ( + platform_stats[platform].get(entry.get("status", "unknown"), 0) + 1 + ) + if entry.get("timestamp"): + timestamps.append(entry["timestamp"]) + if entry.get("status") == "failed": + failed_items.append((key, entry.get("error", "unknown error"))) + + # Timing + now = time.time() + last_update = max(timestamps) if timestamps else 0 + first_update = min(timestamps) if timestamps else 0 + + if output_json: + console.print( + json.dumps( + { + "total": total, + "submitted": submitted, + "failed": failed, + "remaining": remaining, + "entries": entries, + } + ) + ) + return + + # Visual output + console.print("\n [bold]Cold-Start Import Status[/bold]\n") + + # Progress bar + pct = int(done / total * 100) if total else 0 + bar = ProgressBar(total=total, completed=done, width=30) + console.print(" ", bar, f" {pct}% {done}/{total}") + + cancel_file = state.cancel_path + if cancel_file.exists() and remaining > 0: + console.print(" [yellow]Cancelled[/yellow]\n") + else: + console.print() + + # Platform table + table = Table(show_header=True, box=None, padding=(0, 2, 0, 0)) + table.add_column("Platform", style="bold") + table.add_column("Submitted", justify="right", style="green") + if failed: + table.add_column("Failed", justify="right", style="yellow") + table.add_column("Remaining", justify="right") + table.add_column("Total", justify="right") + for plat, stats in sorted(platform_stats.items()): + display_name = PLATFORM_DISPLAY_NAMES.get(plat, plat) + plat_done = stats.get("submitted", 0) + stats.get("failed", 0) + plat_remaining = stats["total"] - plat_done + row = [display_name, str(stats.get("submitted", 0))] + if failed: + row.append(str(stats.get("failed", 0))) + row.append(str(plat_remaining)) + row.append(str(stats["total"])) + table.add_row(*row) + console.print(table) + + # Timing + console.print() + if first_update and last_update: + duration = int(last_update - first_update) + mins, secs = divmod(duration, 60) + console.print(f" Duration: {mins}m {secs}s") + if last_update: + ago = int(now - last_update) + ago_mins, ago_secs = divmod(ago, 60) + if ago_mins: + console.print(f" Updated: {ago_mins}m {ago_secs}s ago") + else: + console.print(f" Updated: {ago_secs}s ago") + + log_path = get_logs_dir() / "import.log" + console.print(f" Log: {log_path}") + + # Failed items + if failed_items: + console.print() + console.print(" [yellow]Failed items:[/yellow]") + for key, error in failed_items: + console.print(f" {key}: {error}") + console.print() + console.print(" Run `raven import run` to retry failed items.") + + console.print() + + +@import_app.command("stop") +def stop_cmd() -> None: + """Cancel a running background import.""" + state = _default_state() + if not state._path.exists(): + console.print("No import in progress.") + return + cancel = state.cancel_path + if cancel.exists(): + console.print("Import is already being cancelled.") + return + cancel.touch() + console.print("Import cancelled. Running item will complete, remaining items skipped.") + console.print("Use `raven import status` to view progress.") + + +# --------------------------------------------------------------------------- +# run +# --------------------------------------------------------------------------- + + +@import_app.command("run") +def run_cmd( + platform: Optional[str] = typer.Option(None, "--platform", help="Platform to import from"), + tier: Optional[str] = typer.Option(None, "--tier", help="Import tier: memory_files or full"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Interactive cold-start import: scan, select, execute.""" + asyncio.run(_run_async(platform=platform, tier=tier, yes=yes)) + + +async def _run_async( + *, + platform: str | None, + tier: str | None, + yes: bool, +) -> None: + from loguru import logger as _logger + + from raven.cli._log_file import redirect_loguru_to_file + + log_path = redirect_loguru_to_file("import.log", terminal_level=None) + + cancel = _default_state().cancel_path + if cancel.exists(): + cancel.unlink(missing_ok=True) + + platform_filter = _platform_option(platform) + from raven.importer.scanners import scan_all + + all_results = await scan_all(platform_filter=platform_filter) + + if not all_results: + console.print("No importable data found.") + return + + if platform_filter is None: + platforms_found = sorted({r.platform for r in all_results}) + if len(platforms_found) == 1: + platform_filter = platforms_found[0] + else: + picked = _pick_platform(platforms_found) + if picked is None: + return + platform_filter = picked + all_results = [r for r in all_results if r.platform == platform_filter] + + if tier is not None: + try: + selected_tier = Tier(tier) + except ValueError: + console.print(f"[red]Unknown tier {tier!r}. Use 'memory_files' or 'full'.[/red]") + raise typer.Exit(1) + else: + selected_tier = _pick_tier(all_results) + if selected_tier is None: + return + + filtered = filter_by_tier(all_results, selected_tier) + if not filtered: + console.print("No items match the selected tier.") + return + + mem = sum(1 for r in filtered if r.kind == SourceKind.MEMORY_FILE) + conv = sum(1 for r in filtered if r.kind == SourceKind.CONVERSATION) + console.print( + f"\nAbout to import {len(filtered)} items " + f"({mem} memory files, {conv} conversations) " + f"from {platform_filter.value if platform_filter else 'all platforms'}.", + ) + if not yes: + if not typer.confirm("Proceed?", default=True): + return + + from raven.importer.scanners import build_scanners + + scanners = build_scanners() + scanner_map = {s.platform: s for s in scanners} + items: list[tuple[Scanner, ScanResult]] = [] + for r in filtered: + scanner = scanner_map.get(r.platform) + if scanner: + items.append((scanner, r)) + + state = _default_state() + state.set_total(len(items)) + + try: + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TaskProgressColumn(), + console=console, + ) as progress: + task_id = progress.add_task("Importing...", total=len(items)) + + def on_progress(event: ProgressEvent) -> None: + progress.update( + task_id, + advance=1, + description=f"[{event.current}/{event.total}] {event.platform}/{event.source_key}", + ) + + summary = await _build_and_run(items, state, on_progress=on_progress, cancel_path=state.cancel_path) + finally: + _logger.remove() + _logger.add(sys.stderr, level="WARNING") + + _print_summary(summary, log_path=log_path) + + +def _pick_platform(platforms: list[Platform]) -> Platform | None: + try: + questionary = _require_questionary() + except SystemExit: + return None + choices = [{"name": p.value, "value": p} for p in platforms] + picked = questionary.select( + "Select platform:", + choices=choices, + ).ask() + return picked + + +def _pick_tier(results: list[ScanResult]) -> Tier | None: + try: + questionary = _require_questionary() + except SystemExit: + return None + mem_count = sum(1 for r in results if r.kind == SourceKind.MEMORY_FILE) + conv_count = sum(1 for r in results if r.kind == SourceKind.CONVERSATION) + choices = [] + if mem_count: + choices.append( + {"name": f"Memory files only ({mem_count} items, fast)", "value": Tier.MEMORY_FILES}, + ) + choices.append( + { + "name": f"Full import ({mem_count + conv_count} items, includes conversations)", + "value": Tier.FULL, + }, + ) + picked = questionary.select( + "Select import tier:", + choices=choices, + ).ask() + return picked + + +def _require_questionary() -> Any: + try: + import questionary + + return questionary + except ImportError: + console.print( + "[red]questionary is required for interactive mode. Install it or use --platform and --tier flags.[/red]", + ) + raise typer.Exit(1) + + +def _print_summary(summary: ImportSummary, *, log_path: Path | None = None) -> None: + console.print() + if summary.cancelled: + console.print("[bold yellow]Import Cancelled[/bold yellow]\n") + elif summary.failed: + console.print("[bold yellow]Import Complete (with errors)[/bold yellow]\n") + else: + console.print("[bold green]Import Complete[/bold green]\n") + console.print(f" Submitted: {summary.submitted} [green]✅[/green]") + if summary.skipped: + console.print(f" Skipped: {summary.skipped} (already imported)") + if summary.failed: + console.print(f" Failed: {summary.failed} [yellow]⚠️[/yellow]") + console.print() + for err in summary.errors: + console.print(f" {err.platform}/{err.source_key}: {err.error}") + if summary.cancelled: + remaining = summary.total - summary.submitted - summary.skipped - summary.failed + console.print(f" Remaining: {remaining}") + console.print() + console.print("Run `raven import run` to continue remaining items.") + elif summary.failed: + console.print() + console.print("Run `raven import run` to retry failed items.") + if log_path: + console.print(f" Log: {log_path}") + console.print() diff --git a/raven/cli/onboard_commands.py b/raven/cli/onboard_commands.py index b8adf39..f92bc9c 100644 --- a/raven/cli/onboard_commands.py +++ b/raven/cli/onboard_commands.py @@ -1,4 +1,4 @@ -"""Five-step onboarding wizard: LLM provider → sandbox → channel → memory → deep research. +"""Six-step onboarding wizard: LLM provider → sandbox → channel → memory → deep research → import. Goal: get a new user from ``pip install`` to a working agent in a few minutes, without ever opening ``~/.raven/config.json`` or @@ -12,7 +12,8 @@ 4. EverOS long-term memory (optional; llm/embedding required once enabled, rerank/multimodal optional) 5. deep_research tool (optional; MiroThinker key + model) - 6. Done + 6. Cold-start import from other AI tools (optional) + 7. Done All writes go through the ``update_providers`` / ``update_channels`` / ``update`` / ``update_everos`` ops libraries — this module owns the UX layer, @@ -43,7 +44,7 @@ console = Console() -_TOTAL_STEPS = 5 +_TOTAL_STEPS = 6 # Sentinel returned by a screen function to ask the runner to go back one # screen; ``None`` from a picker means Ctrl+C (exit). @@ -2087,17 +2088,17 @@ def _everos_section(section: str) -> dict[str, Any]: def _memory_enabled() -> bool: """True iff EverOS memory is both selected AND usable on disk. - "Usable" requires both required models (llm and embedding) in the EverOS - toml: the seed/schema default sets ``memory.backend="everos"`` before any - models exist, so a bare backend check would mis-report a fresh, modelless - install as "enabled". Both roles are ``optional: False``; gating on either - alone would treat a half-configured setup (e.g. llm written but embedding - skipped) as enabled and offer "keep current" over a non-functional memory. + "Usable" requires both required models (llm and embedding) with api_key + in the EverOS toml. The seed/schema default sets model names but leaves + api_key empty, so checking model alone would mis-report a fresh, + unconfigured install as "enabled". """ data = _load_raw_config() if (data.get("memory") or {}).get("backend") != "everos": return False - return bool(_everos_section("llm").get("model") and _everos_section("embedding").get("model")) + llm = _everos_section("llm") + emb = _everos_section("embedding") + return bool(llm.get("model") and llm.get("api_key") and emb.get("model") and emb.get("api_key")) # Providers whose main model can be reused as the EverOS memory LLM: they @@ -2784,6 +2785,23 @@ def _fetch_multimodal_models( return filtered or None +def _match_everos_default(example: str, models: list[str]) -> str: + """Find the best match for ``example`` in the fetched model list. + + The example (e.g. ``gpt-4.1-mini``) is a bare model name, while + ``models`` may carry provider prefixes (``openai/gpt-4.1-mini``). + Returns the first model whose id ends with ``/example`` or equals + ``example`` exactly; falls back to the bare example string so the + autocomplete input is pre-filled even if no exact match exists. + """ + lower = example.lower() + suffix = f"/{lower}" + for mid in models: + if mid.lower() == lower or mid.lower().endswith(suffix): + return mid + return example + + def _everos_pick_model( *, base_url: Optional[str], @@ -2804,12 +2822,14 @@ def _everos_pick_model( if recommendation: console.print(f" [dim]{_t(*recommendation)}[/dim]") if models: + default_model = _match_everos_default(example, models) question = questionary.autocomplete( _t( f"Model ({len(models)} available — type to filter):", f"模型(共 {len(models)} 个 — 输入可筛选):", ), choices=models, + default=default_model, ignore_case=True, match_middle=True, placeholder=_back_placeholder(allow_back), @@ -3056,7 +3076,8 @@ def _config_everos_role( ) while True: # role-menu loop — a back-out of the source picker returns here - current = _everos_section(section).get("model") + sec = _everos_section(section) + current = sec.get("model") if sec.get("api_key") else None if current: choices = [ questionary.Choice(_t(f"Keep current: {current}", f"沿用当前:{current}"), value="keep"), @@ -3209,6 +3230,22 @@ def _step4_memory( """ _step_header(4, _t("EverOS long-term memory", "EverOS 长期记忆")) + import sys + + if sys.platform == "win32": + console.print( + _t( + " [yellow]⚠ EverOS memory engine does not support native Windows.[/yellow]\n" + " [dim]Run Raven inside WSL for full memory support.[/dim]\n" + " [dim]Skipping memory configuration.[/dim]", + " [yellow]⚠ EverOS 记忆引擎暂不支持 Windows 原生环境。[/yellow]\n" + " [dim]在 WSL 中运行 Raven 可获得完整记忆支持。[/dim]\n" + " [dim]已跳过记忆配置。[/dim]", + ) + ) + _set_memory_backend(None) + return None + if skip or non_interactive: # Never configured the required models here → disable backend-driven # memory so runtime doesn't activate EverOS without an llm/embedding. @@ -3311,6 +3348,69 @@ def _step4_memory( ) ) return None + + # Verify EverOS server is reachable (auto-starts if needed) + import asyncio + + from raven.plugin.memory.everos._server import ensure_everos_server + + console.print() + console.print( + _t( + " [dim]Starting EverOS service...[/dim]", + " [dim]正在启动 EverOS 服务...[/dim]", + ) + ) + try: + asyncio.run(ensure_everos_server()) + console.print( + _t( + " [green]✓ EverOS service is running.[/green]", + " [green]✓ EverOS 服务已启动。[/green]", + ) + ) + except RuntimeError as exc: + console.print( + _t( + f" [red]✗ EverOS service failed to start: {exc}[/red]\n" + " [dim]Check: everos installed? Port 18791 free? " + "See ~/.raven/logs/everos-server.log[/dim]", + f" [red]✗ EverOS 服务启动失败：{exc}[/red]\n" + " [dim]请检查：everos 是否安装？端口 18791 是否被占用？" + "查看 ~/.raven/logs/everos-server.log[/dim]", + ) + ) + retry = questionary.select( + _t("What to do?", "怎么办？"), + choices=[ + questionary.Choice(_t("Retry", "重试"), value="retry"), + questionary.Choice(_t("Skip (memory disabled)", "跳过（记忆禁用）"), value="skip"), + ], + style=RAVEN_STYLE, + qmark=_QMARK, + ).ask() + if retry == "retry": + # Recurse once — the loop in _step4_memory handles further retries + try: + asyncio.run(ensure_everos_server()) + console.print( + _t( + " [green]✓ EverOS service is running.[/green]", + " [green]✓ EverOS 服务已启动。[/green]", + ) + ) + except RuntimeError: + console.print( + _t( + " [red]✗ Still failed. Disabling memory.[/red]", + " [red]✗ 仍然失败。禁用记忆功能。[/red]", + ) + ) + _set_memory_backend(None) + return None + else: + _set_memory_backend(None) + return None _set_memory_backend("everos") return None @@ -3393,6 +3493,8 @@ def _print_next_steps(*, warnings: list[str]) -> None: table.add_row('raven agent -m "hello, world"', _t("ask a one-shot question", "一次性提问")) table.add_row("raven channels list", _t("see connected chat channels", "查看已接入的渠道")) table.add_row("raven provider list", _t("check your provider config", "检查当前服务商配置")) + table.add_row("raven import run", _t("import AI tool history into Raven", "将其他 AI 工具的记忆导入 Raven")) + table.add_row("raven --help", _t("see all available commands", "查看所有可用命令")) console.print( Panel( table, @@ -3404,6 +3506,379 @@ def _print_next_steps(*, warnings: list[str]) -> None: ) +# --------------------------------------------------------------------------- +# Step 5 — cold-start import +# --------------------------------------------------------------------------- + + +def _step5_import(*, skip: bool, non_interactive: bool) -> object: + """Step 5 — optionally import conversation history from other AI tools.""" + _step_header(6, _t("Import history from other AI tools", "从其他 AI 工具导入历史")) + + if skip: + console.print( + _t( + " [dim]Skipped via --skip-import.[/dim]", + " [dim]已通过 --skip-import 跳过。[/dim]", + ) + ) + return None + + if non_interactive: + console.print( + _t( + " [dim]Skipped (non-interactive).[/dim]", + " [dim]已跳过（非交互）。[/dim]", + ) + ) + return None + + if not _memory_enabled(): + console.print( + _t( + " [dim]Skipped — EverOS long-term memory is required for history import.[/dim]", + " [dim]已跳过——历史导入需要先启用 EverOS 长期记忆。[/dim]", + ) + ) + return None + + import sys + + from raven.cli._log_file import redirect_loguru_to_file + from raven.cli._styles import RAVEN_STYLE + + questionary = _require_questionary() + action = questionary.select( + _t( + "Would you like to import conversation history from other AI tools? (Claude Code, Codex, etc.)", + "是否要从其他 AI 工具（Claude Code、Codex 等）导入对话历史？", + ), + choices=[ + questionary.Choice(_t("Yes", "是"), value="yes"), + questionary.Choice(_t("No", "否"), value="no"), + ], + style=RAVEN_STYLE, + qmark=_QMARK, + ).ask() + if action is None: + raise typer.Exit(1) + if action == "no": + console.print(_t(" [dim]Skipped.[/dim]", " [dim]已跳过。[/dim]")) + return None + + # Set up file logging for the entire import lifecycle. + # Wrapped in try/finally so logging is restored on any exit path. + from loguru import logger as _restore_logger + + log_path = redirect_loguru_to_file("import.log", terminal_level=None) + _restore_logger.enable("raven") + try: + return _step5_import_body( + questionary=questionary, + log_path=log_path, + ) + finally: + _restore_logger.remove() + _restore_logger.add(sys.stderr, level="WARNING") + _restore_logger.disable("raven") + + +def _step5_import_body( + *, + questionary: Any, + log_path: Any, +) -> object: + """Inner body of step 5, runs with file logging active.""" + import asyncio + + from raven.cli._styles import RAVEN_STYLE + from raven.cli.import_commands import ( + PLATFORM_DISPLAY_NAMES, + _build_and_run, + _default_state, + _print_summary, + ) + from raven.importer.orchestrator import ImportSummary, ProgressEvent + from raven.importer.scanners import build_scanners, scan_all + from raven.importer.types import Platform, Scanner, ScanResult, SourceKind, Tier, filter_by_tier + + all_results = asyncio.run(scan_all()) + if not all_results: + console.print(_t(" No importable data found.", " 未找到可导入的数据。")) + return None + + # Platform selection (sync questionary) + + def _platform_label(items: list[ScanResult], name: str) -> str: + m = sum(1 for r in items if r.kind == SourceKind.MEMORY_FILE) + c = sum(1 for r in items if r.kind == SourceKind.CONVERSATION) + if m and c: + return _t( + f"{name} ({m} memory files, {c} conversations)", + f"{name}（{m} 个记忆文件，{c} 个对话）", + ) + if m: + return _t(f"{name} ({m} memory files)", f"{name}（{m} 个记忆文件）") + return _t(f"{name} ({c} conversations)", f"{name}（{c} 个对话）") + + by_platform: dict[str, list[ScanResult]] = {} + for r in all_results: + by_platform.setdefault(r.platform.value, []).append(r) + + back_value = "back" + + while True: + # -- Platform selection -- + platform_choices = [] + for p in Platform: + name = PLATFORM_DISPLAY_NAMES.get(p.value, p.value) + if p.value in by_platform: + platform_choices.append( + questionary.Choice( + _platform_label(by_platform[p.value], name), + value=p.value, + ) + ) + else: + platform_choices.append( + questionary.Choice( + _t(f"{name} (coming soon)", f"{name}（即将支持）"), + value=f"coming:{p.value}", + ) + ) + platform_choices.append( + questionary.Choice( + _platform_label(all_results, _t("All platforms", "全部平台")), + value="all", + ) + ) + selected_platform = questionary.select( + _t("Select platform:", "选择平台："), + choices=platform_choices, + style=RAVEN_STYLE, + qmark=_QMARK, + ).ask() + if selected_platform is None: + raise typer.Exit(1) + + if selected_platform.startswith("coming:"): + coming_name = PLATFORM_DISPLAY_NAMES.get( + selected_platform.removeprefix("coming:"), + selected_platform.removeprefix("coming:"), + ) + console.print( + _t( + f" [dim]{coming_name} is not yet supported. Stay tuned![/dim]", + f" [dim]{coming_name} 尚未支持，敬请期待！[/dim]", + ) + ) + _step_header(6, _t("Import history from other AI tools", "从其他 AI 工具导入历史")) + continue + + if selected_platform == "all": + results = all_results + else: + results = [r for r in all_results if r.platform.value == selected_platform] + + mem = sum(1 for r in results if r.kind == SourceKind.MEMORY_FILE) + conv = sum(1 for r in results if r.kind == SourceKind.CONVERSATION) + console.print( + _t( + f" {len(results)} items selected ({mem} memory files, {conv} conversations).", + f" 已选 {len(results)} 项（{mem} 个记忆文件，{conv} 个对话）。", + ) + ) + + # -- Tier selection -- + console.print( + _t( + "\n [bold #fbe23f]Memory files only[/bold #fbe23f] " + "[dim]Fast (minutes). Imports preferences, rules, and project knowledge. Low LLM cost.[/dim]\n\n" + " [bold #fbe23f]Full import[/bold #fbe23f] " + "[dim]Slow (may take hours). Imports memory files + all conversation history. Higher LLM cost.[/dim]", + "\n [bold #fbe23f]仅记忆文件[/bold #fbe23f] " + "[dim]快速（分钟级）。导入偏好、规则和项目知识。LLM 开销较少。[/dim]\n\n" + " [bold #fbe23f]完整导入[/bold #fbe23f] " + "[dim]较慢（可能数小时）。导入记忆文件 + 全部对话历史。LLM 开销较多。[/dim]", + ), + highlight=False, + ) + console.print() + tier_choices = [] + if mem: + tier_choices.append( + questionary.Choice( + _t( + f"Memory files only ({mem} items, fast)", + f"仅记忆文件（{mem} 项，快速）", + ), + value=Tier.MEMORY_FILES, + ) + ) + tier_choices.append( + questionary.Choice( + _t( + f"Full import ({mem + conv} items, includes conversations)", + f"完整导入（{mem + conv} 项，含对话）", + ), + value=Tier.FULL, + ) + ) + tier_choices.append( + questionary.Choice( + _t("Back", "返回"), + value=back_value, + ) + ) + selected_tier = questionary.select( + _t("Select import tier:", "选择导入档位："), + choices=tier_choices, + style=RAVEN_STYLE, + qmark=_QMARK, + ).ask() + if selected_tier is None: + raise typer.Exit(1) + if selected_tier == back_value: + _step_header(6, _t("Import history from other AI tools", "从其他 AI 工具导入历史")) + continue + break + + # Filter + filtered = filter_by_tier(results, selected_tier) + if not filtered: + console.print(_t(" No items match the selected tier.", " 所选档位无匹配项。")) + return None + + f_mem = sum(1 for r in filtered if r.kind == SourceKind.MEMORY_FILE) + f_conv = sum(1 for r in filtered if r.kind == SourceKind.CONVERSATION) + + # Execution mode choice + exec_mode = questionary.select( + _t("Select execution mode:", "选择执行方式："), + choices=[ + questionary.Choice( + _t("Run now (wait for completion, show progress)", "立即执行（等待完成，显示进度）"), + value="foreground", + ), + questionary.Choice( + _t( + "Run in background (use raven import status to check progress)", + "后台执行（用 raven import status 查看进度）", + ), + value="background", + ), + ], + style=RAVEN_STYLE, + qmark=_QMARK, + ).ask() + if exec_mode is None: + raise typer.Exit(1) + + # Summary + confirm + platform_display = ( + PLATFORM_DISPLAY_NAMES.get(selected_platform, selected_platform) + if selected_platform != "all" + else _t("All platforms", "全部平台") + ) + tier_display = ( + _t("Memory files only", "仅记忆文件") if selected_tier == Tier.MEMORY_FILES else _t("Full import", "完整导入") + ) + mode_display = _t("Run now", "立即执行") if exec_mode == "foreground" else _t("Background", "后台执行") + console.print( + _t( + f"\n About to import:\n" + f" Platform: {platform_display}\n" + f" Tier: {tier_display}\n" + f" Items: {len(filtered)} ({f_mem} memory files, {f_conv} conversations)\n" + f" Mode: {mode_display}", + f"\n 即将导入:\n" + f" 平台: {platform_display}\n" + f" 档位: {tier_display}\n" + f" 数量: {len(filtered)} 项（{f_mem} 个记忆文件，{f_conv} 个对话）\n" + f" 执行方式: {mode_display}", + ) + ) + if not typer.confirm( + _t(" Start?", " 开始执行？"), + default=True, + ): + return None + + # Build items + scanners = build_scanners() + scanner_map: dict[str, Scanner] = {s.platform: s for s in scanners} + items = [(scanner_map[r.platform], r) for r in filtered if r.platform in scanner_map] + + state = _default_state() + state.set_total(len(items)) + + if exec_mode == "background": + import shutil + import subprocess as _sp + + raven_bin = shutil.which("raven") + if not raven_bin: + console.print( + _t( + " [red]Cannot find 'raven' command. Falling back to foreground execution.[/red]", + " [red]找不到 'raven' 命令。回退到前台执行。[/red]", + ) + ) + exec_mode = "foreground" + else: + platform_flag = selected_platform if selected_platform != "all" else None + cmd = [raven_bin, "import", "run", "--tier", selected_tier.value, "--yes"] + if platform_flag: + cmd.extend(["--platform", platform_flag]) + _sp.Popen( + cmd, + stdout=_sp.DEVNULL, + stderr=_sp.DEVNULL, + start_new_session=True, + ) + console.print( + _t( + f"\n Import started in background.\n" + f" Check progress: [#fbe23f]raven import status[/#fbe23f]\n" + f" Log: {log_path}", + f"\n 导入已在后台启动。\n" + f" 查看进度: [#fbe23f]raven import status[/#fbe23f]\n" + f" 详细日志: {log_path}", + ) + ) + return None + + # Foreground execution (async, with Rich progress) + from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn + + async def _do_import() -> ImportSummary: + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TaskProgressColumn(), + console=console, + ) as progress: + task_id = progress.add_task( + _t("Importing...", "导入中..."), + total=len(items), + ) + + def on_progress(event: ProgressEvent) -> None: + progress.update( + task_id, + advance=1, + description=f"[{event.current}/{event.total}] {event.platform}/{event.source_key}", + ) + + return await _build_and_run(items, state, on_progress=on_progress) + + summary = asyncio.run(_do_import()) + + _print_summary(summary, log_path=log_path) + return None + + # --------------------------------------------------------------------------- # Wizard runner (screen state machine) + reusable entry point # --------------------------------------------------------------------------- @@ -3420,12 +3895,13 @@ def run_wizard( skip_channel: bool = False, skip_memory: bool = False, skip_deep_research: bool = False, + skip_import: bool = False, non_interactive: bool = False, yes: bool = False, reset: bool = False, skip_test: bool = False, ) -> None: - """Run the 5-step onboarding wizard end-to-end. + """Run the 6-step onboarding wizard end-to-end. The reusable entry point: the ``onboard`` CLI command and the startup gate both call this. Screens form a state machine so a ``0) Back`` choice can @@ -3449,6 +3925,7 @@ def run_wizard( skip_channel=skip_channel, skip_memory=skip_memory, skip_deep_research=skip_deep_research, + skip_import=skip_import, non_interactive=non_interactive, yes=yes, reset=reset, @@ -3492,6 +3969,7 @@ def _run_wizard_body( skip_channel: bool = False, skip_memory: bool = False, skip_deep_research: bool = False, + skip_import: bool = False, non_interactive: bool = False, yes: bool = False, reset: bool = False, @@ -3517,13 +3995,13 @@ def _run_wizard_body( "[dim]We'll configure, in order:[/dim]\n" " [#fbe23f]①[/#fbe23f] LLM [#fbe23f]②[/#fbe23f] Run location " "[#fbe23f]③[/#fbe23f] Chat channel [#fbe23f]④[/#fbe23f] Long-term memory " - "[#fbe23f]⑤[/#fbe23f] Deep research\n\n" + "[#fbe23f]⑤[/#fbe23f] Deep research [#fbe23f]⑥[/#fbe23f] Import history\n\n" "[dim]↑↓ select · Enter confirm · Ctrl+C quit anytime — anything already written is kept.[/dim]", "[bold #fbe23f]✨ 欢迎使用 Raven 配置向导[/bold #fbe23f]\n\n" "[dim]我们将依次配置:[/dim]\n" " [#fbe23f]①[/#fbe23f] LLM [#fbe23f]②[/#fbe23f] 运行位置 " "[#fbe23f]③[/#fbe23f] 聊天渠道 [#fbe23f]④[/#fbe23f] 长期记忆 " - "[#fbe23f]⑤[/#fbe23f] 深度研究\n\n" + "[#fbe23f]⑤[/#fbe23f] 深度研究 [#fbe23f]⑥[/#fbe23f] 历史导入\n\n" "[dim]↑↓ 选择 · Enter 确认 · 随时 Ctrl+C 退出 — 已写入的配置会保留。[/dim]", ), border_style="#c8a900", @@ -3560,6 +4038,7 @@ def _run_wizard_body( non_interactive=non_interactive, warnings=warnings, ), + lambda: _step5_import(skip=skip_import, non_interactive=non_interactive), ] index = 0 @@ -3624,6 +4103,7 @@ def onboard( skip_channel: bool = typer.Option(False, "--skip-channel", help="Skip Step 3 (channel setup)"), skip_memory: bool = typer.Option(False, "--skip-memory", help="Skip Step 4 (long-term memory)"), skip_deep_research: bool = typer.Option(False, "--skip-deep-research", help="Skip Step 5 (deep_research tool)"), + skip_import: bool = typer.Option(False, "--skip-import", help="Skip Step 6 (history import)"), non_interactive: bool = typer.Option( False, "--non-interactive", @@ -3641,7 +4121,7 @@ def onboard( help="Skip the one-shot test message (avoids a billed call; connectivity is still checked)", ), ) -> None: - """Five-step setup wizard: LLM provider → sandbox → channel → memory → deep research.""" + """Six-step setup wizard: LLM provider → sandbox → channel → memory → deep research → import.""" run_wizard( provider=provider, api_key=api_key, @@ -3652,6 +4132,7 @@ def onboard( skip_channel=skip_channel, skip_memory=skip_memory, skip_deep_research=skip_deep_research, + skip_import=skip_import, non_interactive=non_interactive, yes=yes, reset=reset, diff --git a/raven/cli/tui_commands.py b/raven/cli/tui_commands.py index a5b3ac1..34126da 100644 --- a/raven/cli/tui_commands.py +++ b/raven/cli/tui_commands.py @@ -27,7 +27,7 @@ import typer -from raven.cli._log_file import _strip_tty_stream_handlers, redirect_loguru_to_file, redirect_terminal_fds_to_file +from raven.cli._log_file import _strip_tty_stream_handlers, redirect_loguru_to_file tui_app = typer.Typer(name="tui", help="Launch Raven native TUI (Ink+React).") @@ -595,100 +595,77 @@ def _agent_loop_factory(): build_error=build_error, ) - from raven.config.paths import get_logs_dir - - # everos embedded structlog uses PrintLogger which calls print() → writes - # directly to fd 1, bypassing stdlib logging entirely. Redirect both fds so - # no everos output can corrupt the full-screen TUI, starting before - # backend.start() (which may trigger lazy warns) through the end of stop(). - # The Node child already inherited the real terminal fds at Popen time (before - # this function ran), so this dup2 does not affect the child's terminal. - async def _start_memory_backend() -> None: - # Bring up the memory backend off the render path: its everos/lancedb - # import (~2-3s) + lifespan must not block the handshake / first render. - # recall/store degrade to empty until it is ready. - try: - await agent_loop.backend.start() - except Exception: - from loguru import logger as _logger - - _logger.exception("tui: memory backend start failed; continuing with degraded memory path") - # everos installs a root stdout StreamHandler during start(); strip it now - # (after the deferred start) so its records reach the file sink. - _strip_tty_stream_handlers() + serve_task = asyncio.create_task(server.serve_forever()) - with redirect_terminal_fds_to_file(get_logs_dir() / "tui.log"): - # Strip any tty handlers installed before serve; the deferred backend - # start strips again after it runs. - _strip_tty_stream_handlers() + try: + # Wait until EITHER handshake completes OR deadline expires OR child exits. + done, pending = await asyncio.wait( + { + asyncio.create_task(handshake_done.wait()), + asyncio.create_task(proc_done.wait()), + }, + timeout=handshake_deadline_s, + return_when=asyncio.FIRST_COMPLETED, + ) + for t in pending: + t.cancel() + # Drain cancelled tasks to suppress warnings. + for t in pending: + try: + await t + except (asyncio.CancelledError, Exception): + pass - serve_task = asyncio.create_task(server.serve_forever()) - backend_start_task: asyncio.Task | None = None + if not handshake_done.is_set(): + return False + # Handshake OK — start memory backend in background (may spawn + # EverOS server, up to 30s) so it doesn't block first render. + if agent_loop is not None and agent_loop.backend is not None: - try: - # Wait until EITHER handshake completes OR deadline expires OR child exits. - done, pending = await asyncio.wait( - { - asyncio.create_task(handshake_done.wait()), - asyncio.create_task(proc_done.wait()), - }, - timeout=handshake_deadline_s, - return_when=asyncio.FIRST_COMPLETED, - ) - for t in pending: - t.cancel() - # Drain cancelled tasks to suppress warnings. - for t in pending: + async def _start_backend() -> None: try: - await t - except (asyncio.CancelledError, Exception): - pass - - if not handshake_done.is_set(): - return False - # Handshake OK (UI rendered) — bring up the memory backend now, in the - # background, so its heavy import + lifespan happens after render. - if agent_loop is not None and agent_loop.backend is not None: - backend_start_task = asyncio.create_task(_start_memory_backend()) - # Continue serving until child exits. - await proc_done.wait() - return True - finally: - # Let the background backend start finish before stop() so stop never - # races a mid-flight start. - if backend_start_task is not None: - try: - await backend_start_task - except (asyncio.CancelledError, Exception): - pass - # Fail-safe any pending confirm so a paused dispatch's worker thread - # is released when the connection drops. - confirm_broker.cancel_all() - if agent_loop is not None and agent_loop.cron_service is not None: - try: - agent_loop.cron_service.stop() - except Exception: - pass - if turn_teardown is not None: - try: - await turn_teardown() - except Exception: - pass - # Release the embedded index lock so the next process can start. - if agent_loop is not None and agent_loop.backend is not None: - try: - await agent_loop.backend.stop() + await agent_loop.backend.start() # type: ignore[union-attr] except Exception: from loguru import logger as _logger _logger.exception( - "tui: memory backend stop failed; continuing shutdown", + "tui: memory backend start failed; continuing with degraded memory path", ) - serve_task.cancel() + _strip_tty_stream_handlers() + + asyncio.create_task(_start_backend()) + + await proc_done.wait() + return True + finally: + # Fail-safe any pending confirm so a paused dispatch's worker thread + # is released when the connection drops. + confirm_broker.cancel_all() + if agent_loop is not None and agent_loop.cron_service is not None: try: - await serve_task - except (asyncio.CancelledError, Exception): + agent_loop.cron_service.stop() + except Exception: + pass + if turn_teardown is not None: + try: + await turn_teardown() + except Exception: pass + # Release the embedded index lock so the next process can start. + if agent_loop is not None and agent_loop.backend is not None: + try: + await agent_loop.backend.stop() + except Exception: + from loguru import logger as _logger + + _logger.exception( + "tui: memory backend stop failed; continuing shutdown", + ) + serve_task.cancel() + try: + await serve_task + except (asyncio.CancelledError, Exception): + pass def _spawn_with_rpc_socket( diff --git a/raven/config/update.py b/raven/config/update.py index 4c9fe6e..895cf76 100644 --- a/raven/config/update.py +++ b/raven/config/update.py @@ -282,8 +282,7 @@ def init_extension_block_defaults(*, config_path: Path | None = None) -> None: plugins.setdefault("config", {}).setdefault( "everos-memory", { - "mode": "embedded", - "base_url": "http://localhost:1995", + "base_url": "http://localhost:18791", "user_id": mem.user_id, "agent_id": mem.agent_id, }, diff --git a/raven/importer/__init__.py b/raven/importer/__init__.py new file mode 100644 index 0000000..ddfc6d8 --- /dev/null +++ b/raven/importer/__init__.py @@ -0,0 +1,32 @@ +"""Cold-start import: discover and ingest history from other AI tools.""" + +from __future__ import annotations + +from raven.importer.orchestrator import ImportFailure, ImportSummary, ProgressEvent, run_import +from raven.importer.scanners import ClaudeCodeScanner +from raven.importer.state import ImportState +from raven.importer.types import ( + ImportMessage, + ImportSession, + Platform, + Scanner, + ScanResult, + SourceKind, + Tier, +) + +__all__ = [ + "ClaudeCodeScanner", + "ImportFailure", + "ImportMessage", + "ImportSession", + "ImportState", + "ImportSummary", + "Platform", + "ProgressEvent", + "ScanResult", + "Scanner", + "SourceKind", + "Tier", + "run_import", +] diff --git a/raven/importer/orchestrator.py b/raven/importer/orchestrator.py new file mode 100644 index 0000000..36f834b --- /dev/null +++ b/raven/importer/orchestrator.py @@ -0,0 +1,246 @@ +"""Cold-start import orchestrator -- read, batch, store, track.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from loguru import logger + +from raven.importer.state import ImportState +from raven.importer.types import ImportMessage, ImportSession, Scanner, ScanResult +from raven.memory_engine.backend import MemoryBackend + +_BATCH_MSG_LIMIT = 100 +_BATCH_CHAR_LIMIT = 30_000 + + +@dataclass(frozen=True) +class ImportFailure: + """One failed import unit.""" + + platform: str + source_key: str + error: str + + +@dataclass(frozen=True) +class ImportSummary: + """Aggregate result of a run_import call.""" + + total: int + submitted: int + skipped: int + failed: int + errors: tuple[ImportFailure, ...] + cancelled: bool = False + + +@dataclass(frozen=True) +class ProgressEvent: + """Progress notification emitted once per ScanResult.""" + + platform: str + source_key: str + status: str + current: int + total: int + error: str | None = None + + +async def run_import( + items: Sequence[tuple[Scanner, ScanResult]], + backend: MemoryBackend, + state: ImportState, + *, + on_progress: Callable[[ProgressEvent], None] | None = None, + cancel_path: Path | None = None, +) -> ImportSummary: + """Import pre-filtered scan results into the memory backend. + + The caller (CLI layer) is responsible for scanning, tier/platform + filtering, and MemoryBackend lifecycle (start/stop). + """ + total = len(items) + logger.info("import started: {} items", total) + + submitted = 0 + skipped = 0 + failed = 0 + errors: list[ImportFailure] = [] + + for i, (scanner, result) in enumerate(items): + if cancel_path is not None and cancel_path.exists(): + logger.info("import cancelled by user after {}/{} items", i, total) + break + + platform = result.platform.value + key = result.source_key + + if state.is_submitted(platform, key): + skipped += 1 + logger.info( + "[{}/{}] skipping {}/{} (already submitted)", + i + 1, + total, + platform, + key, + ) + if on_progress: + on_progress( + ProgressEvent( + platform=platform, + source_key=key, + status="skipped", + current=i + 1, + total=total, + ) + ) + continue + + # NOTE: checkpoint is per source unit, not per batch. A multi-batch + # session that fails mid-way will re-send already-accepted batches + # on retry. EverOS dedup is by session_id so duplicates are safe + # (redundant extraction, no data loss). Per-batch checkpoint is + # deferred until full-conversation import is common enough to + # justify the added state complexity. + logger.info("[{}/{}] importing {}/{}", i + 1, total, platform, key) + try: + session = await scanner.read(result) + await _feed_session(backend, session) + state.mark_submitted(platform, key) + submitted += 1 + logger.info( + "[{}/{}] imported {}/{} ({} messages)", + i + 1, + total, + platform, + key, + len(session.messages), + ) + if on_progress: + on_progress( + ProgressEvent( + platform=platform, + source_key=key, + status="submitted", + current=i + 1, + total=total, + ) + ) + except Exception as e: + err_msg = repr(e) if not str(e) else str(e) + state.mark_failed(platform, key, err_msg) + failed += 1 + errors.append(ImportFailure(platform, key, err_msg)) + logger.warning( + "[{}/{}] failed to import {}/{}: {}", + i + 1, + total, + platform, + key, + err_msg, + ) + if on_progress: + on_progress( + ProgressEvent( + platform=platform, + source_key=key, + status="failed", + current=i + 1, + total=total, + error=err_msg, + ) + ) + + cancelled = cancel_path is not None and cancel_path.exists() + logger.info( + "import finished: {} submitted, {} skipped, {} failed (of {} total){}", + submitted, + skipped, + failed, + total, + " [cancelled]" if cancelled else "", + ) + return ImportSummary( + total=total, + submitted=submitted, + skipped=skipped, + failed=failed, + errors=tuple(errors), + cancelled=cancelled, + ) + + +async def _feed_session(backend: MemoryBackend, session: ImportSession) -> None: + if not session.messages: + return + all_dicts = [_to_store_dict(m) for m in session.messages] + batch: list[dict[str, Any]] = [] + batch_chars = 0 + + async def _flush(*, is_final: bool) -> None: + nonlocal batch, batch_chars + metadata: dict[str, Any] = {"is_final": is_final} + _log_store_request(session.session_id, batch, metadata, batch_chars) + await backend.store(session.session_id, batch, metadata=metadata) + logger.debug("store completed: session_id={}", session.session_id) + batch = [] + batch_chars = 0 + + for msg_dict in all_dicts: + msg_chars = len(msg_dict["content"]) + if batch and (len(batch) >= _BATCH_MSG_LIMIT or batch_chars + msg_chars > _BATCH_CHAR_LIMIT): + await _flush(is_final=False) + batch.append(msg_dict) + batch_chars += msg_chars + + if batch: + await _flush(is_final=True) + + +def _log_store_request( + session_id: str, + batch: list[dict[str, Any]], + metadata: dict[str, Any], + batch_chars: int, +) -> None: + logger.debug( + "store request: session_id={}, metadata={}, messages={}, total_chars={}", + session_id, + metadata, + len(batch), + batch_chars, + ) + for i, msg in enumerate(batch): + content = msg["content"] + if len(content) > 200: + content = content[:200] + f"...(truncated, {len(msg['content'])} chars)" + entry: dict[str, Any] = { + "role": msg["role"], + "content": content, + "timestamp": msg["timestamp"], + } + if "tool_calls" in msg: + entry["tool_calls"] = msg["tool_calls"] + if "tool_call_id" in msg: + entry["tool_call_id"] = msg["tool_call_id"] + logger.debug("store messages[{}]: {}", i, entry) + + +def _to_store_dict(msg: ImportMessage) -> dict[str, Any]: + d: dict[str, Any] = { + "role": msg.role, + "content": msg.content, + "timestamp": msg.timestamp, + } + if msg.tool_calls: + d["tool_calls"] = list(msg.tool_calls) + if msg.tool_call_id: + d["tool_call_id"] = msg.tool_call_id + return d + + +__all__ = ["ImportFailure", "ImportSummary", "ProgressEvent", "run_import"] diff --git a/raven/importer/scanners/__init__.py b/raven/importer/scanners/__init__.py new file mode 100644 index 0000000..fad857b --- /dev/null +++ b/raven/importer/scanners/__init__.py @@ -0,0 +1,43 @@ +"""Platform-specific scanners for cold-start import.""" + +from __future__ import annotations + +import asyncio + +from loguru import logger + +from raven.importer.scanners.claude_code import ClaudeCodeScanner +from raven.importer.types import Platform, Scanner, ScanResult, SourceKind + + +def build_scanners() -> list[Scanner]: + """Return all available scanner instances.""" + return [ClaudeCodeScanner()] + + +async def scan_all( + scanners: list[Scanner] | None = None, + *, + platform_filter: Platform | None = None, +) -> list[ScanResult]: + """Run all scanners concurrently and return aggregated results.""" + if scanners is None: + scanners = build_scanners() + if platform_filter: + scanners = [s for s in scanners if s.platform == platform_filter] + logger.info("scan started: {} scanner(s)", len(scanners)) + + per_scanner = await asyncio.gather(*(s.scan() for s in scanners)) + + results: list[ScanResult] = [] + for scanner, found in zip(scanners, per_scanner): + logger.info("scan {}: {} results", scanner.platform.value, len(found)) + results.extend(found) + + mem = sum(1 for r in results if r.kind == SourceKind.MEMORY_FILE) + conv = sum(1 for r in results if r.kind == SourceKind.CONVERSATION) + logger.info("scan completed: {} results ({} memory_file, {} conversation)", len(results), mem, conv) + return results + + +__all__ = ["ClaudeCodeScanner", "build_scanners", "scan_all"] diff --git a/raven/importer/scanners/claude_code.py b/raven/importer/scanners/claude_code.py new file mode 100644 index 0000000..4e238fb --- /dev/null +++ b/raven/importer/scanners/claude_code.py @@ -0,0 +1,512 @@ +"""ClaudeCodeScanner -- discover and read Claude Code local data.""" + +from __future__ import annotations + +import asyncio +import json +import re +import time +from pathlib import Path +from typing import Any + +from loguru import logger + +from raven.importer.types import ( + ImportMessage, + ImportSession, + Platform, + ScanResult, + SourceKind, +) +from raven.utils.text import is_cjk, parse_frontmatter, parse_iso_ts_ms + +_ACTIVE_THRESHOLD_S = 300 +_MAX_MEMORY_FILE_BYTES = 1_048_576 +_CONTENT_TRUNCATE_LIMIT = 10_000 +_APP_ID = "claude_code" + +_SKIP_CONTENT_TYPES = frozenset({"thinking", "redacted_thinking"}) + +_INTRO_TEMPLATES: dict[str | None, tuple[str, str]] = { + "MEMORY.MD": ( + "以下是项目记忆总览文件 {filename}", + "Here is the project memory overview file named {filename}", + ), + "reference": ( + "以下是关于 {name} 的项目知识，文件 {filename}", + "Here is project knowledge about {name}, file named {filename}", + ), + "feedback": ( + "以下是我对 AI 协作的偏好——{name}，文件 {filename}", + "Here is my preference for AI collaboration -- {name}, file named {filename}", + ), + "project": ( + "以下是关于 {name} 的项目笔记，文件 {filename}", + "Here is a project note about {name}, file named {filename}", + ), + None: ( + "以下是关于 {name} 的笔记，文件 {filename}", + "Here is a note about {name}, file named {filename}", + ), +} + +_FILE_END_TEMPLATES = ( + "{filename} 的内容到此结束。", + "That is all the content from {filename}.", +) + +_GLOBAL_INTRO = ( + "这是我在 Claude Code 中设定的全局偏好和规则。", + "These are my global preferences and rules set in Claude Code.", +) + +_SESSION_PREAMBLE = ( + "这是我在 Claude Code 中 {proj} 项目的记忆文件，共 {count} 个。", + "These are my memory files from Claude Code for the {proj} project, {count} files in total.", +) + +_SESSION_EPILOGUE = ( + "以上是 {proj} 项目的全部 {count} 个记忆文件。", + "End of all {count} memory files for the {proj} project.", +) + + +def _pick(tpl: tuple[str, str], cjk: bool) -> str: + return tpl[0] if cjk else tpl[1] + + +# --------------------------------------------------------------------------- +# Helpers -- content extraction +# --------------------------------------------------------------------------- + + +def _truncate(text: str) -> str: + if len(text) <= _CONTENT_TRUNCATE_LIMIT: + return text + return text[:_CONTENT_TRUNCATE_LIMIT] + "..." + + +def _text_from_content(content: Any) -> str: + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" + parts: list[str] = [] + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") in _SKIP_CONTENT_TYPES: + continue + if block.get("type") == "text": + t = block.get("text", "") + if t: + parts.append(t) + return "\n\n".join(parts) + + +def _tool_calls_from_content(content: list[dict[str, Any]], ts: int) -> tuple[dict[str, Any], ...] | None: + calls: list[dict[str, Any]] = [] + for i, block in enumerate(content): + if not isinstance(block, dict) or block.get("type") != "tool_use": + continue + call_id = block.get("id") or f"claude_tool_{ts}_{i}" + raw_input = block.get("input", {}) + calls.append( + { + "id": call_id, + "type": "function", + "function": { + "name": block.get("name", "unknown"), + "arguments": json.dumps(raw_input) if not isinstance(raw_input, str) else raw_input, + }, + } + ) + return tuple(calls) if calls else None + + +def _tool_results_from_content(content: list[dict[str, Any]], ts: int, sender: str) -> list[ImportMessage]: + msgs: list[ImportMessage] = [] + for block in content: + if not isinstance(block, dict) or block.get("type") != "tool_result": + continue + tool_use_id = block.get("tool_use_id") or block.get("toolCallId") or block.get("tool_call_id") + if not tool_use_id: + logger.debug("tool_result block without tool_use_id, dropped") + continue + inner = block.get("content", "") + if isinstance(inner, list): + inner = _text_from_content(inner) + elif not isinstance(inner, str): + inner = json.dumps(inner) + if not inner: + inner = "(empty tool result)" + msgs.append( + ImportMessage( + role="tool", + content=_truncate(inner), + timestamp=ts, + sender_id=sender, + tool_call_id=tool_use_id, + ) + ) + return msgs + + +# --------------------------------------------------------------------------- +# Helpers -- memory file parsing +# --------------------------------------------------------------------------- + + +def _make_intro(filename: str, fm: dict[str, Any], cjk: bool) -> str: + meta = fm.get("metadata", {}) if isinstance(fm.get("metadata"), dict) else {} + name = fm.get("name") or Path(filename).stem + + key = filename.upper() if filename.upper() == "MEMORY.MD" else meta.get("type") + tpl = _INTRO_TEMPLATES.get(key, _INTRO_TEMPLATES[None]) + return _pick(tpl, cjk).format(name=name, filename=filename) + + +def _make_file_end(filename: str, cjk: bool) -> str: + return _pick(_FILE_END_TEMPLATES, cjk).format(filename=filename) + + +def _split_paragraphs(text: str) -> list[str]: + """Split text by blank lines, discard empty paragraphs.""" + paragraphs = re.split(r"\n\s*\n", text) + return [p.strip() for p in paragraphs if p.strip()] + + +def _build_file_messages(intro: str, body: str, file_end: str, mtime_ms: int) -> list[ImportMessage]: + paragraphs = _split_paragraphs(body) + messages = [ + ImportMessage(role="user", content=intro, timestamp=mtime_ms, sender_id="user"), + ] + for i, para in enumerate(paragraphs): + messages.append( + ImportMessage( + role="user", + content=para, + timestamp=mtime_ms + i + 1, + sender_id="user", + ) + ) + messages.append( + ImportMessage( + role="user", + content=file_end, + timestamp=mtime_ms + len(paragraphs) + 1, + sender_id="user", + ) + ) + return messages + + +def _memory_files_sorted(paths: tuple[Path, ...]) -> list[Path]: + """Sort memory files with MEMORY.md first (index), rest alphabetical.""" + index = [p for p in paths if p.name.upper() == "MEMORY.MD"] + rest = sorted(p for p in paths if p.name.upper() != "MEMORY.MD") + return index + rest + + +# --------------------------------------------------------------------------- +# Scanner +# --------------------------------------------------------------------------- + + +class ClaudeCodeScanner: + """Discovers and reads Claude Code local data for cold-start import.""" + + platform = Platform.CLAUDE_CODE + + def __init__(self, claude_dir: Path | None = None) -> None: + base = claude_dir or (Path.home() / ".claude") + self._claude_dir = base + self._projects_dir = base / "projects" + + async def scan(self) -> list[ScanResult]: + return await asyncio.to_thread(self._scan_sync) + + async def read(self, result: ScanResult) -> ImportSession: + if result.kind == SourceKind.CONVERSATION: + return await asyncio.to_thread(self._read_conversation, result) + return await asyncio.to_thread(self._read_memory, result) + + # -- scan --------------------------------------------------------------- + + def _scan_sync(self) -> list[ScanResult]: + results: list[ScanResult] = [] + self._scan_global_md(results) + self._scan_projects(results) + return results + + def _scan_global_md(self, out: list[ScanResult]) -> None: + gmd = self._claude_dir / "CLAUDE.md" + if not gmd.is_file(): + return + try: + st = gmd.stat() + except OSError: + return + out.append( + ScanResult( + source_key="global-claude-md", + platform=Platform.CLAUDE_CODE, + kind=SourceKind.MEMORY_FILE, + file_paths=(gmd,), + estimated_size=st.st_size, + mtime=st.st_mtime, + ) + ) + + def _scan_projects(self, out: list[ScanResult]) -> None: + if not self._projects_dir.is_dir(): + return + now = time.time() + try: + proj_dirs = sorted(self._projects_dir.iterdir()) + except OSError: + return + for proj in proj_dirs: + if not proj.is_dir(): + continue + self._scan_project_memory(proj, out) + self._scan_project_sessions(proj, out, now) + + def _scan_project_memory(self, proj: Path, out: list[ScanResult]) -> None: + mem_dir = proj / "memory" + if not mem_dir.is_dir(): + return + md_files: list[Path] = [] + total_size = 0 + max_mtime = 0.0 + try: + for f in sorted(mem_dir.iterdir()): + if not f.is_file() or f.suffix.lower() != ".md": + continue + try: + st = f.stat() + except OSError: + continue + if st.st_size > _MAX_MEMORY_FILE_BYTES: + logger.info("Skipping oversized memory file: {} ({} bytes)", f, st.st_size) + continue + md_files.append(f) + total_size += st.st_size + max_mtime = max(max_mtime, st.st_mtime) + except OSError: + return + if not md_files: + return + out.append( + ScanResult( + source_key=f"{proj.name}-memory", + platform=Platform.CLAUDE_CODE, + kind=SourceKind.MEMORY_FILE, + file_paths=tuple(md_files), + estimated_size=total_size, + mtime=max_mtime, + ) + ) + + def _scan_project_sessions(self, proj: Path, out: list[ScanResult], now: float) -> None: + try: + entries = sorted(proj.iterdir()) + except OSError: + return + for f in entries: + if not f.is_file() or f.suffix.lower() != ".jsonl": + continue + try: + st = f.stat() + except OSError: + continue + if now - st.st_mtime < _ACTIVE_THRESHOLD_S: + logger.debug("Skipping active session: {}", f.name) + continue + out.append( + ScanResult( + source_key=f.stem, + platform=Platform.CLAUDE_CODE, + kind=SourceKind.CONVERSATION, + file_paths=(f,), + estimated_size=st.st_size, + mtime=st.st_mtime, + ) + ) + + # -- read: conversation ------------------------------------------------- + + def _read_conversation(self, result: ScanResult) -> ImportSession: + path = result.file_paths[0] + + messages: list[ImportMessage] = [] + with open(path, encoding="utf-8", errors="replace") as fh: + for line_num, raw_line in enumerate(fh, start=1): + raw_line = raw_line.strip() + if not raw_line: + continue + try: + ev = json.loads(raw_line) + except json.JSONDecodeError: + logger.debug("Bad JSON at {}:{}, skipped", path.name, line_num) + continue + self._extract_event(ev, messages) + + return ImportSession( + session_id=f"import-{_APP_ID}-{result.source_key}", + messages=tuple(messages), + ) + + def _extract_event( + self, + ev: dict[str, Any], + out: list[ImportMessage], + ) -> None: + if ev.get("isMeta") is True: + return + if ev.get("isCompactSummary") is True: + return + if ev.get("isApiErrorMessage") is True: + return + + msg = ev.get("message") + if not isinstance(msg, dict): + return + role = msg.get("role") + if role not in ("user", "assistant"): + return + + content = msg.get("content") + if content is None: + return + + ts = parse_iso_ts_ms(ev.get("timestamp")) + if ts is None: + return + sender = "user" if role == "user" else "assistant" + + if isinstance(content, str): + text = content.strip() + if text: + out.append( + ImportMessage( + role=role, + content=_truncate(text), + timestamp=ts, + sender_id=sender, + ) + ) + return + + if not isinstance(content, list): + return + + if role == "user": + for tr in _tool_results_from_content(content, ts, sender): + out.append(tr) + text = _text_from_content(content) + if text: + out.append(ImportMessage(role="user", content=_truncate(text), timestamp=ts, sender_id=sender)) + else: + text = _text_from_content(content) + tool_calls = _tool_calls_from_content(content, ts) + if text or tool_calls: + out.append( + ImportMessage( + role="assistant", + content=_truncate(text), + timestamp=ts, + sender_id=sender, + tool_calls=tool_calls, + ) + ) + + # -- read: memory ------------------------------------------------------- + + def _read_memory(self, result: ScanResult) -> ImportSession: + if result.source_key == "global-claude-md": + return self._read_global_md(result) + return self._read_project_memory(result) + + def _read_global_md(self, result: ScanResult) -> ImportSession: + path = result.file_paths[0] + text = path.read_text(encoding="utf-8", errors="replace") + _, body = parse_frontmatter(text) + + if not body.strip(): + return ImportSession(session_id=f"import-{_APP_ID}-global") + + mtime_ms = int(result.mtime * 1000) + cjk = is_cjk(body) + intro = _pick(_GLOBAL_INTRO, cjk) + file_end = _make_file_end("CLAUDE.md", cjk) + file_msgs = _build_file_messages(intro, body, file_end, mtime_ms) + + return ImportSession( + session_id=f"import-{_APP_ID}-global", + messages=tuple(file_msgs), + ) + + def _read_project_memory(self, result: ScanResult) -> ImportSession: + proj_name = result.source_key.removesuffix("-memory") + base_mtime_ms = int(result.mtime * 1000) + + paths = _memory_files_sorted(result.file_paths) + + parsed: list[tuple[Path, dict[str, Any], str]] = [] + for p in paths: + try: + text = p.read_text(encoding="utf-8", errors="replace") + except OSError: + logger.warning("Cannot read memory file: {}", p) + continue + fm, body = parse_frontmatter(text) + if body.strip(): + parsed.append((p, fm, body)) + + first_body = parsed[0][2] if parsed else "" + cjk = is_cjk(first_body) + + preamble_tpl = _pick(_SESSION_PREAMBLE, cjk) + epilogue_tpl = _pick(_SESSION_EPILOGUE, cjk) + readable_count = len(parsed) + + messages: list[ImportMessage] = [ + ImportMessage( + role="user", + content=preamble_tpl.format(proj=proj_name, count=readable_count), + timestamp=base_mtime_ms, + sender_id="user", + ), + ] + + ts_offset = 1 + for path, fm, body in parsed: + try: + file_mtime_ms = int(path.stat().st_mtime * 1000) + except OSError: + file_mtime_ms = base_mtime_ms + + file_cjk = is_cjk(body) + intro = _make_intro(path.name, fm, file_cjk) + file_end = _make_file_end(path.name, file_cjk) + file_msgs = _build_file_messages(intro, body, file_end, file_mtime_ms) + messages.extend(file_msgs) + ts_offset += len(file_msgs) + + messages.append( + ImportMessage( + role="user", + content=epilogue_tpl.format(proj=proj_name, count=readable_count), + timestamp=base_mtime_ms + ts_offset, + sender_id="user", + ), + ) + + return ImportSession( + session_id=f"import-{_APP_ID}-mem-{proj_name}", + messages=tuple(messages), + ) + + +__all__ = ["ClaudeCodeScanner"] diff --git a/raven/importer/state.py b/raven/importer/state.py new file mode 100644 index 0000000..a5af39e --- /dev/null +++ b/raven/importer/state.py @@ -0,0 +1,132 @@ +"""Idempotent state tracker for cold-start import.""" + +from __future__ import annotations + +import json +import os +import time +from collections import Counter +from pathlib import Path +from typing import Any + +from loguru import logger + +from raven.utils.atomic_io import atomic_replace + + +def _default_state_path() -> Path: + from raven.config.paths import get_data_dir + + return get_data_dir() / "import_state.json" + + +class ImportState: + """Tracks which sources have been imported to enable resume. + + Storage layout (``~/.raven/import_state.json``, configurable):: + + { + "meta": {"total": 42}, + "entries": { + "claude_code:proj-memory": {"status": "submitted", ...}, + ... + } + } + + The state dict is cached in memory after the first read. Mutations + update the cache and flush to disk atomically via + :func:`raven.utils.atomic_io.atomic_replace`. + """ + + def __init__(self, path: Path | None = None) -> None: + self._path = path or _default_state_path() + self._cache: dict[str, Any] | None = None + + @property + def cancel_path(self) -> Path: + return self._path.parent / "import_cancel" + + def is_submitted(self, platform: str, source_key: str) -> bool: + entry = self._entries().get(f"{platform}:{source_key}") + return entry is not None and entry.get("status") == "submitted" + + def mark_submitted(self, platform: str, source_key: str) -> None: + self._mark(platform, source_key, "submitted") + + def mark_failed(self, platform: str, source_key: str, error: str) -> None: + self._mark(platform, source_key, "failed", error=error) + + def set_total(self, total: int) -> None: + """Record the total number of importable units from a scan.""" + data = self._ensure_loaded() + data.setdefault("entries", {}) + data.setdefault("meta", {})["total"] = total + self._flush() + + def get_summary(self) -> dict[str, int]: + """Return ``{"total", "submitted", "failed"}`` counts.""" + entries = self._entries() + counts = Counter(v.get("status") for v in entries.values()) + meta = self._ensure_loaded().get("meta", {}) + return { + "total": meta.get("total", len(entries)), + "submitted": counts.get("submitted", 0), + "failed": counts.get("failed", 0), + } + + def get_progress(self) -> dict[str, Any]: + return { + "meta": dict(self._ensure_loaded().get("meta", {})), + "entries": dict(self._entries()), + } + + # -- internals ---------------------------------------------------------- + + def _mark( + self, + platform: str, + source_key: str, + status: str, + *, + error: str | None = None, + ) -> None: + self._entries()[f"{platform}:{source_key}"] = { + "status": status, + "timestamp": time.time(), + "error": error, + } + self._flush() + + def _entries(self) -> dict[str, dict[str, Any]]: + return self._ensure_loaded().setdefault("entries", {}) + + def _ensure_loaded(self) -> dict[str, Any]: + if self._cache is None: + self._cache = self._read_from_disk() + return self._cache + + def _read_from_disk(self) -> dict[str, Any]: + if not self._path.exists(): + return {} + try: + raw = json.loads(self._path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, ValueError): + backup = self._path.with_suffix(".json.corrupt") + logger.warning( + "Corrupt import state at {} -- backing up to {}", + self._path, + backup, + ) + os.replace(self._path, backup) + return {} + if "entries" in raw: + return raw + # Migrate flat layout from earlier drafts. + return {"entries": raw} + + def _flush(self) -> None: + data = self._ensure_loaded() + atomic_replace(self._path, json.dumps(data, indent=2)) + + +__all__ = ["ImportState"] diff --git a/raven/importer/types.py b/raven/importer/types.py new file mode 100644 index 0000000..b1943bf --- /dev/null +++ b/raven/importer/types.py @@ -0,0 +1,102 @@ +"""Cold-start import data types and Scanner protocol.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import Any, Protocol, runtime_checkable + + +class Platform(StrEnum): + """Supported source platforms for cold-start import.""" + + CLAUDE_CODE = "claude_code" + CODEX = "codex" + KIMICODE = "kimicode" + HERMES = "hermes" + OPENCLAW = "openclaw" + + +class SourceKind(StrEnum): + """What kind of data a scan result represents.""" + + MEMORY_FILE = "memory_file" + CONVERSATION = "conversation" + + +class Tier(StrEnum): + """User-facing import scope choice. + + MEMORY_FILES imports only memory/config files (fast). + FULL imports memory files plus conversation history (slow). + """ + + MEMORY_FILES = "memory_files" + FULL = "full" + + +@dataclass(frozen=True) +class ImportMessage: + """One message in a cold-start import session.""" + + role: str + content: str + timestamp: int + sender_id: str + tool_calls: tuple[dict[str, Any], ...] | None = None + tool_call_id: str | None = None + + +@dataclass(frozen=True) +class ImportSession: + """A complete importable unit ready for store().""" + + session_id: str + messages: tuple[ImportMessage, ...] = () + + +@dataclass(frozen=True) +class ScanResult: + """One importable unit discovered by a Scanner.""" + + source_key: str + platform: Platform + kind: SourceKind + file_paths: tuple[Path, ...] + estimated_size: int + mtime: float + + +@runtime_checkable +class Scanner(Protocol): + """Discovers and reads importable units for one platform.""" + + platform: Platform + + async def scan(self) -> list[ScanResult]: + """Discover all importable units -- no tier filtering.""" + ... + + async def read(self, result: ScanResult) -> ImportSession: + """Load one discovered unit into an ImportSession.""" + ... + + +def filter_by_tier(results: list[ScanResult], tier: Tier) -> list[ScanResult]: + """Filter scan results by the user's chosen import tier.""" + if tier == Tier.FULL: + return results + return [r for r in results if r.kind == SourceKind.MEMORY_FILE] + + +__all__ = [ + "ImportMessage", + "ImportSession", + "Platform", + "ScanResult", + "Scanner", + "SourceKind", + "Tier", + "filter_by_tier", +] diff --git a/raven/memory_engine/backend.py b/raven/memory_engine/backend.py index c769b96..622781e 100644 --- a/raven/memory_engine/backend.py +++ b/raven/memory_engine/backend.py @@ -127,6 +127,8 @@ async def store( self, session_id: str, messages: list[dict[str, Any]], + *, + metadata: dict[str, Any] | None = None, ) -> None: """Persist a session slice. @@ -136,6 +138,12 @@ async def store( that want to chunk / deduplicate / extract are free to; the Protocol is fire-and-forget per call. + ``metadata`` is an optional dict for caller-supplied context + that does not fit the message list. Callers may pass + backend-specific fields such as ``app_id``, ``project_id``, + or ``is_final``; normal AgentLoop turns leave it ``None``. + Backends that do not consume metadata ignore it silently. + Raises on transport / auth errors so AgentLoop can surface them; the host does **not** silently swallow store failures. """ diff --git a/raven/memory_engine/contract_test.py b/raven/memory_engine/contract_test.py index dd69a42..76c8f7d 100644 --- a/raven/memory_engine/contract_test.py +++ b/raven/memory_engine/contract_test.py @@ -103,6 +103,14 @@ async def test_recall_after_store_does_not_raise(self, backend) -> None: ) assert isinstance(hits, list) + async def test_store_accepts_metadata(self, backend) -> None: + """``store`` with metadata must not raise.""" + await backend.store( + "contract-metadata", + [{"role": "user", "content": "test"}], + metadata={"app_id": "test", "project_id": "test", "is_final": True}, + ) + async def test_feedback_accepts_arbitrary_signals(self, backend) -> None: """No-op feedback is valid; any dict must be tolerated.""" await backend.feedback({"unknown_signal": "should not crash"}) diff --git a/raven/plugin/memory/everos/_server.py b/raven/plugin/memory/everos/_server.py new file mode 100644 index 0000000..eee63b6 --- /dev/null +++ b/raven/plugin/memory/everos/_server.py @@ -0,0 +1,97 @@ +"""EverOS server lifecycle manager: health probe + auto-start.""" + +from __future__ import annotations + +import asyncio +import shutil +import subprocess +from pathlib import Path +from urllib.parse import urlparse + +from loguru import logger + +from raven.config.paths import get_data_dir, get_logs_dir +from raven.utils.portable_lock import LockTimeoutError, file_lock + +_POLL_INTERVAL = 0.5 + + +def _extract_port(base_url: str) -> str: + parsed = urlparse(base_url) + return str(parsed.port or 80) + + +def _probe_health(base_url: str) -> bool: + import httpx + + try: + r = httpx.get(f"{base_url}/health", timeout=2.0) + return r.status_code == 200 + except httpx.ConnectError: + return False + except Exception: + return False + + +def _lock_path() -> Path: + return get_data_dir() / "everos-server.lock" + + +def _start_server_if_unlocked(port: str) -> bool: + """Try to acquire the startup lock and launch the server. + + Returns True if this process launched the server, False if the lock + was already held (another process is spawning). Uses the cross- + platform ``portable_lock`` so Windows does not crash on import. + """ + everos = shutil.which("everos") + if not everos: + raise RuntimeError("everos not found. Please install the everos CLI.") + + try: + with file_lock(_lock_path(), blocking=False): + log_path = get_logs_dir() / "everos-server.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + with open(log_path, "a") as log_file: + subprocess.Popen( + [everos, "server", "start", "--port", port], + stdout=log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + logger.info("started everos server on port {} (log: {})", port, log_path) + return True + except LockTimeoutError: + logger.debug("everos server startup lock held by another process; skipping spawn") + return False + + +async def ensure_everos_server( + base_url: str = "http://localhost:18791", + *, + timeout: float = 30.0, +) -> None: + if await asyncio.to_thread(_probe_health, base_url): + logger.info("everos server already running at {}", base_url) + return + + port = _extract_port(base_url) + await asyncio.to_thread(_start_server_if_unlocked, port) + + elapsed = 0.0 + while elapsed < timeout: + await asyncio.sleep(_POLL_INTERVAL) + elapsed += _POLL_INTERVAL + if await asyncio.to_thread(_probe_health, base_url): + logger.info("everos server ready at {}", base_url) + return + + raise RuntimeError( + f"EverOS server failed to start within {timeout}s at {base_url}. " + f"Check: (1) everos is installed (`uv run everos --help`), " + f"(2) port {port} is not occupied, " + f"(3) logs at {get_logs_dir() / 'everos-server.log'}" + ) + + +__all__ = ["ensure_everos_server"] diff --git a/raven/plugin/memory/everos/backend.py b/raven/plugin/memory/everos/backend.py index 6b3f409..ef2f4d4 100644 --- a/raven/plugin/memory/everos/backend.py +++ b/raven/plugin/memory/everos/backend.py @@ -1,23 +1,13 @@ -"""EverosBackend — EM-2 embedded mode (with EM-3 HTTP slot reserved). +"""EverosBackend — HTTP-only memory backend. -The backend is the host's :class:`MemoryBackend` implementation. Three -operating modes: - -- **embedded** (EM-2, this PR): delegate to ``everos.service`` in - the same process. The adapter build (the everos/lancedb import) is - deferred to ``start()`` to keep construction off the render path; if - ``everos`` is not installed (or fails to import — version skew, missing - native deps, etc.), the backend degrades to a :class:`_NoOpAdapter`. -- **http** (EM-3, next PR): HTTP client over EverOS's - ``POST /api/v1/memory/{search,add,...}``. Currently shadowed by the - same no-op adapter so wiring code can already select the mode - without breaking. +The backend is the host's :class:`MemoryBackend` implementation, +delegating to a running EverOS server over HTTP +(``POST /api/v1/memory/{search,add,...}``). Constructor accepts an explicit ``adapter`` so tests can inject a fake without monkeypatching module-level imports. Production wiring -goes through :func:`make_backend` → ``EverosBackend(ctx)`` → -``_try_make_real_adapter`` which is the only code path that touches -``everos.service``. +goes through :func:`make_backend` -> ``EverosBackend(ctx)`` -> +``_make_http_adapter``. Three architectural invariants worth re-stating: @@ -36,7 +26,6 @@ from __future__ import annotations -import asyncio import logging import time from types import SimpleNamespace @@ -51,11 +40,6 @@ _OwnerType = Literal["user", "agent"] -# Documented operating modes (mirrors ``config_schema.mode`` in the -# plugin manifest). A ``mode`` outside this set is a config typo, not a -# request for a new adapter — see ``EverosBackend._validate_config``. -_VALID_MODES: tuple[str, ...] = ("embedded", "http") - _DEFAULT_AGENT_ID: str = "default" _DEFAULT_USER_ID: str = "default" @@ -72,11 +56,9 @@ class _Adapter(Protocol): Two production implementations: - - :class:`_RealEverosAdapter` — lazy-imports ``everos.service`` - and calls in-process. + - :class:`_HttpEverosAdapter` — HTTP client over EverOS's REST API. - :class:`_NoOpAdapter` — returns ``None`` / swallows writes. - Used when everos can't be imported, when ``mode != "embedded"`` - until EM-3 lands, and by tests that don't care about everos. + Used by tests that don't care about everos. """ async def search( @@ -94,6 +76,8 @@ async def memorize( payload_messages: list[dict[str, Any]], *, is_final: bool = False, + app_id: str | None = None, + project_id: str | None = None, ) -> None: ... @@ -108,239 +92,8 @@ async def memorize(self, *a: Any, **kw: Any) -> None: return None -class _RealEverosAdapter: - """In-process delegation to ``everos.service``. The imports happen - in ``__init__`` so a missing / broken everos fails loudly at - construction time rather than mysteriously at first ``recall``.""" - - def __init__(self) -> None: - from everos.config import load_settings - from everos.memory.search.dto import SearchMethod, SearchRequest - from everos.service.memorize import memorize as _memorize - from everos.service.search import search as _search - - self._SearchRequest = SearchRequest - self._SearchMethod = SearchMethod - self._search_fn = _search - self._memorize_fn = _memorize - # Agent-track HYBRID routes skills through everos's cross-encoder - # lane, which everos refuses (RuntimeError in _validate_components) - # when no [rerank] provider is configured. Mirror everos's own - # "configured" test (model + base_url) so we can degrade instead - # of letting that hard error surface. - cfg = load_settings().rerank - self._rerank_configured = bool(cfg.model and cfg.base_url) - self._degrade_logged = False - - async def search( - self, - *, - user_id: str | None, - agent_id: str | None, - query: str, - top_k: int, - ) -> Any: - # everos 1.0.0's SearchRequest takes user_id XOR agent_id (the - # owner_id / owner_type pair are read-only derived properties); - # the backend has already resolved exactly one of these. - method = self._SearchMethod.HYBRID - # Agent-track HYBRID needs a rerank provider; when none is - # configured, degrade to VECTOR (embedding-ranked, single-route, - # no cross-encoder) so skills still surface rather than erroring. - # User-track HYBRID never touches the reranker, so it is left as-is. - if agent_id is not None and not self._rerank_configured: - method = self._SearchMethod.VECTOR - if not self._degrade_logged: - logger.warning( - "rerank not configured; agent-track recall degrades " - "HYBRID -> VECTOR (no cross-encoder rerank). Configure " - "[rerank] (model + base_url) in everos settings to " - "enable skill cross-encoder ranking.", - ) - self._degrade_logged = True - req = self._SearchRequest( - user_id=user_id, - agent_id=agent_id, - query=query, - top_k=top_k, - method=method, - ) - resp = await self._search_fn(req) - return resp.data - - async def memorize( - self, - session_id: str, - payload_messages: list[dict[str, Any]], - *, - is_final: bool = False, - ) -> None: - await self._memorize_fn( - {"session_id": session_id, "messages": payload_messages}, - is_final=is_final, - ) - - -def _try_make_real_adapter() -> _Adapter: - """Return a real adapter if everos imports cleanly; otherwise a - no-op adapter. Failure is logged at WARNING level so a misconfigured - deploy is visible without the host crashing.""" - # everos hard-imports the POSIX ``fcntl`` module at import time. On Windows - # that raised ModuleNotFoundError and the whole memory backend silently - # degraded to a no-op (mis-logged as "everos not installed"). Install a - # Windows fcntl shim first so the bundled backend actually loads. - from raven.utils.win_fcntl_shim import install as _install_fcntl_shim - - _install_fcntl_shim() - try: - return _RealEverosAdapter() - except ModuleNotFoundError as e: - logger.warning( - "everos not installed (%s); EverosBackend embedded mode " - "will degrade to no-op until the package is installed.", - e, - ) - return _NoOpAdapter() - except Exception as e: - # Distinct from "not installed": the package imported but a - # symbol/submodule failed to resolve (version skew, rename, etc.). - # Surfaced louder so a real wiring bug isn't mistaken for an - # absent optional dependency. - logger.warning( - "everos present but failed to initialize (%s); EverosBackend " - "embedded mode will degrade to no-op. This is likely a " - "version mismatch or wiring bug, not a missing package.", - e, - ) - return _NoOpAdapter() - - -# --------------------------------------------------------------------------- -# Embedded everos runtime — process-shared, refcounted lifespan -# --------------------------------------------------------------------------- -# -# everos creates its schema (sqlite tables, lancedb indexes) and its OME -# extraction engine in the FastAPI app *lifespan*, not on first service -# call — so embedded mode must drive that lifespan or store()/recall() -# hit "no such table: unprocessed_buffer". everos's engine / stores are -# process-global singletons, so the lifespan is entered once per process -# and shared by every embedded backend, refcounted so the last stop() -# tears it down. - -_embedded_lifespan_cm: Any = None -_embedded_lifespan_refs: int = 0 - - -async def _migrate_lancedb_schemas( - log: logging.Logger, - *, - _schemas: Any = None, - _get_connection: Any = None, - _get_table: Any = None, -) -> bool: - """Add missing columns to existing LanceDB tables for forward compatibility. - - Returns True if at least one column was added (caller should retry - the lifespan), False when nothing needed migration. - - The underscore-prefixed kwargs exist solely for test injection; - production callers never pass them. - """ - # Deferred: optional dependency — everos may not be installed. - import pyarrow as pa - - if _schemas is None: - from everos.infra.persistence.lancedb import ( - _BUSINESS_SCHEMAS, - get_connection, - get_table, - ) - - _schemas = _BUSINESS_SCHEMAS - _get_connection = get_connection - _get_table = get_table - - migrated = False - await _get_connection() - for schema in _schemas: - table = await _get_table(schema.TABLE_NAME, schema) - arrow_schema = await table.schema() - actual = set(arrow_schema.names) - expected = set(schema.model_fields.keys()) - missing = expected - actual - if not missing: - continue - fields = [pa.field(col, pa.utf8(), nullable=True) for col in sorted(missing)] - await table.add_columns(pa.schema(fields)) - log.info( - "EverosBackend: migrated LanceDB table %r — added columns %s", - schema.TABLE_NAME, - sorted(missing), - ) - migrated = True - return migrated - - -async def _acquire_embedded_everos(log: logging.Logger) -> None: - """Enter the shared everos app lifespan (idempotent + refcounted).""" - global _embedded_lifespan_cm, _embedded_lifespan_refs - _embedded_lifespan_refs += 1 - if _embedded_lifespan_cm is not None: - return - try: - from everos.entrypoints.api.app import create_app - - app = create_app() - cm = app.router.lifespan_context(app) - await cm.__aenter__() - _embedded_lifespan_cm = cm - log.info("EverosBackend: embedded everos runtime started") - except Exception as e: - # Deferred: optional dependency — everos may not be installed. - schema_mismatch_cls: type | None = None - try: - from everos.infra.persistence.lancedb import LanceDBSchemaMismatchError - - schema_mismatch_cls = LanceDBSchemaMismatchError - except ImportError: - pass - if schema_mismatch_cls is not None and isinstance(e, schema_mismatch_cls): - log.warning("EverosBackend: LanceDB schema drift detected, attempting auto-migration …") - try: - if await _migrate_lancedb_schemas(log): - app = create_app() - cm = app.router.lifespan_context(app) - await cm.__aenter__() - _embedded_lifespan_cm = cm - log.info("EverosBackend: embedded everos runtime started (after schema migration)") - return - except Exception as retry_err: - log.warning( - "EverosBackend: auto-migration failed (%s); falling back to degraded mode.", - retry_err, - ) - log.warning( - "EverosBackend: embedded everos init failed (%s); store / recall will degrade until it is available.", - e, - ) - - -async def _release_embedded_everos(log: logging.Logger) -> None: - """Release one ref; tear the lifespan down when the last one drops.""" - global _embedded_lifespan_cm, _embedded_lifespan_refs - _embedded_lifespan_refs = max(0, _embedded_lifespan_refs - 1) - if _embedded_lifespan_refs > 0 or _embedded_lifespan_cm is None: - return - cm = _embedded_lifespan_cm - _embedded_lifespan_cm = None - try: - await cm.__aexit__(None, None, None) - except Exception as e: - log.warning("EverosBackend: embedded everos teardown failed (%s)", e) - - # --------------------------------------------------------------------------- -# HTTP adapter — EM-3 +# HTTP adapter # --------------------------------------------------------------------------- @@ -362,8 +115,8 @@ def _jsonify(obj: Any) -> Any: return obj -# Default timeout — HTTP mode is per-turn, so we keep it tight. -_DEFAULT_HTTP_TIMEOUT_S: float = 10.0 +_DEFAULT_HTTP_TIMEOUT_S: float = 60.0 +_MEMORIZE_TIMEOUT_S: float = 360.0 class _HttpEverosAdapter: @@ -438,18 +191,32 @@ async def memorize( payload_messages: list[dict[str, Any]], *, is_final: bool = False, + app_id: str | None = None, + project_id: str | None = None, ) -> None: - body = {"session_id": session_id, "messages": payload_messages} + body: dict[str, Any] = { + "session_id": session_id, + "messages": payload_messages, + } + if app_id is not None: + body["app_id"] = app_id + if project_id is not None: + body["project_id"] = project_id url = f"{self._base_url}/api/v1/memory/add" - r = await self._client.post(url, json=body, headers=self._headers()) + r = await self._client.post(url, json=body, headers=self._headers(), timeout=_MEMORIZE_TIMEOUT_S) r.raise_for_status() if is_final: - # Promote accumulated raw messages to episodes / cases / skills. + flush_body: dict[str, Any] = {"session_id": session_id} + if app_id is not None: + flush_body["app_id"] = app_id + if project_id is not None: + flush_body["project_id"] = project_id flush_url = f"{self._base_url}/api/v1/memory/flush" fr = await self._client.post( flush_url, - json={"session_id": session_id}, + json=flush_body, headers=self._headers(), + timeout=_MEMORIZE_TIMEOUT_S, ) fr.raise_for_status() @@ -471,57 +238,26 @@ def __init__( self._config = ctx.config self._services = ctx.services self._logger = ctx.logger - self._mode = self._config.get("mode", "embedded") self._agent_id: str = self._config.get("agent_id") or _DEFAULT_AGENT_ID self._user_id: str = self._config.get("user_id") or _DEFAULT_USER_ID - # everos accumulates raw turns and only extracts episodes / cases / - # skills on a boundary flush. Flush every N store() calls so short - # sessions still build memory (mirrors the EverMe plugin's - # flush_every_turns=1 default). 0 disables flushing entirely. self._flush_every_turns: int = int( self._config.get("flush_every_turns", 1), ) self._turn_counts: dict[str, int] = {} self._feedback_noop_logged = False - self._embedded_started = False - # Adapter selection. Tests inject explicit adapters; production - # wires through one of the per-mode factories below. if adapter is not None: self._adapter: _Adapter | None = adapter else: - self._validate_config() - if self._mode == "embedded": - # Defer the heavy everos/lancedb import (~2-3s) to start() so it - # runs off the render-blocking path. recall/store degrade to - # empty until the adapter is built. - self._adapter = None - else: # "http" — _validate_config rejected anything else - self._adapter = self._make_http_adapter() - - def _validate_config(self) -> None: - """Fail fast on a misconfigured plugin config. - - A typo'd ``mode`` (e.g. ``"embeded"``) used to fall through to a - silent no-op adapter, leaving the agent running with memory - quietly disabled. Validating the documented enum here surfaces - the mistake at construction — the registry logs the raised error - instead of degrading without a trace. - """ - if self._mode not in _VALID_MODES: - raise ValueError( - f"EverosBackend: invalid mode {self._mode!r}; expected one of {', '.join(_VALID_MODES)}", - ) + self._adapter = self._make_http_adapter() def _make_http_adapter(self) -> _Adapter: """Construct an :class:`_HttpEverosAdapter` from plugin config. Pulls ``base_url`` / ``api_key`` / ``timeout_s`` out of - ``ctx.config`` with documented defaults. ``base_url`` defaults - to the EverOS dev port (1995) so a local-dev workflow with the - server running on the same host needs no extra config. + ``ctx.config`` with documented defaults. """ - base_url = self._config.get("base_url") or "http://localhost:1995" + base_url = self._config.get("base_url") or "http://localhost:18791" api_key = self._config.get("api_key") timeout_s = float( self._config.get("timeout_s", _DEFAULT_HTTP_TIMEOUT_S), @@ -535,31 +271,38 @@ def _make_http_adapter(self) -> _Adapter: # ── Lifecycle ─────────────────────────────────────────────────── async def start(self) -> None: - # Build the embedded adapter now (deferred from __init__). The everos / - # lancedb import is ~2-3s of sync CPU, so run it in a thread to keep it - # off the event loop. - if self._mode == "embedded" and self._adapter is None: - self._adapter = await asyncio.to_thread(_try_make_real_adapter) self._logger.info( - "EverosBackend.start (mode=%s, adapter=%s)", - self._mode, + "EverosBackend.start (adapter=%s)", type(self._adapter).__name__, ) - # Embedded real adapter: bring up the in-process everos runtime - # (schema + OME engine) so store / recall actually work. HTTP and - # no-op adapters need no local everos lifespan. - if isinstance(self._adapter, _RealEverosAdapter): - await _acquire_embedded_everos(self._logger) - self._embedded_started = True + if isinstance(self._adapter, _HttpEverosAdapter): + import sys + + if sys.platform == "win32": + from rich.console import Console + + Console(stderr=True).print( + "[yellow]EverOS memory is not available on native Windows.[/yellow]\n" + "[dim]Run Raven inside WSL for full memory support, " + "or run `raven onboard` to reconfigure.[/dim]" + ) + self._adapter = _NoOpAdapter() + return + + from raven.plugin.memory.everos._server import ensure_everos_server + + base_url = self._config.get("base_url") or "http://localhost:18791" + try: + await ensure_everos_server(base_url) + except Exception as e: + self._logger.error( + "EverosBackend: failed to start EverOS server (%s)", + e, + ) + raise async def stop(self) -> None: self._logger.info("EverosBackend.stop") - if self._embedded_started: - await _release_embedded_everos(self._logger) - self._embedded_started = False - # HTTP adapter owns an httpx client when no client was injected; - # closing it here releases the connection pool. Embedded / - # no-op adapters expose no aclose so getattr returns None. aclose = getattr(self._adapter, "aclose", None) if aclose is not None: try: @@ -621,6 +364,8 @@ async def store( self, session_id: str, messages: list[dict[str, Any]], + *, + metadata: dict[str, Any] | None = None, ) -> None: """Forward a turn's messages to EverOS for indexing. @@ -645,14 +390,21 @@ async def store( if not payload: return if self._adapter is None: - return # adapter still building (start() not finished); drop this turn's store - n = self._turn_counts.get(session_id, 0) + 1 - self._turn_counts[session_id] = n - is_final = self._flush_every_turns > 0 and n % self._flush_every_turns == 0 - try: - await self._adapter.memorize(session_id, payload, is_final=is_final) - except Exception as e: - self._logger.warning("EverosBackend.store failed (%s)", e) + return + if metadata and "is_final" in metadata: + is_final = bool(metadata["is_final"]) + else: + n = self._turn_counts.get(session_id, 0) + 1 + self._turn_counts[session_id] = n + is_final = self._flush_every_turns > 0 and n % self._flush_every_turns == 0 + + await self._adapter.memorize( + session_id, + payload, + is_final=is_final, + app_id=metadata.get("app_id") if metadata else None, + project_id=metadata.get("project_id") if metadata else None, + ) async def feedback(self, signals: dict[str, Any]) -> None: """Deliberate no-op pending an upstream everos feedback sink. @@ -848,4 +600,4 @@ def make_backend(ctx: PluginContext) -> EverosBackend: return EverosBackend(ctx) -__all__ = ["EverosBackend", "_HttpEverosAdapter", "make_backend"] +__all__ = ["EverosBackend", "make_backend"] diff --git a/raven/plugin/memory/everos/raven-plugin.toml b/raven/plugin/memory/everos/raven-plugin.toml index d914824..31ed476 100644 --- a/raven/plugin/memory/everos/raven-plugin.toml +++ b/raven/plugin/memory/everos/raven-plugin.toml @@ -23,10 +23,5 @@ factory = "raven.plugin.memory.everos.tools:make_understand_media_tool" # ``plugins.config["everos-memory"]`` dict to ``make_backend(ctx)`` # verbatim; full schema validation will land alongside EM-2. [plugin.config_schema] -mode = { type = "string", enum = ["embedded", "http"], default = "embedded" } -base_url = { type = "string", default = "http://localhost:1995" } -recall_method = { type = "string", enum = ["keyword", "vector", "hybrid"], default = "hybrid" } -# Pre-configured, stable agent identity: owner of the agent track (cases / -# skills accrue under it across sessions). Bare id, matched verbatim -# against the ``agent_id`` the host passes to recall. -agent_id = { type = "string", default = "agent:default" } +base_url = { type = "string", default = "http://localhost:18791" } +agent_id = { type = "string", default = "agent:default" } diff --git a/raven/proactive_engine/sentinel/predictor/routine_learner.py b/raven/proactive_engine/sentinel/predictor/routine_learner.py index 4667a30..83c0d1e 100644 --- a/raven/proactive_engine/sentinel/predictor/routine_learner.py +++ b/raven/proactive_engine/sentinel/predictor/routine_learner.py @@ -37,6 +37,7 @@ from loguru import logger from raven.proactive_engine.sentinel.types import Routine +from raven.utils.text import CJK_RE # Default half-life for ``learn_with_decay``. 14d → entries 14 days old # count half, 28 days a quarter — fresh habits dominate without erasing @@ -198,7 +199,7 @@ def _extract_keywords(content: str, max_keywords: int = 5) -> list[str]: continue if tok.isdigit(): continue - if len(tok) < 2 and not re.search(r"[一-鿿]", tok): + if len(tok) < 2 and not CJK_RE.search(tok): continue # drop single ASCII chars; keep single CJK kept.append(tok) if not kept: diff --git a/raven/utils/text.py b/raven/utils/text.py new file mode 100644 index 0000000..fc984e9 --- /dev/null +++ b/raven/utils/text.py @@ -0,0 +1,64 @@ +"""Shared text utilities: frontmatter, timestamps, CJK detection.""" + +from __future__ import annotations + +import re +from datetime import datetime, timezone +from typing import Any + +import yaml + +CJK_RE = re.compile(r"[一-鿿]") +_FM_CLOSE_RE = re.compile(r"^---\s*$", re.MULTILINE) + + +def is_cjk(text: str, sample: int = 200) -> bool: + """Return True if *text* (first *sample* chars) contains CJK ideographs.""" + return bool(CJK_RE.search(text[:sample])) + + +def parse_frontmatter(text: str) -> tuple[dict[str, Any], str]: + """Extract YAML frontmatter delimited by ``---`` lines. + + Returns ``(metadata_dict, body_after_frontmatter)``. + Returns ``({}, original_text)`` when no valid frontmatter is found. + """ + if not text.startswith("---"): + return {}, text + m = _FM_CLOSE_RE.search(text, pos=4) + if m is None: + return {}, text + fm_str = text[3 : m.start()].strip() + body = text[m.end() :].lstrip("\n") + try: + fm = yaml.safe_load(fm_str) + return (fm if isinstance(fm, dict) else {}), body + except yaml.YAMLError: + return {}, text + + +def parse_iso_ts_ms(raw: Any) -> int | None: + """Parse a timestamp value to millisecond epoch. + + Accepts ISO 8601 strings, epoch-millisecond ints, and epoch-second + floats. Returns ``None`` on unparseable input. + """ + if isinstance(raw, (int, float)): + return int(raw) if raw > 1e12 else int(raw * 1000) + if not isinstance(raw, str) or not raw: + return None + try: + dt = datetime.fromisoformat(raw) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return int(dt.timestamp() * 1000) + except (ValueError, TypeError): + return None + + +__all__ = [ + "CJK_RE", + "is_cjk", + "parse_frontmatter", + "parse_iso_ts_ms", +] diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ace9a06..4382bec 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -150,27 +150,14 @@ async def everos_env( _reset_everos_singletons(monkeypatch) - # Force a fresh lifespan entry by clearing the refcounted singleton - # in Raven's backend module. Without this, a previous test's - # _embedded_lifespan_cm survives and _acquire skips re-entry — but - # the everos singletons above were already nulled, so the old - # lifespan's OME engine is gone. - import raven.plugin.memory.everos.backend as _be_mod - - monkeypatch.setattr(_be_mod, "_embedded_lifespan_cm", None) - monkeypatch.setattr(_be_mod, "_embedded_lifespan_refs", 0) - - # Bring up the in-process everos runtime via the production path: - # EverosBackend.start() drives the (refcounted, process-shared) everos - # lifespan that creates schema + the OME engine. L2 tests then call - # everos.service directly against this runtime; L3 backends start() - # again and just share the same lifespan (refcount > 1). + # Bring up the everos runtime via the production path: + # EverosBackend.start() ensures the everos server is running. from raven.plugin import PluginContext, ServiceLocator from raven.plugin.memory.everos.backend import EverosBackend be = EverosBackend( PluginContext( - config={"mode": "embedded"}, + config={}, services=ServiceLocator(workspace=tmp_path), ) ) diff --git a/tests/integration/test_everos_backend_e2e.py b/tests/integration/test_everos_backend_e2e.py index b4f0ffd..6d99588 100644 --- a/tests/integration/test_everos_backend_e2e.py +++ b/tests/integration/test_everos_backend_e2e.py @@ -24,7 +24,7 @@ from raven.memory_engine import Memory from raven.plugin import PluginContext, ServiceLocator -from raven.plugin.memory.everos.backend import EverosBackend, _RealEverosAdapter +from raven.plugin.memory.everos.backend import EverosBackend, _HttpEverosAdapter pytestmark = pytest.mark.real_llm @@ -32,13 +32,11 @@ def _backend(tmp_path: Path, *, agent_id: str) -> EverosBackend: be = EverosBackend( PluginContext( - config={"mode": "embedded", "agent_id": agent_id}, + config={"agent_id": agent_id}, services=ServiceLocator(workspace=tmp_path), ) ) - # Embedded mode must resolve the real adapter now that everos is - # installed — if it degraded to no-op the e2e would be meaningless. - assert isinstance(be._adapter, _RealEverosAdapter), "embedded backend did not bind the real everos adapter" + assert isinstance(be._adapter, _HttpEverosAdapter), "backend did not bind the HTTP adapter" return be diff --git a/tests/integration/test_everos_skill_evolution_e2e.py b/tests/integration/test_everos_skill_evolution_e2e.py index 80e2405..208e388 100644 --- a/tests/integration/test_everos_skill_evolution_e2e.py +++ b/tests/integration/test_everos_skill_evolution_e2e.py @@ -30,7 +30,7 @@ from raven.memory_engine import Memory from raven.plugin import PluginContext, ServiceLocator -from raven.plugin.memory.everos.backend import EverosBackend, _RealEverosAdapter +from raven.plugin.memory.everos.backend import EverosBackend, _HttpEverosAdapter from tests.integration.conftest import as_everos_payload pytestmark = pytest.mark.real_llm @@ -58,11 +58,11 @@ def _backend(tmp_path: Path, *, agent_id: str) -> EverosBackend: be = EverosBackend( PluginContext( - config={"mode": "embedded", "agent_id": agent_id}, + config={"agent_id": agent_id}, services=ServiceLocator(workspace=tmp_path), ) ) - assert isinstance(be._adapter, _RealEverosAdapter), "embedded backend did not bind the real everos adapter" + assert isinstance(be._adapter, _HttpEverosAdapter), "backend did not bind the HTTP adapter" return be diff --git a/tests/integration/test_import_e2e.py b/tests/integration/test_import_e2e.py new file mode 100644 index 0000000..5e620ac --- /dev/null +++ b/tests/integration/test_import_e2e.py @@ -0,0 +1,341 @@ +"""L5 -- cold-start import end-to-end: Scanner -> orchestrator -> MemoryBackend. + +Drives the real :class:`ClaudeCodeScanner` against a synthetic ``~/.claude`` +tree and feeds its output through the real :func:`run_import` orchestrator, +recording what would have reached a memory backend via a fake. This proves +the full pipeline wiring (scan -> read -> batch -> store -> state) without +requiring a live EverOS instance or LLM. +""" + +from __future__ import annotations + +import json +import os +import time +from pathlib import Path +from typing import Any + +import pytest + +from raven.importer.orchestrator import ImportSummary, ProgressEvent, run_import +from raven.importer.scanners.claude_code import ClaudeCodeScanner +from raven.importer.state import ImportState +from raven.importer.types import Scanner, ScanResult, SourceKind +from raven.utils.text import parse_iso_ts_ms + +_OLD_MTIME = time.time() - 600 + +# --------------------------------------------------------------------------- +# Fixture data builders +# --------------------------------------------------------------------------- + + +def _write_conversation(path: Path) -> None: + events = [ + { + "type": "user", + "timestamp": "2026-07-15T10:00:00Z", + "message": {"role": "user", "content": "Hello, help me write a function"}, + }, + { + "type": "assistant", + "timestamp": "2026-07-15T10:00:05Z", + "message": { + "role": "assistant", + "content": [ + {"type": "text", "text": "Sure, here's a function:"}, + { + "type": "tool_use", + "id": "toolu_abc", + "name": "write_file", + "input": {"path": "test.py", "content": "def hello(): pass"}, + }, + ], + }, + }, + { + "type": "user", + "timestamp": "2026-07-15T10:00:10Z", + "message": { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_abc", "content": "File written successfully"}, + {"type": "text", "text": "Great, now add a docstring"}, + ], + }, + }, + ] + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + for event in events: + f.write(json.dumps(event) + "\n") + os.utime(path, (_OLD_MTIME, _OLD_MTIME)) + + +def _write_memory_files(project_dir: Path) -> None: + mem_dir = project_dir / "memory" + mem_dir.mkdir(parents=True, exist_ok=True) + + (mem_dir / "MEMORY.md").write_text( + "# Project Memory\n\n- [arch](architecture.md) - Architecture notes\n", + encoding="utf-8", + ) + (mem_dir / "architecture.md").write_text( + "---\nname: architecture\ndescription: System architecture\nmetadata:\n type: reference\n---\n\n" + "The system uses a layered architecture.\n\nEach layer has a single responsibility.\n", + encoding="utf-8", + ) + + +def _write_large_conversation(path: Path, n_events: int = 160) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + for i in range(n_events): + role = "user" if i % 2 == 0 else "assistant" + event = { + "type": role, + "timestamp": f"2026-07-15T10:{i // 60:02d}:{i % 60:02d}Z", + "message": {"role": role, "content": f"Message number {i}"}, + } + f.write(json.dumps(event) + "\n") + os.utime(path, (_OLD_MTIME, _OLD_MTIME)) + + +@pytest.fixture +def claude_home(tmp_path: Path) -> Path: + """Build a fake ~/.claude/ directory with a conversation, memory files, + and a large conversation, all under one project.""" + claude_dir = tmp_path / ".claude" + project_dir = claude_dir / "projects" / "test-project" + + _write_conversation(project_dir / "sess-001.jsonl") + _write_memory_files(project_dir) + _write_large_conversation(project_dir / "sess-large.jsonl") + + return claude_dir + + +@pytest.fixture +def scanner(claude_home: Path) -> ClaudeCodeScanner: + return ClaudeCodeScanner(claude_dir=claude_home) + + +# --------------------------------------------------------------------------- +# Recording backend +# --------------------------------------------------------------------------- + + +class RecordingBackend: + """Fake MemoryBackend that records every store() call verbatim.""" + + def __init__(self) -> None: + self.store_calls: list[dict[str, Any]] = [] + + async def recall( + self, + query: str, + *, + user_id: str | None = None, + agent_id: str | None = None, + top_k: int, + ) -> list[Any]: + return [] + + async def store( + self, + session_id: str, + messages: list[dict[str, Any]], + *, + metadata: dict[str, Any] | None = None, + ) -> None: + self.store_calls.append( + {"session_id": session_id, "messages": list(messages), "metadata": dict(metadata or {})} + ) + + async def feedback(self, signals: dict[str, Any]) -> None: + pass + + async def start(self) -> None: + pass + + async def stop(self) -> None: + pass + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _items_of_kind( + scanner: Scanner, + results: list[ScanResult], + kind: SourceKind, + *, + source_key: str | None = None, +) -> list[tuple[Scanner, ScanResult]]: + matches = [r for r in results if r.kind == kind and (source_key is None or r.source_key == source_key)] + return [(scanner, r) for r in matches] + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_full_pipeline_conversation(scanner: ClaudeCodeScanner, tmp_path: Path) -> None: + """Scanner -> run_import -> verify store calls for a conversation.""" + results = await scanner.scan() + items = _items_of_kind(scanner, results, SourceKind.CONVERSATION, source_key="sess-001") + assert len(items) == 1 + + backend = RecordingBackend() + state = ImportState(path=tmp_path / "state.json") + + summary = await run_import(items, backend, state) + + assert summary == ImportSummary(total=1, submitted=1, skipped=0, failed=0, errors=()) + assert len(backend.store_calls) == 1 + + call = backend.store_calls[0] + assert call["session_id"] == "import-claude_code-sess-001" + assert call["metadata"]["is_final"] is True + + roles = [m["role"] for m in call["messages"]] + assert roles == ["user", "assistant", "tool", "user"] + assert call["messages"][0]["content"] == "Hello, help me write a function" + assert call["messages"][3]["content"] == "Great, now add a docstring" + + +@pytest.mark.asyncio +async def test_full_pipeline_memory_files(scanner: ClaudeCodeScanner, tmp_path: Path) -> None: + """Scanner -> run_import -> verify store calls for memory files.""" + results = await scanner.scan() + items = _items_of_kind(scanner, results, SourceKind.MEMORY_FILE) + assert len(items) == 1 + + backend = RecordingBackend() + state = ImportState(path=tmp_path / "state.json") + + summary = await run_import(items, backend, state) + + assert summary.total == 1 + assert summary.submitted == 1 + assert len(backend.store_calls) == 1 + + call = backend.store_calls[0] + assert call["session_id"] == "import-claude_code-mem-test-project" + assert call["metadata"]["is_final"] is True + + contents = [m["content"] for m in call["messages"]] + assert any("test-project" in c for c in contents) + assert any("architecture.md" in c for c in contents) + assert "The system uses a layered architecture." in contents + assert "Each layer has a single responsibility." in contents + + +@pytest.mark.asyncio +async def test_batching_large_conversation(scanner: ClaudeCodeScanner, tmp_path: Path) -> None: + """160 messages -> multiple store calls with is_final only on the last.""" + results = await scanner.scan() + items = _items_of_kind(scanner, results, SourceKind.CONVERSATION, source_key="sess-large") + assert len(items) == 1 + + backend = RecordingBackend() + state = ImportState(path=tmp_path / "state.json") + + summary = await run_import(items, backend, state) + + assert summary.submitted == 1 + assert len(backend.store_calls) == 2 + + first, second = backend.store_calls + assert len(first["messages"]) == 100 + assert first["metadata"]["is_final"] is False + assert len(second["messages"]) == 60 + assert second["metadata"]["is_final"] is True + + total_messages = len(first["messages"]) + len(second["messages"]) + assert total_messages == 160 + + +@pytest.mark.asyncio +async def test_idempotent_resume(scanner: ClaudeCodeScanner, tmp_path: Path) -> None: + """Run twice -> second run skips all, submitted count = 0.""" + results = await scanner.scan() + items = _items_of_kind(scanner, results, SourceKind.CONVERSATION, source_key="sess-001") + + backend = RecordingBackend() + state = ImportState(path=tmp_path / "state.json") + + first_summary = await run_import(items, backend, state) + assert first_summary.submitted == 1 + assert len(backend.store_calls) == 1 + + second_summary = await run_import(items, backend, state) + assert second_summary.submitted == 0 + assert second_summary.skipped == 1 + assert len(backend.store_calls) == 1 + + +@pytest.mark.asyncio +async def test_progress_callback(scanner: ClaudeCodeScanner, tmp_path: Path) -> None: + """Verify on_progress fires once per item, with correct current/total.""" + results = await scanner.scan() + items = _items_of_kind(scanner, results, SourceKind.CONVERSATION, source_key="sess-001") + items += _items_of_kind(scanner, results, SourceKind.MEMORY_FILE) + assert len(items) == 2 + + backend = RecordingBackend() + state = ImportState(path=tmp_path / "state.json") + events: list[ProgressEvent] = [] + + await run_import(items, backend, state, on_progress=events.append) + + assert len(events) == 2 + assert [e.current for e in events] == [1, 2] + assert all(e.total == 2 for e in events) + assert all(e.status == "submitted" for e in events) + assert all(e.error is None for e in events) + + +@pytest.mark.asyncio +async def test_message_shape(scanner: ClaudeCodeScanner, tmp_path: Path) -> None: + """Verify stored messages carry role, content, sender_id, timestamp, + and tool_calls / tool_call_id exactly where expected.""" + results = await scanner.scan() + items = _items_of_kind(scanner, results, SourceKind.CONVERSATION, source_key="sess-001") + + backend = RecordingBackend() + state = ImportState(path=tmp_path / "state.json") + + await run_import(items, backend, state) + + messages = backend.store_calls[0]["messages"] + assert len(messages) == 4 + + user_msg, assistant_msg, tool_msg, followup_msg = messages + + assert user_msg["role"] == "user" + assert user_msg["content"] == "Hello, help me write a function" + assert "sender_id" not in user_msg + assert user_msg["timestamp"] == parse_iso_ts_ms("2026-07-15T10:00:00Z") + assert "tool_calls" not in user_msg + assert "tool_call_id" not in user_msg + + assert assistant_msg["role"] == "assistant" + assert assistant_msg["content"] == "Sure, here's a function:" + assert "sender_id" not in assistant_msg + assert assistant_msg["timestamp"] == parse_iso_ts_ms("2026-07-15T10:00:05Z") + assert assistant_msg["tool_calls"][0]["id"] == "toolu_abc" + assert assistant_msg["tool_calls"][0]["function"]["name"] == "write_file" + + assert tool_msg["role"] == "tool" + assert tool_msg["content"] == "File written successfully" + assert tool_msg["tool_call_id"] == "toolu_abc" + assert tool_msg["timestamp"] == parse_iso_ts_ms("2026-07-15T10:00:10Z") + + assert followup_msg["role"] == "user" + assert followup_msg["content"] == "Great, now add a docstring" + assert followup_msg["timestamp"] == parse_iso_ts_ms("2026-07-15T10:00:10Z") diff --git a/tests/test_ag1_backend_dispatch.py b/tests/test_ag1_backend_dispatch.py index 902b7a0..53d7737 100644 --- a/tests/test_ag1_backend_dispatch.py +++ b/tests/test_ag1_backend_dispatch.py @@ -51,7 +51,7 @@ async def feedback(self, signals): async def recall(self, query, *, user_id=None, agent_id=None, top_k): return [] - async def store(self, session_id, messages): + async def store(self, session_id, messages, *, metadata=None): self.store_calls.append( { "session_id": session_id, diff --git a/tests/test_agent_loop_everos_pipeline.py b/tests/test_agent_loop_everos_pipeline.py index 4ae6758..660c31b 100644 --- a/tests/test_agent_loop_everos_pipeline.py +++ b/tests/test_agent_loop_everos_pipeline.py @@ -98,7 +98,7 @@ async def recall(self, query, *, user_id=None, agent_id=None, top_k): ] return [] - async def store(self, session_id, messages): + async def store(self, session_id, messages, *, metadata=None): self.store_calls.append({"session_id": session_id, "messages": messages}) if self.store_raises is not None: raise self.store_raises diff --git a/tests/test_cli_import_commands.py b/tests/test_cli_import_commands.py new file mode 100644 index 0000000..d18c716 --- /dev/null +++ b/tests/test_cli_import_commands.py @@ -0,0 +1,218 @@ +"""Tests for raven import CLI commands.""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import AsyncMock, patch + +from typer.testing import CliRunner + +from raven.cli.import_commands import import_app +from raven.importer.orchestrator import ImportSummary +from raven.importer.state import ImportState +from raven.importer.types import Platform, ScanResult, SourceKind + +runner = CliRunner() + + +def _scan_result( + key: str = "k1", + platform: Platform = Platform.CLAUDE_CODE, + kind: SourceKind = SourceKind.CONVERSATION, + size: int = 1000, +) -> ScanResult: + return ScanResult( + source_key=key, + platform=platform, + kind=kind, + file_paths=(Path("/fake"),), + estimated_size=size, + mtime=1000.0, + ) + + +def _make_scan_results() -> list[ScanResult]: + return [ + _scan_result("global-claude-md", kind=SourceKind.MEMORY_FILE, size=2048), + _scan_result("proj-memory", kind=SourceKind.MEMORY_FILE, size=48000), + _scan_result("sess-001", kind=SourceKind.CONVERSATION, size=120000), + ] + + +# --------------------------------------------------------------------------- +# scan +# --------------------------------------------------------------------------- + + +class TestScan: + def test_scan_shows_results(self) -> None: + with patch( + "raven.importer.scanners.scan_all", + new=AsyncMock(return_value=_make_scan_results()), + ): + result = runner.invoke(import_app, ["scan"]) + + assert result.exit_code == 0 + assert "Claude Code" in result.stdout + assert "global-claude-md" in result.stdout + + def test_scan_empty(self) -> None: + with patch( + "raven.importer.scanners.scan_all", + new=AsyncMock(return_value=[]), + ): + result = runner.invoke(import_app, ["scan"]) + + assert result.exit_code == 0 + assert "No importable data found" in result.stdout + + +# --------------------------------------------------------------------------- +# status +# --------------------------------------------------------------------------- + + +class TestStatus: + def test_status_shows_summary(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + state.set_total(10) + state.mark_submitted("claude_code", "a") + state.mark_submitted("claude_code", "b") + state.mark_failed("claude_code", "c", "err") + + with patch("raven.cli.import_commands._default_state", return_value=state): + result = runner.invoke(import_app, ["status"]) + + assert result.exit_code == 0 + assert "10" in result.stdout + assert "2" in result.stdout + + def test_status_json(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + state.set_total(5) + state.mark_submitted("claude_code", "a") + + with patch("raven.cli.import_commands._default_state", return_value=state): + result = runner.invoke(import_app, ["status", "--json"]) + + data = json.loads(result.stdout) + assert data["total"] == 5 + assert data["submitted"] == 1 + + def test_status_no_state(self) -> None: + state = ImportState(path=Path("/nonexistent/state.json")) + + with patch("raven.cli.import_commands._default_state", return_value=state): + result = runner.invoke(import_app, ["status"]) + + assert result.exit_code == 0 + assert "No import in progress" in result.stdout + + +# --------------------------------------------------------------------------- +# run +# --------------------------------------------------------------------------- + + +class TestRun: + def test_run_non_interactive(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + summary = ImportSummary(total=2, submitted=2, skipped=0, failed=0, errors=()) + + with ( + patch( + "raven.importer.scanners.scan_all", + new=AsyncMock(return_value=_make_scan_results()), + ), + patch( + "raven.cli.import_commands._build_and_run", + new=AsyncMock(return_value=summary), + ), + patch("raven.cli.import_commands._default_state", return_value=state), + ): + result = runner.invoke( + import_app, + ["run", "--platform", "claude_code", "--tier", "full", "--yes"], + ) + + assert result.exit_code == 0 + + def test_run_no_backend(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + + with ( + patch( + "raven.importer.scanners.scan_all", + new=AsyncMock(return_value=_make_scan_results()), + ), + patch("raven.cli.import_commands._default_state", return_value=state), + patch( + "raven.cli.import_commands.maybe_build_memory_backend", + return_value=None, + ), + ): + result = runner.invoke( + import_app, + ["run", "--platform", "claude_code", "--tier", "full", "--yes"], + ) + + assert result.exit_code == 1 + + def test_run_no_sources(self) -> None: + with patch( + "raven.importer.scanners.scan_all", + new=AsyncMock(return_value=[]), + ): + result = runner.invoke( + import_app, + ["run", "--platform", "claude_code", "--tier", "full", "--yes"], + ) + + assert result.exit_code == 0 + assert "No importable data found" in result.stdout + + +# --------------------------------------------------------------------------- +# stop +# --------------------------------------------------------------------------- + + +class TestStop: + def test_stop_creates_cancel_file(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + state.set_total(5) + state.mark_submitted("claude_code", "item1") + with patch("raven.cli.import_commands._default_state", return_value=state): + result = runner.invoke(import_app, ["stop"]) + assert result.exit_code == 0 + assert state.cancel_path.exists() + assert "cancel" in result.output.lower() or "Cancel" in result.output + + def test_stop_no_import(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + with patch("raven.cli.import_commands._default_state", return_value=state): + result = runner.invoke(import_app, ["stop"]) + assert result.exit_code == 0 + assert "No import" in result.output + + def test_stop_already_cancelled(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + state.set_total(5) + state.cancel_path.touch() + with patch("raven.cli.import_commands._default_state", return_value=state): + result = runner.invoke(import_app, ["stop"]) + assert result.exit_code == 0 + assert "already" in result.output.lower() + + +class TestStatusCancelled: + def test_status_shows_cancelled(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + state.set_total(5) + state.mark_submitted("claude_code", "item1") + state.cancel_path.touch() + with patch("raven.cli.import_commands._default_state", return_value=state): + result = runner.invoke(import_app, ["status"]) + assert result.exit_code == 0 + assert "Cancelled" in result.output or "cancelled" in result.output diff --git a/tests/test_cli_onboard_commands.py b/tests/test_cli_onboard_commands.py index 60287c2..da689e8 100644 --- a/tests/test_cli_onboard_commands.py +++ b/tests/test_cli_onboard_commands.py @@ -1,4 +1,4 @@ -"""CLI tests for ``raven onboard`` — the three-step wizard. +"""CLI tests for ``raven onboard`` — the five-step wizard. Most tests exercise ``--non-interactive`` so we can drive the wizard deterministically without a real TTY. Interactive paths are covered by @@ -156,6 +156,7 @@ def test_onboard_help_lists_all_flags() -> None: "--skip-channel", "--skip-memory", "--skip-deep-research", + "--skip-import", "--non-interactive", "--yes", "--reset", @@ -241,6 +242,46 @@ def test_onboard_skip_channel_default(tmp_env: Path, stub_verify, stub_step3) -> assert "Skipped via --skip-channel" in r.stdout +def test_onboard_skip_import_default(tmp_env: Path, stub_verify, stub_step3) -> None: + """``--skip-import`` produces the dim skip line in Step 5.""" + r = runner.invoke( + app, + [ + "onboard", + "--non-interactive", + "--provider", + "openai", + "--api-key", + "sk-fake", + "--skip-channel", + "--skip-import", + "--yes", + ], + ) + assert r.exit_code == 0 + assert "Skipped via --skip-import" in r.stdout + + +def test_onboard_non_interactive_skips_import_step(tmp_env: Path, stub_verify, stub_step3) -> None: + """Non-interactive mode auto-skips Step 5 even without ``--skip-import``.""" + r = runner.invoke( + app, + [ + "onboard", + "--non-interactive", + "--provider", + "openai", + "--api-key", + "sk-fake", + "--skip-channel", + "--yes", + ], + ) + assert r.exit_code == 0, r.stdout + assert "Skipped (non-interactive)" in r.stdout + assert "Setup complete" in r.stdout + + # --------------------------------------------------------------------------- error paths @@ -469,6 +510,7 @@ def test_onboard_interactive_uses_stubbed_pickers( monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) r = runner.invoke(app, ["onboard"]) assert r.exit_code == 0, r.stdout @@ -604,6 +646,7 @@ def _fake_autocomplete(message, choices, default=None, **kwargs): monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) r = runner.invoke(app, ["onboard"]) assert r.exit_code == 0, r.stdout @@ -688,7 +731,7 @@ def test_registry_default_models_present() -> None: assert spec.default_model, f"{name} has empty default_model" -# --------------------------------------------------------------------------- fixtures (4-step) +# --------------------------------------------------------------------------- fixtures (5-step) @pytest.fixture @@ -989,6 +1032,13 @@ def ask(self): monkeypatch.setattr(onboard_commands, "_probe_everos_chat", lambda *a, **kw: (True, "ok")) monkeypatch.setattr(onboard_commands, "_verify_embedding_dim", lambda **kw: True) + import raven.plugin.memory.everos._server as everos_server + + async def _fake_ensure_everos_server(*a: object, **kw: object) -> None: + return None + + monkeypatch.setattr(everos_server, "ensure_everos_server", _fake_ensure_everos_server) + onboard_commands._step4_memory( skip=False, non_interactive=False, @@ -1346,6 +1396,7 @@ def _s3(**_): monkeypatch.setattr(onboard_commands, "_step3_channel", _s3) monkeypatch.setattr(onboard_commands, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) onboard_commands.run_wizard(non_interactive=False) # s2 returns BACK once → s1 replays → s2 again → forward. @@ -1374,6 +1425,7 @@ def test_first_screen_back_does_not_skip_step1( monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) onboard_commands.run_wizard(non_interactive=False) @@ -1420,6 +1472,7 @@ def _verify(name, *a, **kw): monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) # Should complete (not raise typer.Exit) — steps 2/3/4 ran. onboard_commands.run_wizard(non_interactive=False) @@ -1507,7 +1560,8 @@ def test_fresh_bootstrap_seeds_extension_blocks(tmp_env: Path, monkeypatch: pyte assert data["memory"]["backend"] == "everos" # schema default seeded assert data["memory"]["memoryTopK"] == 5 - assert data["plugins"]["config"]["everos-memory"]["mode"] == "embedded" + assert "mode" not in data["plugins"]["config"]["everos-memory"] + assert data["plugins"]["config"]["everos-memory"]["base_url"] == "http://localhost:18791" assert data["skillForge"]["everos"] == {"enabled": True} assert data["skillForge"]["router"]["hub"]["endpoint"] == "https://skillhub.evermind.ai" assert data["skillForge"]["router"]["hub"]["apiKey"] is None @@ -1540,7 +1594,7 @@ def test_bootstrap_backfills_preexisting_config(tmp_env: Path, monkeypatch: pyte assert data["memory"]["memoryTopK"] == 20 # Missing blocks / keys backfilled. assert data["memory"]["userId"] == "default" - assert data["plugins"]["config"]["everos-memory"]["mode"] == "embedded" + assert data["plugins"]["config"]["everos-memory"]["base_url"] == "http://localhost:18791" assert data["skillForge"]["router"]["hub"]["endpoint"] == "https://skillhub.evermind.ai" @@ -1589,10 +1643,10 @@ def _ph_text(placeholder: Any) -> Any: # --------------------------------------------------------------------------- step 5 (deep_research) -def test_total_steps_is_five() -> None: - # Adding the deep_research step bumped the wizard from 4 to 5; the progress - # dots + "Step n/N" header derive from this constant. - assert onboard_commands._TOTAL_STEPS == 5 +def test_total_steps_is_six() -> None: + # deep_research (step 5) + import (step 6) bumped the wizard from 4 to 6; + # the progress dots + "Step n/N" header derive from this constant. + assert onboard_commands._TOTAL_STEPS == 6 def test_step5_skip_or_non_interactive_never_configures(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_cli_smoke.py b/tests/test_cli_smoke.py index 5e8c7cd..6cb0bd1 100644 --- a/tests/test_cli_smoke.py +++ b/tests/test_cli_smoke.py @@ -161,6 +161,7 @@ def test_cron_list_body_does_not_crash(tmp_config: Path) -> None: "deep-research", "doctor", "gateway", + "import", "onboard", "plugins", "provider", diff --git a/tests/test_cli_tui_commands.py b/tests/test_cli_tui_commands.py index c765cf7..869634a 100644 --- a/tests/test_cli_tui_commands.py +++ b/tests/test_cli_tui_commands.py @@ -455,67 +455,6 @@ async def _start_with_root_handler(): ) -# --------------------------------------------------------------------------- -# fd-level stdout/stderr redirect spans the serve region -# --------------------------------------------------------------------------- - - -async def test_rpc_runner_activates_fd_redirect_before_backend_start(rpc_server_deps, monkeypatch, tmp_path) -> None: - """``_run_rpc_server_until_done`` must activate redirect_terminal_fds_to_file - before calling backend.start() so that everos structlog PrintLogger output - during start and serve lands in the log file, not on the terminal. - - Spies on the CM entry/exit and on start/stop to assert ordering: - redirect_enter < start ... stop < redirect_exit (held through the finally). - """ - import contextlib - - call_log: list[str] = [] - - original_start = rpc_server_deps["backend"].start - original_stop = rpc_server_deps["backend"].stop - - async def _spy_start(): - call_log.append("start") - await original_start() - - async def _spy_stop(): - call_log.append("stop") - await original_stop() - - rpc_server_deps["backend"].start = _spy_start - rpc_server_deps["backend"].stop = _spy_stop - - @contextlib.contextmanager - def _spy_redirect(path): - call_log.append("redirect_enter") - try: - yield - finally: - call_log.append("redirect_exit") - - monkeypatch.setattr( - "raven.cli.tui_commands.redirect_terminal_fds_to_file", - _spy_redirect, - ) - monkeypatch.setattr( - "raven.config.paths.get_logs_dir", - lambda: tmp_path, - ) - - await _run_until_done_with_handshake(monkeypatch, rpc_server_deps) - - assert "redirect_enter" in call_log, "redirect_terminal_fds_to_file must be entered" - enter_idx = call_log.index("redirect_enter") - start_idx = call_log.index("start") - assert enter_idx < start_idx, "redirect must be activated BEFORE backend.start()" - - assert "redirect_exit" in call_log, "redirect_terminal_fds_to_file must be exited (restored)" - exit_idx = call_log.index("redirect_exit") - stop_idx = call_log.index("stop") - assert stop_idx < exit_idx, "redirect must be held THROUGH backend.stop()" - - # --------------------------------------------------------------------------- # log-path notice: surfaced only on abnormal child exit (#131) # --------------------------------------------------------------------------- diff --git a/tests/test_config_update.py b/tests/test_config_update.py index eff846b..a112efb 100644 --- a/tests/test_config_update.py +++ b/tests/test_config_update.py @@ -273,8 +273,7 @@ def test_init_extension_defaults_seeds_safe_subset(cfg_path: Path) -> None: # plugins.config is never empty — it carries the everos-memory identity # wiring (snake_case, verbatim pass-through to the plugin factory). assert data["plugins"]["config"]["everos-memory"] == { - "mode": "embedded", - "base_url": "http://localhost:1995", + "base_url": "http://localhost:18791", "user_id": "default", "agent_id": "default", } diff --git a/tests/test_context_invariants.py b/tests/test_context_invariants.py index fc0e599..e154e9b 100644 --- a/tests/test_context_invariants.py +++ b/tests/test_context_invariants.py @@ -165,7 +165,7 @@ async def stop(self): async def feedback(self, signals): pass - async def store(self, session_id, messages): + async def store(self, session_id, messages, *, metadata=None): pass async def recall(self, query, *, user_id=None, agent_id=None, top_k): diff --git a/tests/test_default_context_engine.py b/tests/test_default_context_engine.py index b3e605e..f792ce4 100644 --- a/tests/test_default_context_engine.py +++ b/tests/test_default_context_engine.py @@ -81,7 +81,7 @@ async def stop(self): async def feedback(self, signals): pass - async def store(self, session_id, messages): + async def store(self, session_id, messages, *, metadata=None): pass async def recall(self, query, *, user_id=None, agent_id=None, top_k): diff --git a/tests/test_em1_skeleton.py b/tests/test_em1_skeleton.py index 9461464..754da64 100644 --- a/tests/test_em1_skeleton.py +++ b/tests/test_em1_skeleton.py @@ -130,7 +130,7 @@ def test_build_returns_protocol_compliant_backend( reg = assemble_plugin_registry(bundled_dir=_BUNDLED) backend = reg.build_memory_backend( "everos", - config={"mode": "embedded"}, + config={}, services=ServiceLocator(workspace=tmp_path), ) # @runtime_checkable Protocol: isinstance returns True iff all @@ -145,12 +145,16 @@ def test_build_returns_protocol_compliant_backend( @pytest.fixture def backend(tmp_path: Path): + from raven.plugin.memory.everos.backend import _NoOpAdapter + reg = assemble_plugin_registry(bundled_dir=_BUNDLED) - return reg.build_memory_backend( + be = reg.build_memory_backend( "everos", - config={"mode": "embedded"}, + config={}, services=ServiceLocator(workspace=tmp_path), ) + be._adapter = _NoOpAdapter() + return be class TestStubBehavior: @@ -184,20 +188,25 @@ async def test_feedback_accepts_any_dict(self, backend) -> None: class TestConfigPassthrough: - def test_mode_default_embedded(self, tmp_path: Path) -> None: + def test_default_constructs_http_adapter(self, tmp_path: Path) -> None: + from raven.plugin.memory.everos.backend import _HttpEverosAdapter + reg = assemble_plugin_registry(bundled_dir=_BUNDLED) backend = reg.build_memory_backend( "everos", config={}, services=ServiceLocator(workspace=tmp_path), ) - assert backend._mode == "embedded" + assert isinstance(backend._adapter, _HttpEverosAdapter) + + def test_base_url_passed_through(self, tmp_path: Path) -> None: + from raven.plugin.memory.everos.backend import _HttpEverosAdapter - def test_mode_http_via_config(self, tmp_path: Path) -> None: reg = assemble_plugin_registry(bundled_dir=_BUNDLED) backend = reg.build_memory_backend( "everos", - config={"mode": "http"}, + config={"base_url": "http://custom:9000"}, services=ServiceLocator(workspace=tmp_path), ) - assert backend._mode == "http" + assert isinstance(backend._adapter, _HttpEverosAdapter) + assert backend._adapter._base_url == "http://custom:9000" diff --git a/tests/test_em2_backend.py b/tests/test_em2_backend.py index 6450ae3..ef3269d 100644 --- a/tests/test_em2_backend.py +++ b/tests/test_em2_backend.py @@ -1,4 +1,4 @@ -"""EM-2 — EverosBackend embedded mode. +"""EverosBackend — HTTP-only mode. Adapter injection: tests build :class:`_FakeAdapter` instances and pass them directly into :class:`EverosBackend(ctx, adapter=...)`. This keeps @@ -12,6 +12,7 @@ from pathlib import Path from types import SimpleNamespace from typing import Any +from unittest.mock import AsyncMock, patch import pytest @@ -21,8 +22,6 @@ from raven.plugin import PluginContext, ServiceLocator from raven.plugin.memory.everos.backend import ( EverosBackend, - _NoOpAdapter, - _RealEverosAdapter, make_backend, ) @@ -52,12 +51,22 @@ async def search(self, *, user_id, agent_id, query, top_k): raise self.search_raises return self.search_response - async def memorize(self, session_id, payload_messages, *, is_final=False): + async def memorize( + self, + session_id, + payload_messages, + *, + is_final=False, + app_id=None, + project_id=None, + ): self.memorize_calls.append( { "session_id": session_id, "payload_messages": payload_messages, "is_final": is_final, + "app_id": app_id, + "project_id": project_id, } ) if self.memorize_raises is not None: @@ -66,7 +75,7 @@ async def memorize(self, session_id, payload_messages, *, is_final=False): def _ctx(tmp_path: Path, **config: Any) -> PluginContext: return PluginContext( - config={"mode": "embedded", **config}, + config=config, services=ServiceLocator(workspace=tmp_path), ) @@ -86,19 +95,6 @@ def test_protocol_conformance(self, tmp_path: Path) -> None: b = _backend(tmp_path) assert isinstance(b, MemoryBackend) - def test_invalid_mode_raises(self, tmp_path: Path) -> None: - """A typo'd mode fails fast at construction rather than silently - degrading to a no-op adapter (memory quietly disabled).""" - with pytest.raises(ValueError, match="invalid mode"): - EverosBackend(_ctx(tmp_path, mode="embeded")) - - def test_embedded_mode_defers_adapter_to_start(self, tmp_path: Path) -> None: - """Embedded mode defers the (heavy everos/lancedb) adapter build to - start(): at construction the adapter is None, so the ~2-3s import stays - off the render-blocking build path. start() then selects real/no-op.""" - b = EverosBackend(_ctx(tmp_path, mode="embedded")) - assert b._adapter is None - def test_make_backend_factory(self, tmp_path: Path) -> None: b = make_backend(_ctx(tmp_path)) assert isinstance(b, EverosBackend) @@ -112,28 +108,20 @@ def test_make_backend_factory(self, tmp_path: Path) -> None: class TestLifecycle: async def test_start_stop_idempotent(self, tmp_path: Path) -> None: b = _backend(tmp_path) - await b.start() - await b.stop() - await b.start() - await b.stop() - - async def test_start_builds_deferred_embedded_adapter(self, tmp_path: Path) -> None: - """start() builds the embedded adapter that __init__ deferred.""" - b = EverosBackend(_ctx(tmp_path, mode="embedded")) - assert b._adapter is None - try: + with patch("raven.plugin.memory.everos._server.ensure_everos_server", new=AsyncMock()): + await b.start() + await b.stop() await b.start() - assert isinstance(b._adapter, (_NoOpAdapter, _RealEverosAdapter)) - finally: await b.stop() - async def test_recall_and_store_degrade_before_adapter_built(self, tmp_path: Path) -> None: - """Before start() builds the deferred adapter, recall returns empty and - store is a no-op (the render-window degrade), not a crash.""" - b = EverosBackend(_ctx(tmp_path, mode="embedded")) - assert b._adapter is None - assert await b.recall("hi", user_id="alice", top_k=5) == [] - await b.store("sess", [{"role": "user", "content": "hi"}]) # must not raise + async def test_start_calls_ensure_everos_server(self, tmp_path: Path) -> None: + b = EverosBackend(_ctx(tmp_path)) + with patch( + "raven.plugin.memory.everos._server.ensure_everos_server", + new=AsyncMock(), + ) as mock_ensure: + await b.start() + mock_ensure.assert_called_once() # --------------------------------------------------------------------------- @@ -504,13 +492,12 @@ async def test_explicit_sender_id_preserved(self, tmp_path: Path) -> None: ) assert adapter.memorize_calls[0]["payload_messages"][0]["sender_id"] == "alice-123" - async def test_memorize_exception_swallowed(self, tmp_path: Path) -> None: + async def test_memorize_exception_propagates(self, tmp_path: Path) -> None: adapter = _FakeAdapter() adapter.memorize_raises = RuntimeError("everos down") b = _backend(tmp_path, adapter=adapter) - # Backend.store does NOT raise; AgentLoop's after-turn step - # should never be derailed by a backend store failure. - await b.store("s", [{"role": "user", "content": "x"}]) + with pytest.raises(RuntimeError, match="everos down"): + await b.store("s", [{"role": "user", "content": "x"}]) # --------------------------------------------------------------------------- @@ -566,137 +553,3 @@ async def test_feedback_accepts_any_signals(self, tmp_path: Path) -> None: await b.feedback({}) await b.feedback({"kind": "skill_usage", "ids": ["x"]}) await b.feedback({"arbitrary": object()}) - - -class TestRerankDegrade: - """``_RealEverosAdapter`` picks the everos search method by track + - rerank availability: agent-track HYBRID needs a cross-encoder rerank - provider (everos raises without one), so when rerank is unconfigured - the adapter degrades the agent track to VECTOR (no rerank). The user - track never touches the reranker and stays HYBRID regardless. - """ - - @staticmethod - async def _capture_method(*, rerank_configured: bool, user_id, agent_id): - from types import SimpleNamespace - - from everos.memory.search.dto import SearchMethod - - adapter = _RealEverosAdapter() - adapter._rerank_configured = rerank_configured - captured: dict = {} - - async def _fake_search(req): - captured["method"] = req.method - return SimpleNamespace(data=None) - - adapter._search_fn = _fake_search - await adapter.search(user_id=user_id, agent_id=agent_id, query="q", top_k=5) - return captured["method"], SearchMethod - - async def test_agent_degrades_to_vector_without_rerank(self) -> None: - method, SearchMethod = await self._capture_method( - rerank_configured=False, - user_id=None, - agent_id="agent-x", - ) - assert method == SearchMethod.VECTOR - - async def test_agent_uses_hybrid_with_rerank(self) -> None: - method, SearchMethod = await self._capture_method( - rerank_configured=True, - user_id=None, - agent_id="agent-x", - ) - assert method == SearchMethod.HYBRID - - async def test_user_stays_hybrid_without_rerank(self) -> None: - # User track never hits the cross-encoder lane, so no degrade. - method, SearchMethod = await self._capture_method( - rerank_configured=False, - user_id="user-x", - agent_id=None, - ) - assert method == SearchMethod.HYBRID - - -# --------------------------------------------------------------------------- -# LanceDB schema migration -# --------------------------------------------------------------------------- - - -class TestMigrateLancedbSchemas: - """``_migrate_lancedb_schemas`` adds missing columns to existing - LanceDB tables so upgrading users don't need to manually clear - their index.""" - - @staticmethod - def _stub_schema(table_name: str, fields: set[str]): - class _S: - TABLE_NAME = table_name - model_fields = {f: None for f in fields} - - return _S - - @staticmethod - def _stub_table(columns: set[str]): - import pyarrow as pa - - arrow = pa.schema([pa.field(c, pa.utf8()) for c in sorted(columns)]) - added: list = [] - - class _T: - async def schema(self): - return arrow - - async def add_columns(self, new_schema): - added.append(new_schema) - - return _T(), added - - async def test_adds_missing_columns(self) -> None: - import logging - - from raven.plugin.memory.everos.backend import _migrate_lancedb_schemas - - table, added = self._stub_table({"id", "text"}) - schema_cls = self._stub_schema("tbl", {"id", "text", "new_col"}) - - async def _noop(): - return None - - async def _get_table(_name, _schema): - return table - - result = await _migrate_lancedb_schemas( - logging.getLogger("test"), - _schemas=[schema_cls], - _get_connection=_noop, - _get_table=_get_table, - ) - assert result is True - assert len(added) == 1 - assert "new_col" in added[0].names - - async def test_no_missing_returns_false(self) -> None: - import logging - - from raven.plugin.memory.everos.backend import _migrate_lancedb_schemas - - table, added = self._stub_table({"id", "text"}) - schema_cls = self._stub_schema("tbl", {"id", "text"}) - - async def _noop(): - return None - - async def _get_table(_name, _schema): - return table - - result = await _migrate_lancedb_schemas( - logging.getLogger("test"), - _schemas=[schema_cls], - _get_connection=_noop, - _get_table=_get_table, - ) - assert result is False - assert len(added) == 0 diff --git a/tests/test_em3_http.py b/tests/test_em3_http.py index 7be7c3a..e17e619 100644 --- a/tests/test_em3_http.py +++ b/tests/test_em3_http.py @@ -288,7 +288,7 @@ async def test_trailing_slash_stripped(self, mock, http_client) -> None: # --------------------------------------------------------------------------- -# EverosBackend in mode="http" wires the HTTP adapter end-to-end +# EverosBackend wires the HTTP adapter end-to-end # --------------------------------------------------------------------------- @@ -304,7 +304,6 @@ def test_http_mode_constructs_http_adapter(self, tmp_path: Path) -> None: b = EverosBackend( _ctx( tmp_path, - mode="http", base_url="http://x:9000", ) ) @@ -312,15 +311,14 @@ def test_http_mode_constructs_http_adapter(self, tmp_path: Path) -> None: assert b._adapter._base_url == "http://x:9000" def test_base_url_default(self, tmp_path: Path) -> None: - b = EverosBackend(_ctx(tmp_path, mode="http")) + b = EverosBackend(_ctx(tmp_path)) assert isinstance(b._adapter, _HttpEverosAdapter) - assert b._adapter._base_url == "http://localhost:1995" + assert b._adapter._base_url == "http://localhost:18791" def test_api_key_threaded_through(self, tmp_path: Path) -> None: b = EverosBackend( _ctx( tmp_path, - mode="http", api_key="my-key", ) ) @@ -360,7 +358,7 @@ async def test_end_to_end_recall_through_http( # Build the backend with the explicit adapter b = EverosBackend( - _ctx(tmp_path, mode="http"), + _ctx(tmp_path), adapter=adapter, ) hits = await b.recall("git", agent_id="agent:default", top_k=5) @@ -377,7 +375,7 @@ async def test_backend_stop_closes_http_adapter( self, tmp_path: Path, ) -> None: - b = EverosBackend(_ctx(tmp_path, mode="http")) + b = EverosBackend(_ctx(tmp_path)) # Get a handle to the adapter to verify close happens adapter = b._adapter assert isinstance(adapter, _HttpEverosAdapter) diff --git a/tests/test_everos_server.py b/tests/test_everos_server.py new file mode 100644 index 0000000..5a8cc51 --- /dev/null +++ b/tests/test_everos_server.py @@ -0,0 +1,73 @@ +"""Tests for raven.plugin.memory.everos._server.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from raven.plugin.memory.everos._server import ensure_everos_server + + +class TestEnsureEverosServer: + @pytest.mark.asyncio + async def test_server_already_running(self) -> None: + mock_response = MagicMock() + mock_response.status_code = 200 + + with patch( + "raven.plugin.memory.everos._server._probe_health", + return_value=True, + ): + await ensure_everos_server("http://localhost:18791") + + @pytest.mark.asyncio + async def test_auto_start_on_connection_error(self, tmp_path) -> None: + call_count = 0 + + def probe_side_effect(*_args, **_kwargs): + nonlocal call_count + call_count += 1 + return call_count >= 3 + + with ( + patch( + "raven.plugin.memory.everos._server._probe_health", + side_effect=probe_side_effect, + ), + patch( + "raven.plugin.memory.everos._server._start_server_if_unlocked", + ) as mock_start, + patch( + "raven.plugin.memory.everos._server.get_logs_dir", + return_value=tmp_path, + ), + ): + await ensure_everos_server("http://localhost:18791", timeout=10.0) + + mock_start.assert_called_once() + + @pytest.mark.asyncio + async def test_timeout_raises(self, tmp_path) -> None: + with ( + patch( + "raven.plugin.memory.everos._server._probe_health", + return_value=False, + ), + patch( + "raven.plugin.memory.everos._server._start_server_if_unlocked", + ), + patch( + "raven.plugin.memory.everos._server.get_logs_dir", + return_value=tmp_path, + ), + pytest.raises(RuntimeError, match="EverOS server failed to start"), + ): + await ensure_everos_server("http://localhost:18791", timeout=1.0) + + def test_port_extraction(self) -> None: + from raven.plugin.memory.everos._server import _extract_port + + assert _extract_port("http://localhost:18791") == "18791" + assert _extract_port("http://127.0.0.1:9999") == "9999" + assert _extract_port("http://localhost") == "80" diff --git a/tests/test_fb1_feedback_dispatch.py b/tests/test_fb1_feedback_dispatch.py index 78052fb..b948a78 100644 --- a/tests/test_fb1_feedback_dispatch.py +++ b/tests/test_fb1_feedback_dispatch.py @@ -45,7 +45,7 @@ async def start(self): async def stop(self): pass - async def store(self, session_id, messages): + async def store(self, session_id, messages, *, metadata=None): pass async def recall(self, query, *, user_id=None, agent_id=None, top_k): diff --git a/tests/test_importer_claude_code_scanner.py b/tests/test_importer_claude_code_scanner.py new file mode 100644 index 0000000..a20f7b5 --- /dev/null +++ b/tests/test_importer_claude_code_scanner.py @@ -0,0 +1,526 @@ +"""Tests for ClaudeCodeScanner -- cold-start import from Claude Code.""" + +from __future__ import annotations + +import json +import os +import time +from pathlib import Path + +import pytest + +from raven.importer import ( + ClaudeCodeScanner, + Platform, + Scanner, + SourceKind, +) +from raven.importer.scanners.claude_code import ( + _build_file_messages, + _make_file_end, + _make_intro, + _memory_files_sorted, + _split_paragraphs, + _truncate, +) +from raven.utils.text import is_cjk, parse_frontmatter, parse_iso_ts_ms + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +_OLD_MTIME = time.time() - 600 + + +def _write_jsonl(path: Path, events: list[dict], *, active: bool = False) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + for ev in events: + f.write(json.dumps(ev) + "\n") + if not active: + os.utime(path, (_OLD_MTIME, _OLD_MTIME)) + + +def _event(role: str, content, *, ts="2026-07-01T10:00:00Z", **extra): + ev = { + "type": role, + "timestamp": ts, + "message": {"role": role, "content": content}, + } + ev.update(extra) + return ev + + +def _write_md(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +@pytest.fixture +def claude_dir(tmp_path: Path) -> Path: + return tmp_path / ".claude" + + +@pytest.fixture +def scanner(claude_dir: Path) -> ClaudeCodeScanner: + return ClaudeCodeScanner(claude_dir=claude_dir) + + +# --------------------------------------------------------------------------- +# Protocol conformance +# --------------------------------------------------------------------------- + + +class TestProtocol: + def test_satisfies_scanner_protocol(self) -> None: + assert isinstance(ClaudeCodeScanner(), Scanner) + + def test_platform_attribute(self) -> None: + assert ClaudeCodeScanner.platform == Platform.CLAUDE_CODE + + +# --------------------------------------------------------------------------- +# scan() +# --------------------------------------------------------------------------- + + +class TestScan: + async def test_missing_claude_dir(self, scanner: ClaudeCodeScanner) -> None: + assert await scanner.scan() == [] + + async def test_global_claude_md(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + _write_md(claude_dir / "CLAUDE.md", "## Rules\n- Use English") + results = await scanner.scan() + mem = [r for r in results if r.kind == SourceKind.MEMORY_FILE] + assert len(mem) == 1 + assert mem[0].source_key == "global-claude-md" + + async def test_project_memory_bundle(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + mem_dir = claude_dir / "projects" / "test-proj" / "memory" + _write_md(mem_dir / "arch.md", "---\nname: arch\nmetadata:\n type: reference\n---\nBody") + _write_md(mem_dir / "MEMORY.md", "# Index") + results = await scanner.scan() + mem = [r for r in results if r.kind == SourceKind.MEMORY_FILE and "memory" in r.source_key] + assert len(mem) == 1 + assert len(mem[0].file_paths) == 2 + + async def test_project_sessions(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "test-proj" + _write_jsonl(proj / "abc-123.jsonl", [_event("user", "hello")]) + results = await scanner.scan() + conv = [r for r in results if r.kind == SourceKind.CONVERSATION] + assert len(conv) == 1 + assert conv[0].source_key == "abc-123" + + async def test_active_session_skipped(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "test-proj" + _write_jsonl(proj / "active.jsonl", [_event("user", "hello")], active=True) + results = await scanner.scan() + assert not [r for r in results if r.kind == SourceKind.CONVERSATION] + + async def test_subagent_files_excluded(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "test-proj" + _write_jsonl(proj / "main.jsonl", [_event("user", "hello")]) + _write_jsonl(proj / "main" / "subagents" / "agent-abc.jsonl", [_event("user", "sub")]) + results = await scanner.scan() + conv = [r for r in results if r.kind == SourceKind.CONVERSATION] + assert len(conv) == 1 + assert conv[0].source_key == "main" + + async def test_oversized_memory_file_skipped(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + mem_dir = claude_dir / "projects" / "proj" / "memory" + _write_md(mem_dir / "huge.md", "x" * (1_048_576 + 1)) + _write_md(mem_dir / "small.md", "ok") + results = await scanner.scan() + mem = [r for r in results if "memory" in r.source_key] + assert len(mem[0].file_paths) == 1 + assert mem[0].file_paths[0].name == "small.md" + + +# --------------------------------------------------------------------------- +# read(CONVERSATION) +# --------------------------------------------------------------------------- + + +class TestReadConversation: + async def test_basic_user_assistant(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + _write_jsonl(proj / "s1.jsonl", [_event("user", "Q"), _event("assistant", "A")]) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + session = await scanner.read(r) + assert len(session.messages) == 2 + assert session.messages[0].role == "user" + assert session.messages[1].role == "assistant" + + async def test_tool_use_extraction(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + _write_jsonl( + proj / "s2.jsonl", + [ + _event( + "assistant", + [ + {"type": "text", "text": "Reading."}, + {"type": "tool_use", "id": "t1", "name": "Read", "input": {"file_path": "/f"}}, + ], + ) + ], + ) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + msg = (await scanner.read(r)).messages[0] + assert msg.tool_calls is not None + assert msg.tool_calls[0]["function"]["name"] == "Read" + + async def test_tool_result_extraction(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + _write_jsonl( + proj / "s3.jsonl", + [ + _event( + "user", + [ + {"type": "tool_result", "tool_use_id": "t1", "content": "file data"}, + {"type": "text", "text": "Continue."}, + ], + ) + ], + ) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + msgs = (await scanner.read(r)).messages + assert msgs[0].role == "tool" + assert msgs[0].tool_call_id == "t1" + assert msgs[1].role == "user" + + async def test_tool_result_without_id_dropped(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + _write_jsonl( + proj / "s4.jsonl", + [_event("user", [{"type": "tool_result", "content": "no id"}, {"type": "text", "text": "ok"}])], + ) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + msgs = (await scanner.read(r)).messages + assert len(msgs) == 1 + assert msgs[0].role == "user" + + async def test_ismeta_skipped(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + _write_jsonl( + proj / "s5.jsonl", + [_event("user", "real"), _event("user", "meta", isMeta=True), _event("assistant", "ok")], + ) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + msgs = (await scanner.read(r)).messages + assert len(msgs) == 2 + + async def test_compact_summary_skipped(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + _write_jsonl( + proj / "s6.jsonl", + [_event("user", "real"), _event("user", "summary", isCompactSummary=True)], + ) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + assert len((await scanner.read(r)).messages) == 1 + + async def test_api_error_skipped(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + _write_jsonl( + proj / "s7.jsonl", + [_event("user", "q"), _event("assistant", "err", isApiErrorMessage=True), _event("assistant", "ok")], + ) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + msgs = (await scanner.read(r)).messages + assert len(msgs) == 2 + assert msgs[1].content == "ok" + + async def test_non_conversation_events_skipped(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + events = [ + _event("user", "hello"), + {"type": "system", "subtype": "turn_duration"}, + {"type": "attachment", "attachment": {"type": "diagnostics"}}, + {"type": "ai-title", "aiTitle": "Test"}, + _event("assistant", "world"), + ] + _write_jsonl(proj / "s8.jsonl", events) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + assert len((await scanner.read(r)).messages) == 2 + + async def test_thinking_blocks_skipped(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + _write_jsonl( + proj / "s9.jsonl", + [_event("assistant", [{"type": "thinking", "thinking": "hmm"}, {"type": "text", "text": "reply"}])], + ) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + assert (await scanner.read(r)).messages[0].content == "reply" + + async def test_malformed_json_skipped(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + path = proj / "s10.jsonl" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + f.write("{bad\n") + f.write(json.dumps(_event("user", "good")) + "\n") + os.utime(path, (_OLD_MTIME, _OLD_MTIME)) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + assert len((await scanner.read(r)).messages) == 1 + + async def test_content_truncation(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + _write_jsonl(proj / "s11.jsonl", [_event("user", "x" * 15_000)]) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + msg = (await scanner.read(r)).messages[0] + assert len(msg.content) == 10_003 + assert msg.content.endswith("...") + + async def test_timestamp_parsing(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + _write_jsonl(proj / "s12.jsonl", [_event("user", "hi", ts="2026-07-01T12:00:00.500Z")]) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + assert (await scanner.read(r)).messages[0].timestamp == 1782907200500 + + async def test_numeric_timestamp(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + ev = {"type": "user", "timestamp": 1720000000000, "message": {"role": "user", "content": "num ts"}} + _write_jsonl(proj / "s13.jsonl", [ev]) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + msgs = (await scanner.read(r)).messages + assert len(msgs) == 1 + assert msgs[0].timestamp == 1720000000000 + + async def test_no_timestamp_event_skipped(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + ev = {"type": "user", "message": {"role": "user", "content": "no ts"}} + _write_jsonl(proj / "s14.jsonl", [ev, _event("user", "with ts")]) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + assert len((await scanner.read(r)).messages) == 1 + + async def test_empty_content_skipped(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + _write_jsonl( + proj / "s15.jsonl", + [ + _event("user", ""), + _event("user", " "), + _event("assistant", [{"type": "thinking", "thinking": "only"}]), + _event("user", "real"), + ], + ) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + msgs = (await scanner.read(r)).messages + assert len(msgs) == 1 + assert msgs[0].content == "real" + + async def test_complete_tool_trajectory(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + proj = claude_dir / "projects" / "proj" + _write_jsonl( + proj / "s16.jsonl", + [ + _event("user", "Read file.py", ts="2026-07-01T10:00:00Z"), + _event( + "assistant", + [ + {"type": "text", "text": "Let me read it."}, + {"type": "tool_use", "id": "t1", "name": "Read", "input": {"file_path": "f.py"}}, + ], + ts="2026-07-01T10:00:01Z", + ), + _event( + "user", + [{"type": "tool_result", "tool_use_id": "t1", "content": "def hello(): pass"}], + ts="2026-07-01T10:00:02Z", + ), + _event("assistant", "It defines a hello function.", ts="2026-07-01T10:00:03Z"), + ], + ) + r = [r for r in await scanner.scan() if r.kind == SourceKind.CONVERSATION][0] + msgs = (await scanner.read(r)).messages + assert [m.role for m in msgs] == ["user", "assistant", "tool", "assistant"] + assert msgs[1].tool_calls is not None + assert msgs[1].tool_calls[0]["id"] == "t1" + assert msgs[2].tool_call_id == "t1" + assert [m.timestamp for m in msgs] == sorted(m.timestamp for m in msgs) + + +# --------------------------------------------------------------------------- +# read(MEMORY_FILE) +# --------------------------------------------------------------------------- + + +class TestReadMemory: + async def test_global_md_english(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + _write_md(claude_dir / "CLAUDE.md", "## Rules\nUse English.\n\n## Style\nBe concise.") + r = [r for r in await scanner.scan() if r.source_key == "global-claude-md"][0] + session = await scanner.read(r) + assert session.session_id == "import-claude_code-global" + assert "Claude Code" in session.messages[0].content + assert session.messages[-1].content.endswith("CLAUDE.md.") + assert len(session.messages) >= 4 # intro + 2 paragraphs + file end + + async def test_global_md_chinese(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + _write_md(claude_dir / "CLAUDE.md", "## 规则\n永远用中文\n\n## 风格\n简洁") + r = [r for r in await scanner.scan() if r.source_key == "global-claude-md"][0] + session = await scanner.read(r) + assert "Claude Code" in session.messages[0].content + assert "全局" in session.messages[0].content + + async def test_global_md_empty_returns_no_messages(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + _write_md(claude_dir / "CLAUDE.md", "") + r = [r for r in await scanner.scan() if r.source_key == "global-claude-md"][0] + session = await scanner.read(r) + assert len(session.messages) == 0 + + async def test_project_memory_frontmatter(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + mem = claude_dir / "projects" / "proj" / "memory" + _write_md(mem / "arch.md", "---\nname: architecture\nmetadata:\n type: reference\n---\nLayer 1\n\nLayer 2") + r = [r for r in await scanner.scan() if "memory" in r.source_key][0] + session = await scanner.read(r) + bodies = [m.content for m in session.messages] + assert "Layer 1" in bodies + assert "Layer 2" in bodies + assert any("arch.md" in m.content for m in session.messages) + assert session.messages[0].content.startswith("These are") or session.messages[0].content.startswith("这是") + + async def test_memory_md_no_frontmatter(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + mem = claude_dir / "projects" / "proj" / "memory" + _write_md(mem / "MEMORY.md", "# Index\n\n- item 1") + r = [r for r in await scanner.scan() if "memory" in r.source_key][0] + session = await scanner.read(r) + assert any("overview" in m.content.lower() for m in session.messages) + + async def test_feedback_type_intro(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + mem = claude_dir / "projects" / "proj" / "memory" + _write_md(mem / "fb.md", "---\nname: style\nmetadata:\n type: feedback\n---\nPrefer short answers") + r = [r for r in await scanner.scan() if "memory" in r.source_key][0] + session = await scanner.read(r) + assert any("preference" in m.content.lower() for m in session.messages) + + async def test_paragraph_splitting(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + _write_md(claude_dir / "CLAUDE.md", "Para one.\n\nPara two.\n\nPara three.") + r = [r for r in await scanner.scan() if r.source_key == "global-claude-md"][0] + session = await scanner.read(r) + assert len(session.messages) == 5 # intro + 3 paragraphs + file end + assert session.messages[1].content == "Para one." + assert "CLAUDE.md" in session.messages[-1].content + + async def test_all_files_one_session(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + mem = claude_dir / "projects" / "proj" / "memory" + _write_md(mem / "a.md", "Content A") + _write_md(mem / "b.md", "Content B") + r = [r for r in await scanner.scan() if "memory" in r.source_key][0] + session = await scanner.read(r) + contents = [m.content for m in session.messages] + assert "Content A" in contents and "Content B" in contents + + async def test_empty_body_skipped(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + mem = claude_dir / "projects" / "proj" / "memory" + _write_md(mem / "empty.md", "---\nname: empty\n---\n \n") + _write_md(mem / "real.md", "Real content") + r = [r for r in await scanner.scan() if "memory" in r.source_key][0] + session = await scanner.read(r) + assert not any("empty" in m.content.lower() for m in session.messages) + + async def test_language_detection_per_file(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + mem = claude_dir / "projects" / "proj" / "memory" + _write_md(mem / "en.md", "---\nname: arch\nmetadata:\n type: reference\n---\nEnglish content") + _write_md(mem / "zh.md", "---\nname: conv\nmetadata:\n type: reference\n---\n中文内容说明") + r = [r for r in await scanner.scan() if "memory" in r.source_key][0] + session = await scanner.read(r) + intros = [m.content for m in session.messages if "knowledge" in m.content.lower() or "知识" in m.content] + assert any("Here is" in i for i in intros) + assert any("以下" in i for i in intros) + + async def test_stat_failure_falls_back(self, claude_dir: Path, scanner: ClaudeCodeScanner) -> None: + """B1 fix: stat() failure should not crash the entire read.""" + mem = claude_dir / "projects" / "proj" / "memory" + _write_md(mem / "a.md", "Content A") + _write_md(mem / "b.md", "Content B") + r = [r for r in await scanner.scan() if "memory" in r.source_key][0] + session = await scanner.read(r) + assert len(session.messages) >= 2 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +class TestHelpers: + def testparse_frontmatter(self) -> None: + fm, body = parse_frontmatter("---\nname: test\nmetadata:\n type: reference\n---\nBody") + assert fm["name"] == "test" + assert body == "Body" + + def test_parse_frontmatter_none(self) -> None: + fm, body = parse_frontmatter("# No frontmatter") + assert fm == {} + assert body == "# No frontmatter" + + def test_parse_frontmatter_bad_yaml(self) -> None: + fm, _ = parse_frontmatter("---\n: [invalid\n---\nBody") + assert fm == {} + + def test_parse_frontmatter_inner_separator(self) -> None: + """B4 fix: --- inside YAML literal block should not split.""" + text = "---\nname: test\ncontent: |\n line1\n ---\n line2\n---\nReal body" + fm, body = parse_frontmatter(text) + assert fm.get("name") == "test" + assert "Real body" in body + + def test_split_paragraphs(self) -> None: + assert _split_paragraphs("A\n\nB\n\n\nC\n\n") == ["A", "B", "C"] + + def test_split_paragraphs_single(self) -> None: + assert _split_paragraphs("Just one") == ["Just one"] + + def test_truncate_short(self) -> None: + assert _truncate("short") == "short" + + def test_truncate_long(self) -> None: + result = _truncate("a" * 15_000) + assert len(result) == 10_003 + assert result.endswith("...") + + def testis_cjk(self) -> None: + assert is_cjk("这是中文") + assert not is_cjk("English only") + assert is_cjk("Mixed 中文") + + def test_parse_iso_ts_string(self) -> None: + assert parse_iso_ts_ms("2026-07-01T12:00:00Z") == 1782907200000 + assert parse_iso_ts_ms("2026-07-01T12:00:00.500Z") == 1782907200500 + + def test_parse_iso_ts_numeric(self) -> None: + assert parse_iso_ts_ms(1720000000000) == 1720000000000 + assert parse_iso_ts_ms(1720000000.5) == 1720000000500 + + def test_parse_iso_ts_invalid(self) -> None: + assert parse_iso_ts_ms("invalid") is None + assert parse_iso_ts_ms("") is None + assert parse_iso_ts_ms(None) is None + + def test_make_intro_data_driven(self) -> None: + fm = {"name": "arch", "metadata": {"type": "reference"}} + assert "arch" in _make_intro("arch.md", fm, cjk=False) + assert "knowledge" in _make_intro("arch.md", fm, cjk=False).lower() + assert "知识" in _make_intro("arch.md", fm, cjk=True) + + def test_build_file_messages(self) -> None: + msgs = _build_file_messages("Intro", "Para 1\n\nPara 2", "End.", 1000) + assert len(msgs) == 4 + assert msgs[0].content == "Intro" + assert msgs[1].content == "Para 1" + assert msgs[2].content == "Para 2" + assert msgs[3].content == "End." + + def test_make_file_end(self) -> None: + assert "CLAUDE.md" in _make_file_end("CLAUDE.md", cjk=False) + assert "CLAUDE.md" in _make_file_end("CLAUDE.md", cjk=True) + + def test_memory_files_sorted_index_first(self) -> None: + from pathlib import Path + + paths = (Path("b.md"), Path("MEMORY.md"), Path("a.md")) + result = _memory_files_sorted(paths) + assert result[0].name == "MEMORY.md" + assert [p.name for p in result[1:]] == ["a.md", "b.md"] diff --git a/tests/test_importer_orchestrator.py b/tests/test_importer_orchestrator.py new file mode 100644 index 0000000..691cee5 --- /dev/null +++ b/tests/test_importer_orchestrator.py @@ -0,0 +1,457 @@ +"""Tests for raven.importer.orchestrator.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from raven.importer.orchestrator import ImportSummary, ProgressEvent, run_import +from raven.importer.state import ImportState +from raven.importer.types import ( + ImportMessage, + ImportSession, + Platform, + ScanResult, + SourceKind, +) + +# --------------------------------------------------------------------------- +# Test doubles +# --------------------------------------------------------------------------- + + +class FakeBackend: + """Records store() calls for assertion.""" + + def __init__(self, *, fail_on: set[str] | None = None) -> None: + self.calls: list[dict[str, Any]] = [] + self._fail_on = fail_on or set() + + async def recall(self, query: str, *, user_id: str | None = None, agent_id: str | None = None, top_k: int) -> list: + return [] + + async def store( + self, session_id: str, messages: list[dict[str, Any]], *, metadata: dict[str, Any] | None = None + ) -> None: + if session_id in self._fail_on: + raise RuntimeError(f"store failed for {session_id}") + self.calls.append({"session_id": session_id, "messages": messages, "metadata": metadata}) + + async def feedback(self, signals: dict[str, Any]) -> None: + pass + + async def start(self) -> None: + pass + + async def stop(self) -> None: + pass + + +def _msg( + content: str = "hello", + role: str = "user", + ts: int = 1000, + sender: str = "user", + tool_calls: tuple[dict[str, Any], ...] | None = None, + tool_call_id: str | None = None, +) -> ImportMessage: + return ImportMessage( + role=role, content=content, timestamp=ts, sender_id=sender, tool_calls=tool_calls, tool_call_id=tool_call_id + ) + + +def _session( + n_msgs: int = 3, + session_id: str = "sess-1", + content: str = "hello", +) -> ImportSession: + msgs = tuple(_msg(content=f"{content}-{i}", ts=1000 + i) for i in range(n_msgs)) + return ImportSession(session_id=session_id, messages=msgs) + + +def _scan_result(key: str = "k1", platform: Platform = Platform.CLAUDE_CODE) -> ScanResult: + return ScanResult( + source_key=key, + platform=platform, + kind=SourceKind.CONVERSATION, + file_paths=(Path("/fake"),), + estimated_size=100, + mtime=1000.0, + ) + + +class FakeScanner: + def __init__(self, sessions: dict[str, ImportSession] | None = None, *, fail_on: set[str] | None = None) -> None: + self.platform = Platform.CLAUDE_CODE + self._sessions = sessions or {} + self._fail_on = fail_on or set() + + async def scan(self) -> list[ScanResult]: + return [] + + async def read(self, result: ScanResult) -> ImportSession: + if result.source_key in self._fail_on: + raise OSError(f"read failed for {result.source_key}") + return self._sessions.get(result.source_key, _session(session_id=f"import-{result.source_key}")) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestRunImportBasic: + @pytest.mark.asyncio + async def test_empty_items(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + summary = await run_import([], backend, state) + assert summary == ImportSummary(total=0, submitted=0, skipped=0, failed=0, errors=()) + assert backend.calls == [] + + @pytest.mark.asyncio + async def test_single_session(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + scanner = FakeScanner({"k1": _session(n_msgs=3, session_id="s1")}) + result = _scan_result("k1") + + summary = await run_import([(scanner, result)], backend, state) + + assert summary.total == 1 + assert summary.submitted == 1 + assert summary.skipped == 0 + assert summary.failed == 0 + assert len(backend.calls) == 1 + assert backend.calls[0]["session_id"] == "s1" + assert len(backend.calls[0]["messages"]) == 3 + assert backend.calls[0]["metadata"]["is_final"] is True + assert state.is_submitted("claude_code", "k1") + + @pytest.mark.asyncio + async def test_multiple_sessions(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + scanner = FakeScanner( + { + "a": _session(n_msgs=2, session_id="sa"), + "b": _session(n_msgs=2, session_id="sb"), + } + ) + items = [(scanner, _scan_result("a")), (scanner, _scan_result("b"))] + + summary = await run_import(items, backend, state) + + assert summary.total == 2 + assert summary.submitted == 2 + assert state.is_submitted("claude_code", "a") + assert state.is_submitted("claude_code", "b") + + +class TestIdempotent: + @pytest.mark.asyncio + async def test_skip_already_submitted(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + state.mark_submitted("claude_code", "k1") + backend = FakeBackend() + scanner = FakeScanner() + + summary = await run_import([(scanner, _scan_result("k1"))], backend, state) + + assert summary.skipped == 1 + assert summary.submitted == 0 + assert backend.calls == [] + + @pytest.mark.asyncio + async def test_retry_previously_failed(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + state.mark_failed("claude_code", "k1", "old error") + backend = FakeBackend() + scanner = FakeScanner({"k1": _session(n_msgs=1, session_id="s1")}) + + summary = await run_import([(scanner, _scan_result("k1"))], backend, state) + + assert summary.submitted == 1 + assert summary.skipped == 0 + assert state.is_submitted("claude_code", "k1") + + +class TestErrorIsolation: + @pytest.mark.asyncio + async def test_read_failure_continues(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + scanner = FakeScanner( + {"b": _session(n_msgs=1, session_id="sb")}, + fail_on={"a"}, + ) + items = [(scanner, _scan_result("a")), (scanner, _scan_result("b"))] + + summary = await run_import(items, backend, state) + + assert summary.failed == 1 + assert summary.submitted == 1 + assert len(summary.errors) == 1 + assert summary.errors[0].source_key == "a" + assert not state.is_submitted("claude_code", "a") + assert state.is_submitted("claude_code", "b") + + @pytest.mark.asyncio + async def test_store_failure_continues(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend(fail_on={"import-a"}) + scanner = FakeScanner( + { + "a": _session(n_msgs=1, session_id="import-a"), + "b": _session(n_msgs=1, session_id="import-b"), + } + ) + items = [(scanner, _scan_result("a")), (scanner, _scan_result("b"))] + + summary = await run_import(items, backend, state) + + assert summary.failed == 1 + assert summary.submitted == 1 + assert not state.is_submitted("claude_code", "a") + assert state.is_submitted("claude_code", "b") + + +class TestBatching: + @pytest.mark.asyncio + async def test_msg_count_limit(self, tmp_path: Path) -> None: + """150 messages -> 2 batches (100 + 50).""" + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + scanner = FakeScanner({"k1": _session(n_msgs=150, session_id="s1", content="x")}) + + await run_import([(scanner, _scan_result("k1"))], backend, state) + + assert len(backend.calls) == 2 + assert len(backend.calls[0]["messages"]) == 100 + assert backend.calls[0]["metadata"]["is_final"] is False + assert len(backend.calls[1]["messages"]) == 50 + assert backend.calls[1]["metadata"]["is_final"] is True + + @pytest.mark.asyncio + async def test_char_limit_fallback(self, tmp_path: Path) -> None: + """5 messages of 8000 chars each = 40K total -> splits before exceeding 30K.""" + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + big_content = "x" * 8000 + scanner = FakeScanner({"k1": _session(n_msgs=5, session_id="s1", content=big_content)}) + + await run_import([(scanner, _scan_result("k1"))], backend, state) + + assert len(backend.calls) >= 2 + for call in backend.calls[:-1]: + assert call["metadata"]["is_final"] is False + assert backend.calls[-1]["metadata"]["is_final"] is True + + @pytest.mark.asyncio + async def test_is_final_only_on_last_batch(self, tmp_path: Path) -> None: + """Exactly 100 messages -> 1 batch with is_final=True.""" + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + scanner = FakeScanner({"k1": _session(n_msgs=100, session_id="s1", content="x")}) + + await run_import([(scanner, _scan_result("k1"))], backend, state) + + assert len(backend.calls) == 1 + assert backend.calls[0]["metadata"]["is_final"] is True + + @pytest.mark.asyncio + async def test_empty_session_no_store(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + empty = ImportSession(session_id="s", messages=()) + scanner = FakeScanner({"k1": empty}) + + summary = await run_import([(scanner, _scan_result("k1"))], backend, state) + + assert backend.calls == [] + assert summary.submitted == 1 + assert state.is_submitted("claude_code", "k1") + + +class TestMessageConversion: + @pytest.mark.asyncio + async def test_tool_calls_pass_through(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + tc = ({"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}},) + msg = _msg(role="assistant", content="thinking", tool_calls=tc, sender="assistant") + session = ImportSession(session_id="s", messages=(msg,)) + scanner = FakeScanner({"k1": session}) + + await run_import([(scanner, _scan_result("k1"))], backend, state) + + stored = backend.calls[0]["messages"][0] + assert stored["tool_calls"] == [tc[0]] + + @pytest.mark.asyncio + async def test_tool_call_id_pass_through(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + msg = _msg(role="tool", content="result", tool_call_id="call_1") + session = ImportSession(session_id="s", messages=(msg,)) + scanner = FakeScanner({"k1": session}) + + await run_import([(scanner, _scan_result("k1"))], backend, state) + + stored = backend.calls[0]["messages"][0] + assert stored["tool_call_id"] == "call_1" + + @pytest.mark.asyncio + async def test_no_tool_fields_when_absent(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + msg = _msg(role="user", content="hi") + session = ImportSession(session_id="s", messages=(msg,)) + scanner = FakeScanner({"k1": session}) + + await run_import([(scanner, _scan_result("k1"))], backend, state) + + stored = backend.calls[0]["messages"][0] + assert "tool_calls" not in stored + assert "tool_call_id" not in stored + + +class TestMetadata: + @pytest.mark.asyncio + async def test_metadata_contains_is_final_only(self, tmp_path: Path) -> None: + """app_id/project_id are deliberately omitted so EverOS defaults + to 'default'/'default', matching the daily recall partition.""" + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + scanner = FakeScanner({"k1": _session(n_msgs=1, session_id="s1")}) + + await run_import([(scanner, _scan_result("k1"))], backend, state) + + meta = backend.calls[0]["metadata"] + assert "app_id" not in meta + assert "project_id" not in meta + assert meta["is_final"] is True + + +class TestOnProgress: + @pytest.mark.asyncio + async def test_callback_called_per_item(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + scanner = FakeScanner( + { + "a": _session(n_msgs=1, session_id="sa"), + "b": _session(n_msgs=1, session_id="sb"), + } + ) + items = [(scanner, _scan_result("a")), (scanner, _scan_result("b"))] + events: list[ProgressEvent] = [] + + await run_import(items, backend, state, on_progress=events.append) + + assert len(events) == 2 + assert events[0] == ProgressEvent( + platform="claude_code", + source_key="a", + status="submitted", + current=1, + total=2, + error=None, + ) + assert events[1] == ProgressEvent( + platform="claude_code", + source_key="b", + status="submitted", + current=2, + total=2, + error=None, + ) + + @pytest.mark.asyncio + async def test_callback_reports_skipped_and_failed(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + state.mark_submitted("claude_code", "a") + backend = FakeBackend() + scanner = FakeScanner( + {"c": _session(n_msgs=1, session_id="sc")}, + fail_on={"b"}, + ) + items = [ + (scanner, _scan_result("a")), + (scanner, _scan_result("b")), + (scanner, _scan_result("c")), + ] + events: list[ProgressEvent] = [] + + await run_import(items, backend, state, on_progress=events.append) + + assert events[0].status == "skipped" + assert events[1].status == "failed" + assert events[1].error is not None + assert events[2].status == "submitted" + + @pytest.mark.asyncio + async def test_no_callback_does_not_error(self, tmp_path: Path) -> None: + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + scanner = FakeScanner({"a": _session(n_msgs=1, session_id="sa")}) + + summary = await run_import([(scanner, _scan_result("a"))], backend, state) + + assert summary.submitted == 1 + + +class TestCancel: + @pytest.mark.asyncio + async def test_cancel_stops_before_next_item(self, tmp_path: Path) -> None: + """When cancel file exists before loop starts, no items are processed.""" + cancel = tmp_path / "import_cancel" + cancel.touch() + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + scanner = FakeScanner({"a": _session(n_msgs=1, session_id="sa"), "b": _session(n_msgs=1, session_id="sb")}) + items = [(scanner, _scan_result("a")), (scanner, _scan_result("b"))] + + summary = await run_import(items, backend, state, cancel_path=cancel) + + assert summary.cancelled is True + assert summary.submitted == 0 + assert summary.total == 2 + assert backend.calls == [] + + @pytest.mark.asyncio + async def test_cancel_mid_import(self, tmp_path: Path) -> None: + """Cancel file created after first item completes stops before second.""" + cancel = tmp_path / "import_cancel" + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + scanner = FakeScanner({"a": _session(n_msgs=1, session_id="sa"), "b": _session(n_msgs=1, session_id="sb")}) + items = [(scanner, _scan_result("a")), (scanner, _scan_result("b"))] + + original_store = backend.store + + async def _store_then_cancel(*args: Any, **kwargs: Any) -> None: + await original_store(*args, **kwargs) + cancel.touch() + + backend.store = _store_then_cancel + + summary = await run_import(items, backend, state, cancel_path=cancel) + + assert summary.cancelled is True + assert summary.submitted == 1 + assert summary.total == 2 + + @pytest.mark.asyncio + async def test_no_cancel_path_runs_normally(self, tmp_path: Path) -> None: + """Without cancel_path, import runs all items to completion.""" + state = ImportState(path=tmp_path / "state.json") + backend = FakeBackend() + scanner = FakeScanner({"a": _session(n_msgs=1, session_id="sa")}) + + summary = await run_import([(scanner, _scan_result("a"))], backend, state) + + assert summary.cancelled is False + assert summary.submitted == 1 diff --git a/tests/test_importer_state.py b/tests/test_importer_state.py new file mode 100644 index 0000000..2165ab4 --- /dev/null +++ b/tests/test_importer_state.py @@ -0,0 +1,100 @@ +"""Tests for raven.importer.state.ImportState.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from raven.importer import ImportState + + +@pytest.fixture +def state_path(tmp_path: Path) -> Path: + return tmp_path / "import_state.json" + + +@pytest.fixture +def state(state_path: Path) -> ImportState: + return ImportState(path=state_path) + + +class TestImportState: + def test_initial_state_empty(self, state: ImportState) -> None: + assert not state.is_submitted("claude_code", "some-key") + + def test_mark_submitted(self, state: ImportState) -> None: + state.mark_submitted("claude_code", "k1") + assert state.is_submitted("claude_code", "k1") + + def test_mark_failed_not_submitted(self, state: ImportState) -> None: + state.mark_failed("codex", "k2", "parse error") + assert not state.is_submitted("codex", "k2") + + def test_failed_then_submitted(self, state: ImportState) -> None: + state.mark_failed("hermes", "k3", "timeout") + assert not state.is_submitted("hermes", "k3") + state.mark_submitted("hermes", "k3") + assert state.is_submitted("hermes", "k3") + + def test_persistence_across_instances(self, state_path: Path) -> None: + s1 = ImportState(path=state_path) + s1.mark_submitted("openclaw", "k4") + + s2 = ImportState(path=state_path) + assert s2.is_submitted("openclaw", "k4") + + def test_get_progress_structure(self, state: ImportState) -> None: + state.mark_submitted("claude_code", "a") + state.mark_failed("codex", "b", "err") + progress = state.get_progress() + entries = progress["entries"] + assert entries["claude_code:a"]["status"] == "submitted" + assert entries["codex:b"]["status"] == "failed" + assert entries["codex:b"]["error"] == "err" + + def test_corrupt_json_recovery(self, state_path: Path) -> None: + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text("{invalid json", encoding="utf-8") + + s = ImportState(path=state_path) + assert not s.is_submitted("any", "key") + assert state_path.with_suffix(".json.corrupt").exists() + + def test_missing_file_returns_empty(self, tmp_path: Path) -> None: + s = ImportState(path=tmp_path / "nonexistent" / "state.json") + assert s.get_summary() == {"total": 0, "submitted": 0, "failed": 0} + + def test_atomic_write_creates_parent_dirs(self, tmp_path: Path) -> None: + deep_path = tmp_path / "a" / "b" / "import_state.json" + s = ImportState(path=deep_path) + s.mark_submitted("kimicode", "k5") + assert deep_path.exists() + + def test_key_format(self, state: ImportState) -> None: + state.mark_submitted("claude_code", "proj-memory") + entries = state.get_progress()["entries"] + assert "claude_code:proj-memory" in entries + + def test_set_total_and_get_summary(self, state: ImportState) -> None: + state.set_total(5) + state.mark_submitted("claude_code", "a") + state.mark_submitted("claude_code", "b") + state.mark_failed("codex", "c", "err") + summary = state.get_summary() + assert summary == {"total": 5, "submitted": 2, "failed": 1} + + def test_get_summary_without_set_total(self, state: ImportState) -> None: + state.mark_submitted("claude_code", "a") + state.mark_failed("codex", "b", "err") + summary = state.get_summary() + assert summary["total"] == 2 + assert summary["submitted"] == 1 + assert summary["failed"] == 1 + + def test_meta_separated_from_entries(self, state: ImportState) -> None: + state.set_total(10) + state.mark_submitted("hermes", "x") + summary = state.get_summary() + assert summary["submitted"] == 1 + assert summary["total"] == 10 diff --git a/tests/test_importer_types.py b/tests/test_importer_types.py new file mode 100644 index 0000000..aa42c56 --- /dev/null +++ b/tests/test_importer_types.py @@ -0,0 +1,191 @@ +"""Tests for raven.importer types and Scanner protocol.""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path + +import pytest + +from raven.importer import ( + ImportMessage, + ImportSession, + Platform, + Scanner, + ScanResult, + SourceKind, + Tier, +) + +# --------------------------------------------------------------------------- +# Enums +# --------------------------------------------------------------------------- + + +class TestPlatform: + def test_values(self) -> None: + assert set(Platform) == { + Platform.CLAUDE_CODE, + Platform.CODEX, + Platform.KIMICODE, + Platform.HERMES, + Platform.OPENCLAW, + } + + def test_str_serialization(self) -> None: + assert str(Platform.CLAUDE_CODE) == "claude_code" + assert str(Platform.KIMICODE) == "kimicode" + + +class TestSourceKind: + def test_values(self) -> None: + assert set(SourceKind) == { + SourceKind.MEMORY_FILE, + SourceKind.CONVERSATION, + } + + +class TestTier: + def test_values(self) -> None: + assert set(Tier) == {Tier.MEMORY_FILES, Tier.FULL} + + def test_str_serialization(self) -> None: + assert str(Tier.FULL) == "full" + + +# --------------------------------------------------------------------------- +# Frozen dataclasses +# --------------------------------------------------------------------------- + + +class TestImportMessage: + def test_required_fields(self) -> None: + msg = ImportMessage( + role="user", + content="hello", + timestamp=1720000000000, + sender_id="alice", + ) + assert msg.role == "user" + assert msg.content == "hello" + assert msg.timestamp == 1720000000000 + assert msg.sender_id == "alice" + + def test_tool_calls_default_none(self) -> None: + msg = ImportMessage(role="assistant", content="ok", timestamp=0, sender_id="bot") + assert msg.tool_calls is None + + def test_tool_calls_optional(self) -> None: + tc = ({"id": "1", "type": "function", "function": {"name": "f"}},) + msg = ImportMessage( + role="assistant", + content="", + timestamp=0, + sender_id="bot", + tool_calls=tc, + ) + assert msg.tool_calls == tc + + def test_tool_call_id_default_none(self) -> None: + msg = ImportMessage(role="user", content="x", timestamp=0, sender_id="u") + assert msg.tool_call_id is None + + def test_tool_call_id_on_tool_message(self) -> None: + msg = ImportMessage(role="tool", content="result", timestamp=0, sender_id="u", tool_call_id="t1") + assert msg.tool_call_id == "t1" + + def test_frozen(self) -> None: + msg = ImportMessage(role="user", content="x", timestamp=0, sender_id="u") + with pytest.raises(dataclasses.FrozenInstanceError): + msg.content = "y" # type: ignore[misc] + + +class TestImportSession: + def test_required_fields(self) -> None: + sess = ImportSession(session_id="s1") + assert sess.session_id == "s1" + + def test_messages_default_empty(self) -> None: + sess = ImportSession(session_id="s") + assert sess.messages == () + + def test_messages_are_tuple(self) -> None: + msg = ImportMessage(role="user", content="hi", timestamp=0, sender_id="u") + sess = ImportSession(session_id="s", messages=(msg,)) + assert isinstance(sess.messages, tuple) + assert len(sess.messages) == 1 + + def test_frozen(self) -> None: + sess = ImportSession(session_id="s") + with pytest.raises(dataclasses.FrozenInstanceError): + sess.session_id = "b" # type: ignore[misc] + + +class TestScanResult: + def test_all_fields(self) -> None: + r = ScanResult( + source_key="proj-memory", + platform=Platform.CLAUDE_CODE, + kind=SourceKind.MEMORY_FILE, + file_paths=(Path("/a/b.md"), Path("/a/c.md")), + estimated_size=4096, + mtime=1720000000.0, + ) + assert r.source_key == "proj-memory" + assert r.platform == Platform.CLAUDE_CODE + assert r.kind == SourceKind.MEMORY_FILE + assert len(r.file_paths) == 2 + assert r.estimated_size == 4096 + + def test_file_paths_is_tuple(self) -> None: + r = ScanResult( + source_key="k", + platform=Platform.CODEX, + kind=SourceKind.CONVERSATION, + file_paths=(Path("/x.jsonl"),), + estimated_size=0, + mtime=0.0, + ) + assert isinstance(r.file_paths, tuple) + + def test_frozen(self) -> None: + r = ScanResult( + source_key="k", + platform=Platform.HERMES, + kind=SourceKind.CONVERSATION, + file_paths=(), + estimated_size=0, + mtime=0.0, + ) + with pytest.raises(dataclasses.FrozenInstanceError): + r.source_key = "other" # type: ignore[misc] + + +# --------------------------------------------------------------------------- +# Scanner Protocol +# --------------------------------------------------------------------------- + + +class _CompleteScanner: + platform = Platform.CLAUDE_CODE + + async def scan(self) -> list[ScanResult]: + return [] + + async def read(self, result: ScanResult) -> ImportSession: + return ImportSession(session_id="s") + + +class _IncompleteScanner: + platform = Platform.CODEX + + async def scan(self) -> list[ScanResult]: + return [] + + +class TestScannerProtocol: + def test_complete_scanner_satisfies_protocol(self) -> None: + assert isinstance(_CompleteScanner(), Scanner) + + def test_incomplete_scanner_fails_protocol(self) -> None: + assert not isinstance(_IncompleteScanner(), Scanner) diff --git a/tests/test_memory_backend_contract.py b/tests/test_memory_backend_contract.py index 46f6a00..793e931 100644 --- a/tests/test_memory_backend_contract.py +++ b/tests/test_memory_backend_contract.py @@ -75,6 +75,8 @@ async def store( self, session_id: str, messages: list[dict[str, Any]], + *, + metadata: dict[str, Any] | None = None, ) -> None: self._sessions.setdefault(session_id, []).extend(messages) diff --git a/tests/test_memory_backend_protocol.py b/tests/test_memory_backend_protocol.py index 3269d14..11b558b 100644 --- a/tests/test_memory_backend_protocol.py +++ b/tests/test_memory_backend_protocol.py @@ -54,7 +54,7 @@ class _CompleteBackend: async def recall(self, query, *, user_id=None, agent_id=None, top_k): return [Memory(text=f"hit:{query}", score=0.5)] - async def store(self, session_id, messages): + async def store(self, session_id, messages, *, metadata=None): return None async def feedback(self, signals): @@ -73,7 +73,7 @@ class _IncompleteBackend: async def recall(self, query, *, user_id=None, agent_id=None, top_k): return [] - async def store(self, session_id, messages): + async def store(self, session_id, messages, *, metadata=None): return None async def start(self): @@ -139,7 +139,7 @@ class _EmptyRecallBackend: async def recall(self, query, *, user_id=None, agent_id=None, top_k): return [] - async def store(self, session_id, messages): + async def store(self, session_id, messages, *, metadata=None): pass async def feedback(self, signals): diff --git a/tests/test_phase_a_default_engine.py b/tests/test_phase_a_default_engine.py index bf9ec36..505c4a3 100644 --- a/tests/test_phase_a_default_engine.py +++ b/tests/test_phase_a_default_engine.py @@ -51,7 +51,7 @@ async def stop(self): async def feedback(self, signals): pass - async def store(self, session_id, messages): + async def store(self, session_id, messages, *, metadata=None): pass async def recall(self, query, *, user_id=None, agent_id=None, top_k): diff --git a/tests/test_skill_router_sr4.py b/tests/test_skill_router_sr4.py index d6d5934..e1355af 100644 --- a/tests/test_skill_router_sr4.py +++ b/tests/test_skill_router_sr4.py @@ -34,7 +34,7 @@ async def stop(self) -> None: async def feedback(self, signals) -> None: pass - async def store(self, session_id, messages) -> None: + async def store(self, session_id, messages, *, metadata=None) -> None: pass async def recall(self, query, *, user_id=None, agent_id=None, top_k): diff --git a/uv.lock b/uv.lock index dc01265..7c8fc5e 100644 --- a/uv.lock +++ b/uv.lock @@ -952,7 +952,7 @@ wheels = [ [[package]] name = "everos" -version = "1.1.2" +version = "1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiosqlite" }, @@ -982,9 +982,9 @@ dependencies = [ { name = "watchdog" }, { name = "watchfiles" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8c/61/e6e250d1bd5b64cffdbe8cb1b849573b2d450472466c830fe42b1613655b/everos-1.1.2.tar.gz", hash = "sha256:4c1a41158e45d4017dfb807756f372c537180ce167c76c06a0251d1c5da1fb8c", size = 1187075, upload-time = "2026-07-07T14:45:01.626Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/49/f62944d4355e438417924c12cbad5ba3bfd03fffe078aa9be740f8ee5a54/everos-1.1.3.tar.gz", hash = "sha256:57a3365748d63780cb375b98b7480a5c01655569f7429a409717be58577c514d", size = 1190171, upload-time = "2026-07-10T12:34:54.591Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/fd/6cac7f188e55a118f41bad7b7e248e08b38be44ea0e274959f00cd0bc490/everos-1.1.2-py3-none-any.whl", hash = "sha256:babeddc5b77ee832d710fda7f41efd2b6dac5cb674dd53021700ec199e89dda3", size = 468521, upload-time = "2026-07-07T14:45:00.096Z" }, + { url = "https://files.pythonhosted.org/packages/17/f4/5850f64499f0cdb12fe2cb8a4a7500b9ad5397609801f314921954e04cba/everos-1.1.3-py3-none-any.whl", hash = "sha256:f54086f9d4e52420eab70030dc8c92b76852c5b5e40d8f485226078f0f78fed0", size = 470289, upload-time = "2026-07-10T12:34:53.092Z" }, ] [package.optional-dependencies] @@ -3375,7 +3375,7 @@ requires-dist = [ { name = "boxlite", marker = "extra == 'sandbox'", specifier = "==0.9.5" }, { name = "croniter", specifier = ">=6.0.0,<7.0.0" }, { name = "dingtalk-stream", marker = "extra == 'channel-dingtalk'", specifier = ">=0.24.0,<1.0.0" }, - { name = "everos", extras = ["multimodal"], specifier = "==1.1.2" }, + { name = "everos", extras = ["multimodal"], specifier = "==1.1.3" }, { name = "httpx", specifier = ">=0.28.0,<1.0.0" }, { name = "huggingface-hub", marker = "extra == 'eval'", specifier = ">=1.11.0" }, { name = "json-repair", specifier = ">=0.30.0" },