From fa4d915af5c5388a3016f3c12ff3032f0d080d2c Mon Sep 17 00:00:00 2001 From: us Date: Mon, 13 Jul 2026 23:03:31 +0300 Subject: [PATCH 1/6] test(python): mock current search transport --- sdks/python/tests/test_client_unit.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/sdks/python/tests/test_client_unit.py b/sdks/python/tests/test_client_unit.py index ff9a298c..d3508161 100644 --- a/sdks/python/tests/test_client_unit.py +++ b/sdks/python/tests/test_client_unit.py @@ -195,16 +195,20 @@ def test_search_http_sends_query(self) -> None: client = CrwClient(api_url="https://fastcrw.com/api", api_key="fc-test") mock_response = [{"url": "https://example.com", "title": "Example"}] - with patch.object(client, "_http_post", return_value=mock_response) as mock_post: + with patch.object( + client, + "_http_request", + return_value={"success": True, "data": mock_response}, + ) as mock_request: result = client.search("python web scraping", limit=5) - mock_post.assert_called_once() - call_args = mock_post.call_args - assert call_args[0][0] == "/v1/search" - body = call_args[0][1] - assert body["query"] == "python web scraping" - assert body["limit"] == 5 - assert result == mock_response + mock_request.assert_called_once_with( + "POST", + "/v1/search", + {"query": "python web scraping", "limit": 5}, + raw=True, + ) + assert list(result) == mock_response # --------------------------------------------------------------------------- From 2218511a4095d6dd4322937d08a9997d505f56d5 Mon Sep 17 00:00:00 2001 From: us Date: Mon, 13 Jul 2026 23:35:15 +0300 Subject: [PATCH 2/6] chore(release): synchronize workspace lockfile --- Cargo.lock | 293 ++++++++++++----------------------------------------- 1 file changed, 62 insertions(+), 231 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f79926d..0ebdbba3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -448,9 +448,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.66" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "jobserver", @@ -693,16 +693,6 @@ dependencies = [ "url", ] -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation" version = "0.10.1" @@ -746,12 +736,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "critical-section" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" - [[package]] name = "crossbeam-channel" version = "0.5.16" @@ -794,7 +778,7 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crw-browse" -version = "0.23.0" +version = "0.24.0" dependencies = [ "anyhow", "async-trait", @@ -824,7 +808,7 @@ dependencies = [ [[package]] name = "crw-cli" -version = "0.23.0" +version = "0.24.0" dependencies = [ "anyhow", "axum", @@ -859,7 +843,7 @@ dependencies = [ [[package]] name = "crw-core" -version = "0.23.0" +version = "0.24.0" dependencies = [ "config", "crw-mcp-proto", @@ -878,7 +862,7 @@ dependencies = [ [[package]] name = "crw-crawl" -version = "0.23.0" +version = "0.24.0" dependencies = [ "crw-core", "crw-diff", @@ -904,7 +888,7 @@ dependencies = [ [[package]] name = "crw-diff" -version = "0.23.0" +version = "0.24.0" dependencies = [ "crw-core", "hex", @@ -919,7 +903,7 @@ dependencies = [ [[package]] name = "crw-extract" -version = "0.23.0" +version = "0.24.0" dependencies = [ "crw-core", "ego-tree", @@ -953,7 +937,7 @@ dependencies = [ [[package]] name = "crw-mcp" -version = "0.23.0" +version = "0.24.0" dependencies = [ "base64", "clap", @@ -969,7 +953,7 @@ dependencies = [ [[package]] name = "crw-mcp-proto" -version = "0.23.0" +version = "0.24.0" dependencies = [ "jsonschema", "serde", @@ -979,7 +963,7 @@ dependencies = [ [[package]] name = "crw-renderer" -version = "0.23.0" +version = "0.24.0" dependencies = [ "async-trait", "axum", @@ -1008,7 +992,7 @@ dependencies = [ [[package]] name = "crw-search" -version = "0.23.0" +version = "0.24.0" dependencies = [ "crw-core", "futures", @@ -1026,7 +1010,7 @@ dependencies = [ [[package]] name = "crw-server" -version = "0.23.0" +version = "0.24.0" dependencies = [ "axum", "axum-test", @@ -1867,76 +1851,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hickory-net" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" -dependencies = [ - "async-trait", - "cfg-if", - "data-encoding", - "futures-channel", - "futures-io", - "futures-util", - "hickory-proto", - "idna", - "ipnet", - "jni", - "rand 0.10.2", - "thiserror 2.0.18", - "tinyvec", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "hickory-proto" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" -dependencies = [ - "data-encoding", - "idna", - "ipnet", - "jni", - "once_cell", - "prefix-trie", - "rand 0.10.2", - "ring", - "thiserror 2.0.18", - "tinyvec", - "tracing", - "url", -] - -[[package]] -name = "hickory-resolver" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" -dependencies = [ - "cfg-if", - "futures-util", - "hickory-net", - "hickory-proto", - "ipconfig", - "ipnet", - "jni", - "moka", - "ndk-context", - "once_cell", - "parking_lot", - "rand 0.10.2", - "resolv-conf", - "smallvec", - "system-configuration", - "thiserror 2.0.18", - "tokio", - "tracing", -] - [[package]] name = "htmd" version = "0.5.4" @@ -1994,9 +1908,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -2004,9 +1918,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -2284,27 +2198,11 @@ dependencies = [ "rustversion", ] -[[package]] -name = "ipconfig" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" -dependencies = [ - "socket2", - "widestring", - "windows-registry", - "windows-result", - "windows-sys 0.61.2", -] - [[package]] name = "ipnet" version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -dependencies = [ - "serde", -] [[package]] name = "is_terminal_polyfill" @@ -2320,9 +2218,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" +checksum = "961d16382652bfdd8c6f68b223b26a8c93e0d475c672f414411db31c6c5c900e" dependencies = [ "defmt", "jiff-static", @@ -2336,9 +2234,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" +checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" dependencies = [ "proc-macro2", "quote", @@ -2347,9 +2245,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6142247df1a93c2b3587402a19710be3e6e942f1581a1702e76408f2c21d6590" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" [[package]] name = "jiff-tzdb-platform" @@ -2708,9 +2606,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -2754,12 +2652,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "ndk-context" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" - [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -2883,10 +2775,6 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -dependencies = [ - "critical-section", - "portable-atomic", -] [[package]] name = "once_cell_polyfill" @@ -2965,9 +2853,9 @@ checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pdf-inspector" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "156c914888fd0efdfc53911208b74c0adf00e9322b264ec5bfe10fd320f8ebae" +checksum = "a814c9c6a4f520b34789e4519969b4c77325d2a757f13655091ef5f603e25d4f" dependencies = [ "env_logger", "log", @@ -3092,7 +2980,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared 0.11.3", - "rand 0.8.6", + "rand 0.8.7", ] [[package]] @@ -3225,17 +3113,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" -[[package]] -name = "prefix-trie" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" -dependencies = [ - "either", - "ipnet", - "num-traits", -] - [[package]] name = "pretty_assertions" version = "1.4.1" @@ -3280,7 +3157,7 @@ dependencies = [ "bit-vec", "bitflags 2.13.0", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "rand_chacha", "rand_xorshift", "regex-syntax", @@ -3411,18 +3288,18 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "rand_core 0.6.4", ] [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha", "rand_core 0.9.5", @@ -3573,9 +3450,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.4" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" dependencies = [ "aho-corasick", "memchr", @@ -3585,9 +3462,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" dependencies = [ "aho-corasick", "memchr", @@ -3614,7 +3491,6 @@ dependencies = [ "futures-core", "futures-util", "h2", - "hickory-resolver", "http", "http-body", "http-body-util", @@ -3624,7 +3500,6 @@ dependencies = [ "js-sys", "log", "mime_guess", - "once_cell", "percent-encoding", "pin-project-lite", "quinn", @@ -3656,12 +3531,6 @@ dependencies = [ "thiserror 2.0.18", ] -[[package]] -name = "resolv-conf" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" - [[package]] name = "ring" version = "0.17.14" @@ -3780,9 +3649,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "once_cell", @@ -3820,7 +3689,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ - "core-foundation 0.10.1", + "core-foundation", "core-foundation-sys", "jni", "log", @@ -3949,7 +3818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags 2.13.0", - "core-foundation 0.10.1", + "core-foundation", "core-foundation-sys", "libc", "security-framework-sys", @@ -4118,9 +3987,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", "cpufeatures 0.2.17", @@ -4237,9 +4106,9 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -4247,9 +4116,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" [[package]] name = "stable_deref_trait" @@ -4392,27 +4261,6 @@ dependencies = [ "syn", ] -[[package]] -name = "system-configuration" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" -dependencies = [ - "bitflags 2.13.0", - "core-foundation 0.9.4", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "tagptr" version = "0.2.0" @@ -4494,9 +4342,9 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] @@ -4552,9 +4400,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -4838,7 +4686,7 @@ dependencies = [ "http", "httparse", "log", - "rand 0.9.4", + "rand 0.9.5", "rustls", "rustls-pki-types", "sha1", @@ -4995,9 +4843,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -5191,12 +5039,6 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" -[[package]] -name = "widestring" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" - [[package]] name = "winapi-util" version = "0.1.11" @@ -5247,17 +5089,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - [[package]] name = "windows-result" version = "0.4.1" @@ -5360,9 +5191,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" dependencies = [ "memchr", ] @@ -5460,18 +5291,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.53" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75726053136156d419e285b9b7eddaaea9e3fea6ce32eed44a89901f0bd98de1" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.53" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", @@ -5540,6 +5371,6 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" From 18ec2e20e9b368a0abfd36b68229df79b8c1d344 Mon Sep 17 00:00:00 2001 From: us Date: Tue, 14 Jul 2026 00:09:57 +0300 Subject: [PATCH 3/6] feat(extract): add canonical cancellation lifecycle --- crates/crw-mcp-proto/src/lib.rs | 85 +++++-- crates/crw-mcp/README.md | 1 + crates/crw-mcp/src/main.rs | 17 ++ crates/crw-server/openapi/openapi-3.0.json | 69 +++++- crates/crw-server/openapi/openapi.json | 69 +++++- crates/crw-server/src/routes/extract.rs | 57 ++++- crates/crw-server/src/routes/mcp.rs | 40 ++- crates/crw-server/src/routes/v1/mod.rs | 4 +- crates/crw-server/src/state.rs | 274 ++++++++++++++------ crates/crw-server/tests/mcp.rs | 90 ++++++- crates/crw-server/tests/v1_extract.rs | 275 ++++++++++++++++++++- docs/agent-onboarding/SKILL.md | 11 + docs/docs/extract.md | 30 +++ docs/docs/mcp-clients.md | 12 +- docs/docs/mcp.md | 5 +- docs/openapi-3.0.json | 69 +++++- docs/openapi.json | 69 +++++- mcp/crw-mcp/skills/SKILL.md | 11 + sdks/python/README.md | 9 + sdks/python/src/crw/__init__.py | 22 +- sdks/python/src/crw/client.py | 140 ++++++++--- sdks/python/src/crw/exceptions.py | 14 ++ sdks/python/src/crw/types.py | 68 +++++ sdks/python/tests/test_client_unit.py | 99 +++++++- sdks/typescript/README.md | 9 + sdks/typescript/src/client.ts | 105 ++++++-- sdks/typescript/src/errors.ts | 15 ++ sdks/typescript/src/index.ts | 17 +- sdks/typescript/src/types.ts | 42 +++- sdks/typescript/test/client.test.ts | 121 ++++++++- 30 files changed, 1628 insertions(+), 221 deletions(-) create mode 100644 sdks/python/src/crw/types.py diff --git a/crates/crw-mcp-proto/src/lib.rs b/crates/crw-mcp-proto/src/lib.rs index 58ac8f3a..1290ba52 100644 --- a/crates/crw-mcp-proto/src/lib.rs +++ b/crates/crw-mcp-proto/src/lib.rs @@ -88,6 +88,34 @@ impl JsonRpcResponse { // --- Tool definitions --- +fn extract_accepted_output_schema() -> Value { + json!({ + "type": "object", + "properties": { + "id": { "type": "string" }, + "status": { "type": "string", "enum": ["processing"] }, + "urls": { "type": "integer", "minimum": 0 } + }, + "required": ["id", "status", "urls"] + }) +} + +fn extract_status_output_schema() -> Value { + json!({ + "type": "object", + "properties": { + "id": { "type": "string" }, + "status": { "type": "string" }, + "results": { "type": "array", "items": { "type": "object" } }, + "error": { "type": "string" }, + "expiresAt": { "type": "string" }, + "creditsUsed": { "type": "integer" }, + "tokensUsed": { "type": "integer" } + }, + "required": ["id", "status", "results", "expiresAt", "creditsUsed", "tokensUsed"] + }) +} + pub fn tool_definitions(proxy_mode: bool) -> Value { let mut tools = vec![ json!({ @@ -298,7 +326,8 @@ pub fn tool_definitions(proxy_mode: bool) -> Value { "llmModel": { "type": "string" } }, "required": ["urls"] - } + }, + "outputSchema": extract_accepted_output_schema() }), json!({ "name": "crw_check_extract_status", @@ -316,7 +345,27 @@ pub fn tool_definitions(proxy_mode: bool) -> Value { "id": { "type": "string", "description": "Extract job id from crw_extract" } }, "required": ["id"] - } + }, + "outputSchema": extract_status_output_schema() + }), + json!({ + "name": "crw_cancel_extract", + "title": "Cancel extract job", + "description": "Request cancellation of an extract job. Returns the canonical status; cancelling remains non-terminal until the claimed URL settles.", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": true, + "openWorldHint": true + }, + "inputSchema": { + "type": "object", + "properties": { + "id": { "type": "string", "description": "Extract job id from crw_extract" } + }, + "required": ["id"] + }, + "outputSchema": extract_status_output_schema() }), ]; @@ -821,12 +870,11 @@ mod tests { /// /// Baseline before the Phase 1 trim was 8233 bytes (~2744 est-tok). After the /// Phase 1 trim + Phase 3 annotations/titles + the two native extract tools the - /// full 8-tool list is ~8017 bytes (~2673 est-tok). The ceiling is floor + ~3% - /// headroom so the gate catches real bloat without churning on minor edits. - // Raised from 2300 when the two native extract tools (crw_extract + - // crw_check_extract_status) were added — proportional to the existing - // per-tool footprint, not description bloat. - const TOOLS_LIST_TOKEN_CEILING: usize = 2750; + /// full 8-tool list was ~8017 bytes (~2673 est-tok). The canonical lifecycle + /// adds one cancel tool plus required output schemas for start/status/cancel; + /// after minimizing those schemas, the 9-tool list is ~9621 bytes (~3207 + /// est-tok). The ceiling keeps ~1.3% headroom so further growth still fails. + const TOOLS_LIST_TOKEN_CEILING: usize = 3250; #[test] fn tools_list_token_budget() { @@ -1306,20 +1354,14 @@ mod tests { } /// A1 — every tool advertises annotations + a title; crw_crawl and crw_extract - /// are the non-read-only / non-idempotent tools; crw_parse_file is the only closed-world. + /// are non-idempotent, while cancel is destructive but idempotent. #[test] fn a1_tools_advertise_annotations_and_title() { let defs = tool_definitions(false); for t in defs["tools"].as_array().unwrap() { assert!(t["annotations"].is_object(), "{} annotations", t["name"]); assert!(t["title"].is_string(), "{} title", t["name"]); - // destructiveHint is explicitly false everywhere (the JSON default is true). - assert_eq!( - t["annotations"]["destructiveHint"], - json!(false), - "{}", - t["name"] - ); + assert!(t["annotations"]["destructiveHint"].is_boolean()); } let crawl = tool_by_name(&defs, "crw_crawl"); assert_eq!(crawl["annotations"]["readOnlyHint"], json!(false)); @@ -1328,6 +1370,10 @@ mod tests { let extract = tool_by_name(&defs, "crw_extract"); assert_eq!(extract["annotations"]["readOnlyHint"], json!(false)); assert_eq!(extract["annotations"]["idempotentHint"], json!(false)); + let cancel = tool_by_name(&defs, "crw_cancel_extract"); + assert_eq!(cancel["annotations"]["readOnlyHint"], json!(false)); + assert_eq!(cancel["annotations"]["destructiveHint"], json!(true)); + assert_eq!(cancel["annotations"]["idempotentHint"], json!(true)); let scrape = tool_by_name(&defs, "crw_scrape"); assert_eq!(scrape["annotations"]["readOnlyHint"], json!(true)); assert_eq!(scrape["annotations"]["openWorldHint"], json!(true)); @@ -1335,7 +1381,7 @@ mod tests { assert_eq!(parse["annotations"]["openWorldHint"], json!(false)); } - /// A2 — is_known_tool recognizes all 8 tool names, rejects others. + /// A2 — is_known_tool recognizes all 9 tool names, rejects others. #[test] fn a2_is_known_tool() { for name in [ @@ -1347,6 +1393,7 @@ mod tests { "crw_parse_file", "crw_extract", "crw_check_extract_status", + "crw_cancel_extract", ] { assert!(is_known_tool(name), "{name} should be known"); } @@ -1378,10 +1425,10 @@ mod tests { } let with = list(true); assert!(with.contains(&"crw_search".to_string())); - assert_eq!(with.len(), 8); + assert_eq!(with.len(), 9); let without = list(false); assert!(!without.contains(&"crw_search".to_string())); - assert_eq!(without.len(), 7); + assert_eq!(without.len(), 8); } /// A4 — initialize advertises server usage `instructions` (the model-facing diff --git a/crates/crw-mcp/README.md b/crates/crw-mcp/README.md index 0233d3c9..71fbc643 100644 --- a/crates/crw-mcp/README.md +++ b/crates/crw-mcp/README.md @@ -26,6 +26,7 @@ MCP (Model Context Protocol) server for the [CRW](https://github.com/us/crw) web | `crw_map` | Discover all URLs on a website | | `crw_extract` | Extract structured JSON from URLs via a prompt and/or JSON schema (async job, returns job ID) | | `crw_check_extract_status` | Poll extract job status and retrieve results | +| `crw_cancel_extract` | Idempotently cancel an extract job and retrieve canonical status | | `crw_search` | Search the web (needs a configured search backend; always available in proxy mode, available in embedded mode only when a search backend is configured) | | `crw_parse_file` | Parse a local PDF (base64) to markdown | diff --git a/crates/crw-mcp/src/main.rs b/crates/crw-mcp/src/main.rs index 8d0d791e..2a5bd24d 100644 --- a/crates/crw-mcp/src/main.rs +++ b/crates/crw-mcp/src/main.rs @@ -17,6 +17,7 @@ //! - `crw_search` — web search (embedded: only when a search backend is configured; proxy: forwards to the remote API) //! - `crw_extract` — start an async multi-URL structured extraction job //! - `crw_check_extract_status` — poll extract job status +//! - `crw_cancel_extract` — idempotently cancel an extract job //! - `crw_parse_file` — parse a local PDF (base64) to markdown //! //! # Usage @@ -261,6 +262,8 @@ async fn proxy_call_tool( parse_response(resp).await } "crw_extract" => { + let mut headers = headers; + headers.insert("prefer", "respond-async".parse().unwrap()); let resp = client .post(format!("{base_url}/v1/extract")) .headers(headers) @@ -285,6 +288,20 @@ async fn proxy_call_tool( .map_err(|e| format!("HTTP request failed: {e}"))?; parse_response(resp).await } + "crw_cancel_extract" => { + let id = args + .get("id") + .and_then(|v| v.as_str()) + .ok_or("missing required parameter: id")?; + let resp = client + .delete(format!("{base_url}/v1/extract/{id}")) + .headers(headers) + .timeout(TIMEOUT_CRAWL_STATUS) + .send() + .await + .map_err(|e| format!("HTTP request failed: {e}"))?; + parse_response(resp).await + } "crw_parse_file" => { use base64::Engine; let b64 = args diff --git a/crates/crw-server/openapi/openapi-3.0.json b/crates/crw-server/openapi/openapi-3.0.json index 55d30575..25ae5aa6 100644 --- a/crates/crw-server/openapi/openapi-3.0.json +++ b/crates/crw-server/openapi/openapi-3.0.json @@ -406,8 +406,22 @@ "extract" ], "summary": "Start an async multi-URL structured extraction job", - "description": "Native extraction. Returns a per-URL array via GET /v1/extract/{id}. At least one of `prompt` or `schema` is required.", + "description": "Native extraction. Returns a fixed-cardinality per-URL array via GET /v1/extract/{id}. At least one of `prompt` or `schema` is required. SDK async starters send `Prefer: respond-async` for managed/self-hosted parity.", "operationId": "extractStart", + "parameters": [ + { + "name": "Prefer", + "in": "header", + "required": false, + "schema": { + "type": "string", + "enum": [ + "respond-async" + ] + }, + "description": "Request the canonical async lifecycle across managed and self-hosted deployments." + } + ], "requestBody": { "required": true, "content": { @@ -464,6 +478,46 @@ } } }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + } + } + }, + "delete": { + "tags": [ + "extract" + ], + "summary": "Cancel an extract job", + "description": "Idempotent. Processing becomes cancelling while a claimed URL settles, then cancelled. Completed, failed, and cancelled states are immutable.", + "operationId": "extractCancel", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Extract job id returned by POST /v1/extract" + } + ], + "responses": { + "200": { + "description": "Canonical job status + ordered per-URL results", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExtractStatus" + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, "404": { "$ref": "#/components/responses/NotFound" } @@ -1386,14 +1440,16 @@ "status": { "type": "string", "enum": [ + "processing", "completed", - "failed" + "failed", + "cancelled" ] }, "data": { "type": "object", "additionalProperties": true, - "description": "Extracted JSON object (present when status is completed)." + "description": "Extracted JSON object (present when status is completed). Untouched cancelled slots omit data, error, usage, and basis." }, "error": { "type": "string" @@ -1427,6 +1483,7 @@ "required": [ "id", "status", + "results", "expiresAt", "creditsUsed", "tokensUsed" @@ -1439,8 +1496,10 @@ "type": "string", "enum": [ "processing", + "cancelling", "completed", - "failed" + "failed", + "cancelled" ] }, "results": { @@ -1448,7 +1507,7 @@ "items": { "$ref": "#/components/schemas/ExtractUrlResult" }, - "description": "Per-URL results in original request order." + "description": "Fixed-cardinality per-URL slots in original request order. Pending slots are processing; untouched terminal slots are cancelled." }, "error": { "type": "string", diff --git a/crates/crw-server/openapi/openapi.json b/crates/crw-server/openapi/openapi.json index aebc81aa..2a9ef712 100644 --- a/crates/crw-server/openapi/openapi.json +++ b/crates/crw-server/openapi/openapi.json @@ -777,8 +777,22 @@ "extract" ], "summary": "Start an async multi-URL structured extraction job", - "description": "Native extraction. Returns a per-URL array via GET /v1/extract/{id}. At least one of `prompt` or `schema` is required.", + "description": "Native extraction. Returns a fixed-cardinality per-URL array via GET /v1/extract/{id}. At least one of `prompt` or `schema` is required. SDK async starters send `Prefer: respond-async` for managed/self-hosted parity.", "operationId": "extractStart", + "parameters": [ + { + "name": "Prefer", + "in": "header", + "required": false, + "schema": { + "type": "string", + "enum": [ + "respond-async" + ] + }, + "description": "Request the canonical async lifecycle across managed and self-hosted deployments." + } + ], "requestBody": { "required": true, "content": { @@ -835,6 +849,46 @@ } } }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + } + } + }, + "delete": { + "tags": [ + "extract" + ], + "summary": "Cancel an extract job", + "description": "Idempotent. Processing becomes cancelling while a claimed URL settles, then cancelled. Completed, failed, and cancelled states are immutable.", + "operationId": "extractCancel", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Extract job id returned by POST /v1/extract" + } + ], + "responses": { + "200": { + "description": "Canonical job status + ordered per-URL results", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExtractStatus" + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, "404": { "$ref": "#/components/responses/NotFound" } @@ -2006,14 +2060,16 @@ "status": { "type": "string", "enum": [ + "processing", "completed", - "failed" + "failed", + "cancelled" ] }, "data": { "type": "object", "additionalProperties": true, - "description": "Extracted JSON object (present when status is completed)." + "description": "Extracted JSON object (present when status is completed). Untouched cancelled slots omit data, error, usage, and basis." }, "error": { "type": "string" @@ -2047,6 +2103,7 @@ "required": [ "id", "status", + "results", "expiresAt", "creditsUsed", "tokensUsed" @@ -2059,8 +2116,10 @@ "type": "string", "enum": [ "processing", + "cancelling", "completed", - "failed" + "failed", + "cancelled" ] }, "results": { @@ -2068,7 +2127,7 @@ "items": { "$ref": "#/components/schemas/ExtractUrlResult" }, - "description": "Per-URL results in original request order." + "description": "Fixed-cardinality per-URL slots in original request order. Pending slots are processing; untouched terminal slots are cancelled." }, "error": { "type": "string", diff --git a/crates/crw-server/src/routes/extract.rs b/crates/crw-server/src/routes/extract.rs index 112203ab..533fa8da 100644 --- a/crates/crw-server/src/routes/extract.rs +++ b/crates/crw-server/src/routes/extract.rs @@ -18,7 +18,7 @@ use crw_core::types::{ExtractOptions, LlmUsage, OutputFormat, ScrapeRequest}; use crate::error::AppError; use crate::routes::v2::adapters::expires_at_rfc3339; -use crate::state::{AppState, PreparedUrl, UrlResult}; +use crate::state::{AppState, ExtractRecord, PreparedUrl, UrlResult}; /// Native extract request. camelCase like every other v1 public type. /// NOTE: no `#[derive(Debug)]` — `llm_api_key` is a secret and must never land @@ -268,18 +268,14 @@ pub struct ExtractStatusResponse { pub tokens_used: u32, } -pub async fn get_extract( - State(state): State, - Path(id): Path, -) -> Result, AppError> { - let rec = { - let jobs = state.extract_jobs.read().await; - jobs.get(&id) - .cloned() - .ok_or_else(|| CrwError::NotFound(format!("Extract job {id} not found")))? - }; +/// The one canonical HTTP/MCP serializer for extract lifecycle state. +pub(crate) fn serialize_extract_status( + state: &AppState, + id: Uuid, + rec: ExtractRecord, +) -> ExtractStatusResponse { let expires_at = expires_at_rfc3339(rec.created_at, state.config.crawler.job_ttl_secs); - Ok(Json(ExtractStatusResponse { + ExtractStatusResponse { id: id.to_string(), status: rec.status.as_str().to_string(), results: rec @@ -291,5 +287,40 @@ pub async fn get_extract( expires_at, credits_used: rec.credits_used, tokens_used: rec.tokens_used, - })) + } +} + +pub(crate) async fn get_extract_status( + state: &AppState, + id: Uuid, +) -> Result { + let rec = { + let jobs = state.extract_jobs.read().await; + jobs.get(&id) + .cloned() + .ok_or_else(|| CrwError::NotFound(format!("Extract job {id} not found")))? + }; + Ok(serialize_extract_status(state, id, rec)) +} + +pub(crate) async fn cancel_extract_status( + state: &AppState, + id: Uuid, +) -> Result { + let rec = state.cancel_extract_job(id).await?; + Ok(serialize_extract_status(state, id, rec)) +} + +pub async fn get_extract( + State(state): State, + Path(id): Path, +) -> Result, AppError> { + Ok(Json(get_extract_status(&state, id).await?)) +} + +pub async fn cancel_extract( + State(state): State, + Path(id): Path, +) -> Result, AppError> { + Ok(Json(cancel_extract_status(&state, id).await?)) } diff --git a/crates/crw-server/src/routes/mcp.rs b/crates/crw-server/src/routes/mcp.rs index 14987e38..277199e7 100644 --- a/crates/crw-server/src/routes/mcp.rs +++ b/crates/crw-server/src/routes/mcp.rs @@ -127,7 +127,7 @@ pub async fn call_tool(state: &AppState, tool_name: &str, args: Value) -> Result Ok(json!({"success": true, "id": id.to_string(), "status": "processing", "urls": urls})) } "crw_check_extract_status" => { - use crate::routes::extract::ExtractUrlResult; + use crate::routes::extract::get_extract_status; let id_str = args .get("id") .and_then(|v| v.as_str()) @@ -135,26 +135,24 @@ pub async fn call_tool(state: &AppState, tool_name: &str, args: Value) -> Result let id: Uuid = id_str .parse() .map_err(|_| format!("invalid extract id: {id_str}"))?; - let rec = { - let jobs = state.extract_jobs.read().await; - jobs.get(&id) - .cloned() - .ok_or(format!("extract job {id} not found"))? - }; - let results: Vec = rec - .per_url - .into_iter() - .map(ExtractUrlResult::from) - .collect(); - serde_json::to_value(json!({ - "success": !matches!(rec.status, crate::state::ExtractStatus::Failed), - "status": rec.status.as_str(), - "results": results, - "error": rec.error, - "creditsUsed": rec.credits_used, - "tokensUsed": rec.tokens_used, - })) - .map_err(|e| format!("serialize error: {e}")) + let response = get_extract_status(state, id) + .await + .map_err(|e| format!("{e}"))?; + serde_json::to_value(response).map_err(|e| format!("serialize error: {e}")) + } + "crw_cancel_extract" => { + use crate::routes::extract::cancel_extract_status; + let id_str = args + .get("id") + .and_then(|v| v.as_str()) + .ok_or("missing required parameter: id")?; + let id: Uuid = id_str + .parse() + .map_err(|_| format!("invalid extract id: {id_str}"))?; + let response = cancel_extract_status(state, id) + .await + .map_err(|e| format!("{e}"))?; + serde_json::to_value(response).map_err(|e| format!("serialize error: {e}")) } "crw_parse_file" => { use base64::Engine; diff --git a/crates/crw-server/src/routes/v1/mod.rs b/crates/crw-server/src/routes/v1/mod.rs index c79ca8e9..b8535c12 100644 --- a/crates/crw-server/src/routes/v1/mod.rs +++ b/crates/crw-server/src/routes/v1/mod.rs @@ -45,7 +45,9 @@ pub fn router() -> Router { ) .route( "/v1/extract/{id}", - get(routes::extract::get_extract).fallback(method_not_allowed), + get(routes::extract::get_extract) + .delete(routes::extract::cancel_extract) + .fallback(method_not_allowed), ) .route( "/v1/map", diff --git a/crates/crw-server/src/state.rs b/crates/crw-server/src/state.rs index 084288a1..b64c7d0a 100644 --- a/crates/crw-server/src/state.rs +++ b/crates/crw-server/src/state.rs @@ -100,22 +100,33 @@ const MAX_CONCURRENT_CRAWLS: usize = 10; /// Interval between expired crawl job cleanup runs. const JOB_CLEANUP_INTERVAL: Duration = Duration::from_secs(60); -/// Lifecycle of an async `/v2/extract` job. +/// Canonical lifecycle of an async extract job. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ExtractStatus { Processing, + Cancelling, Completed, Failed, + Cancelled, } impl ExtractStatus { pub fn as_str(self) -> &'static str { match self { ExtractStatus::Processing => "processing", + ExtractStatus::Cancelling => "cancelling", ExtractStatus::Completed => "completed", ExtractStatus::Failed => "failed", + ExtractStatus::Cancelled => "cancelled", } } + + pub fn is_terminal(self) -> bool { + matches!( + self, + ExtractStatus::Completed | ExtractStatus::Failed | ExtractStatus::Cancelled + ) + } } /// One URL's extraction outcome. Powers the native `/v1/extract` per-URL array @@ -161,6 +172,31 @@ pub struct ExtractRecord { pub credits_used: u32, pub error: Option, pub created_at: Instant, + /// The one URL currently dispatched by the sequential worker. Cancellation + /// cannot cross its terminal barrier until this slot has persisted. + pub claimed_index: Option, +} + +impl ExtractRecord { + /// Complete the cancellation barrier. Call only while holding the extract + /// jobs write lock and after observing that no URL remains claimed. + fn finish_cancellation(&mut self) { + if self.status != ExtractStatus::Cancelling || self.claimed_index.is_some() { + return; + } + for result in &mut self.per_url { + if result.status == ExtractStatus::Processing { + result.status = ExtractStatus::Cancelled; + result.data = None; + result.error = None; + result.llm_usage = None; + result.basis = None; + result.basis_warnings.clear(); + result.llm_input_hash = None; + } + } + self.status = ExtractStatus::Cancelled; + } } /// Shared application state. @@ -333,12 +369,11 @@ impl AppState { } drop(jobs); - // Prune finished extract jobs past TTL (keep in-flight ones). + // Prune terminal extract jobs past TTL (keep processing and the + // cancellation barrier in-flight). let mut ejobs = cleanup_state.extract_jobs.write().await; - ejobs.retain(|_id, rec| { - matches!(rec.status, ExtractStatus::Processing) - || rec.created_at.elapsed() < ttl - }); + ejobs + .retain(|_id, rec| !rec.status.is_terminal() || rec.created_at.elapsed() < ttl); } }); @@ -617,6 +652,27 @@ impl AppState { template: ScrapeRequest, ) -> Uuid { let id = Uuid::new_v4(); + // Seed the fixed-cardinality result array before the worker can run. + // Preflight failures are already final; valid URLs remain processing + // until claimed and persisted, or are converted to cancelled at the + // cancellation barrier. + let per_url = entries + .iter() + .map(|entry| UrlResult { + url: entry.url.clone(), + status: if entry.preflight_error.is_some() { + ExtractStatus::Failed + } else { + ExtractStatus::Processing + }, + data: None, + error: entry.preflight_error.clone(), + llm_usage: None, + basis: None, + basis_warnings: Vec::new(), + llm_input_hash: None, + }) + .collect(); { let mut jobs = self.extract_jobs.write().await; jobs.insert( @@ -624,11 +680,12 @@ impl AppState { ExtractRecord { status: ExtractStatus::Processing, data: None, - per_url: Vec::new(), + per_url, tokens_used: 0, credits_used: 0, error: None, created_at: Instant::now(), + claimed_index: None, }, ); } @@ -651,35 +708,42 @@ impl AppState { let deadline_ms = config.effective_deadline_ms(template.deadline_ms, template.wait_for); - let mut merged = serde_json::Map::new(); - let mut per_url: Vec = Vec::with_capacity(entries.len()); - let mut tokens = 0u32; - let mut credits = 0u32; - let mut last_err: Option = None; - let mut any_ok = false; - - for entry in entries { - // Preflight-failed URLs (bad parse / SSRF) surface as - // `failed` without a fetch — never silently dropped. - if let Some(err) = entry.preflight_error { - last_err = Some(err.clone()); - per_url.push(UrlResult { - url: entry.url, - status: ExtractStatus::Failed, - data: None, - error: Some(err), - llm_usage: None, - basis: None, - basis_warnings: Vec::new(), - llm_input_hash: None, - }); + for (index, entry) in entries.into_iter().enumerate() { + // Preflight failures were persisted at admission and + // never count as dispatched work. + if entry.preflight_error.is_some() { continue; } + // Claim exactly one slot while holding the same state + // lock DELETE uses. Once cancelling is visible no new + // URL can cross this point. + { + let mut jobs = extract_jobs.write().await; + let Some(rec) = jobs.get_mut(&id) else { + return; + }; + match rec.status { + ExtractStatus::Processing => { + if rec.per_url[index].status != ExtractStatus::Processing { + continue; + } + rec.claimed_index = Some(index); + } + ExtractStatus::Cancelling => { + rec.finish_cancellation(); + return; + } + ExtractStatus::Completed + | ExtractStatus::Failed + | ExtractStatus::Cancelled => return, + } + } + let mut req = template.clone(); req.url = entry.url.clone(); let deadline = Deadline::from_request_ms(deadline_ms); - match scrape_url( + let (result, merged_fields, tokens, credits) = match scrape_url( &req, &renderer, config.extraction.llm.as_ref(), @@ -692,59 +756,107 @@ impl AppState { .await { Ok(d) => { - any_ok = true; - if let Some(serde_json::Value::Object(obj)) = &d.json { - for (k, v) in obj { - merged.insert(k.clone(), v.clone()); - } - } - if let Some(usage) = &d.llm_usage { - tokens += usage.total_tokens; - } - credits += if d.credit_cost == 0 { 1 } else { d.credit_cost }; - per_url.push(UrlResult { - url: entry.url, - status: ExtractStatus::Completed, - data: d.json, - error: None, - llm_usage: d.llm_usage, - basis: d.basis, - basis_warnings: d.basis_warnings, - llm_input_hash: d.llm_input_hash, - }); + let merged_fields = match &d.json { + Some(serde_json::Value::Object(obj)) => Some(obj.clone()), + _ => None, + }; + let tokens = + d.llm_usage.as_ref().map_or(0, |usage| usage.total_tokens); + let credits = if d.credit_cost == 0 { 1 } else { d.credit_cost }; + ( + UrlResult { + url: entry.url, + status: ExtractStatus::Completed, + data: d.json, + error: None, + llm_usage: d.llm_usage, + basis: d.basis, + basis_warnings: d.basis_warnings, + llm_input_hash: d.llm_input_hash, + }, + merged_fields, + tokens, + credits, + ) } Err(e) => { let msg = e.to_string(); - last_err = Some(msg.clone()); - per_url.push(UrlResult { - url: entry.url, - status: ExtractStatus::Failed, - data: None, - error: Some(msg), - llm_usage: None, - basis: None, - basis_warnings: Vec::new(), - llm_input_hash: None, - }); + ( + UrlResult { + url: entry.url, + status: ExtractStatus::Failed, + data: None, + error: Some(msg), + llm_usage: None, + basis: None, + basis_warnings: Vec::new(), + llm_input_hash: None, + }, + None, + 0, + 0, + ) + } + }; + + // Persist the completed claimed slot and cumulative + // measured usage before dispatching anything else. + let mut jobs = extract_jobs.write().await; + let Some(rec) = jobs.get_mut(&id) else { + return; + }; + if rec.status.is_terminal() || rec.claimed_index != Some(index) { + return; + } + rec.per_url[index] = result; + if let Some(fields) = merged_fields { + let merged = rec.data.get_or_insert_with(|| { + serde_json::Value::Object(Default::default()) + }); + if let serde_json::Value::Object(merged) = merged { + merged.extend(fields); } } + rec.tokens_used = rec.tokens_used.saturating_add(tokens); + rec.credits_used = rec.credits_used.saturating_add(credits); + rec.claimed_index = None; + if rec.status == ExtractStatus::Cancelling { + rec.finish_cancellation(); + return; + } } let mut jobs = extract_jobs.write().await; if let Some(rec) = jobs.get_mut(&id) { - if !any_ok && last_err.is_some() { - rec.status = ExtractStatus::Failed; - rec.error = last_err; - } else { - rec.status = ExtractStatus::Completed; - rec.data = Some(serde_json::Value::Object(merged)); + match rec.status { + ExtractStatus::Processing => { + let any_ok = rec + .per_url + .iter() + .any(|result| result.status == ExtractStatus::Completed); + if any_ok { + rec.status = ExtractStatus::Completed; + rec.data.get_or_insert_with(|| { + serde_json::Value::Object(Default::default()) + }); + } else { + rec.status = ExtractStatus::Failed; + rec.error = rec + .per_url + .iter() + .rev() + .find_map(|result| result.error.clone()); + } + // Preserve the existing one-credit floor for a + // naturally finished all-failed job. Cancelled + // jobs retain only actually measured usage. + rec.credits_used = rec.credits_used.max(1); + } + ExtractStatus::Cancelling => rec.finish_cancellation(), + ExtractStatus::Completed + | ExtractStatus::Failed + | ExtractStatus::Cancelled => {} } - rec.per_url = per_url; - rec.tokens_used = tokens; - // ponytail: 1-credit floor even on an all-failed job — - // preflight-failed URLs add 0 (they `continue` before the - // tally). SaaS settles the real cost separately. - rec.credits_used = credits.max(1); } }) .await; @@ -752,4 +864,20 @@ impl AppState { id } + + /// Request cancellation and return the persisted canonical state. Repeated + /// calls are idempotent; terminal jobs are never rewritten. + pub async fn cancel_extract_job(&self, id: Uuid) -> CrwResult { + let mut jobs = self.extract_jobs.write().await; + let rec = jobs + .get_mut(&id) + .ok_or_else(|| CrwError::NotFound(format!("Extract job {id} not found")))?; + if rec.status == ExtractStatus::Processing { + rec.status = ExtractStatus::Cancelling; + } + if rec.status == ExtractStatus::Cancelling { + rec.finish_cancellation(); + } + Ok(rec.clone()) + } } diff --git a/crates/crw-server/tests/mcp.rs b/crates/crw-server/tests/mcp.rs index 45cd71f3..c4db2d2e 100644 --- a/crates/crw-server/tests/mcp.rs +++ b/crates/crw-server/tests/mcp.rs @@ -6,8 +6,10 @@ use crw_core::types::{ ApiResponse, GroupedSearchData, ImageResult, SearchData, SearchResponseData, SearchResult, }; use crw_server::app::create_app; -use crw_server::state::AppState; +use crw_server::state::{AppState, ExtractRecord, ExtractStatus, UrlResult}; use serde_json::{Value, json}; +use std::time::Instant; +use uuid::Uuid; fn test_app() -> TestServer { let config: AppConfig = toml::from_str("").unwrap(); @@ -64,7 +66,7 @@ async fn mcp_initialize_returns_capabilities() { #[tokio::test] async fn mcp_tools_list_returns_all_tools() { - // With a search backend configured, all 8 tools are advertised. + // With a search backend configured, all 9 tools are advertised. let server = test_app_with_search(); let resp = server .post("/mcp") @@ -76,8 +78,8 @@ async fn mcp_tools_list_returns_all_tools() { let tools = json["result"]["tools"].as_array().unwrap(); assert_eq!( tools.len(), - 8, - "Should have 8 tools: scrape, crawl, check, map, search, parse_file, extract, check_extract" + 9, + "Should have 9 tools including extract start/status/cancel" ); let tool_names: Vec<&str> = tools.iter().map(|t| t["name"].as_str().unwrap()).collect(); @@ -89,10 +91,11 @@ async fn mcp_tools_list_returns_all_tools() { assert!(tool_names.contains(&"crw_parse_file")); assert!(tool_names.contains(&"crw_extract")); assert!(tool_names.contains(&"crw_check_extract_status")); + assert!(tool_names.contains(&"crw_cancel_extract")); // Every tool advertises annotations. Job-starting tools (crw_crawl, // crw_extract) have side effects and must NOT be marked read-only. - let non_read_only = ["crw_crawl", "crw_extract"]; + let non_read_only = ["crw_crawl", "crw_extract", "crw_cancel_extract"]; for t in tools { assert!( t["annotations"].is_object(), @@ -115,6 +118,20 @@ async fn mcp_tools_list_returns_all_tools() { assert_eq!(crawl["annotations"]["idempotentHint"], false); let scrape = tools.iter().find(|t| t["name"] == "crw_scrape").unwrap(); assert_eq!(scrape["annotations"]["readOnlyHint"], true); + let cancel = tools + .iter() + .find(|t| t["name"] == "crw_cancel_extract") + .unwrap(); + assert_eq!(cancel["annotations"]["destructiveHint"], true); + assert_eq!(cancel["annotations"]["idempotentHint"], true); + for name in [ + "crw_extract", + "crw_check_extract_status", + "crw_cancel_extract", + ] { + let tool = tools.iter().find(|t| t["name"] == name).unwrap(); + assert!(tool["outputSchema"].is_object(), "{name} outputSchema"); + } } /// crw_search is suppressed from tools/list when no search backend is configured, @@ -131,11 +148,72 @@ async fn mcp_tools_list_hides_search_without_backend() { let json: serde_json::Value = resp.json(); let tools = json["result"]["tools"].as_array().unwrap(); let names: Vec<&str> = tools.iter().map(|t| t["name"].as_str().unwrap()).collect(); - assert_eq!(tools.len(), 7, "crw_search hidden without a backend"); + assert_eq!(tools.len(), 8, "crw_search hidden without a backend"); assert!(!names.contains(&"crw_search")); assert!(names.contains(&"crw_scrape")); } +#[tokio::test] +async fn mcp_extract_status_and_cancel_share_canonical_http_serializer() { + let config: AppConfig = toml::from_str("").unwrap(); + let state = AppState::new(config).unwrap(); + let id = Uuid::new_v4(); + state.extract_jobs.write().await.insert( + id, + ExtractRecord { + status: ExtractStatus::Processing, + data: None, + per_url: vec![UrlResult { + url: "https://example.com".into(), + status: ExtractStatus::Processing, + data: None, + error: None, + llm_usage: None, + basis: None, + basis_warnings: Vec::new(), + llm_input_hash: None, + }], + tokens_used: 0, + credits_used: 0, + error: None, + created_at: Instant::now(), + claimed_index: None, + }, + ); + let server = TestServer::new(create_app(state)); + + let status: Value = server + .post("/mcp") + .content_type("application/json") + .json(&mcp_request( + "tools/call", + json!(201), + json!({"name": "crw_check_extract_status", "arguments": {"id": id}}), + )) + .await + .json(); + let status = &status["result"]["structuredContent"]; + assert_eq!(status["id"], id.to_string()); + assert_eq!(status["status"], "processing"); + assert_eq!(status["results"][0]["status"], "processing"); + assert!(status["expiresAt"].is_string()); + assert!(status.get("success").is_none()); + + let cancelled: Value = server + .post("/mcp") + .content_type("application/json") + .json(&mcp_request( + "tools/call", + json!(202), + json!({"name": "crw_cancel_extract", "arguments": {"id": id}}), + )) + .await + .json(); + let cancelled = &cancelled["result"]["structuredContent"]; + assert_eq!(cancelled["status"], "cancelled"); + assert_eq!(cancelled["results"][0]["status"], "cancelled"); +} + #[tokio::test] async fn mcp_unknown_method_error_32601() { let server = test_app(); diff --git a/crates/crw-server/tests/v1_extract.rs b/crates/crw-server/tests/v1_extract.rs index 580e9ec8..e3ab5522 100644 --- a/crates/crw-server/tests/v1_extract.rs +++ b/crates/crw-server/tests/v1_extract.rs @@ -9,8 +9,10 @@ use axum::http::StatusCode; use axum_test::TestServer; use crw_core::config::AppConfig; use crw_server::app::create_app; -use crw_server::state::{AppState, ExtractStatus, PreparedUrl}; +use crw_server::state::{AppState, ExtractRecord, ExtractStatus, PreparedUrl, UrlResult}; use serde_json::{Value, json}; +use std::time::Instant; +use uuid::Uuid; fn test_app() -> TestServer { let config: AppConfig = toml::from_str("").unwrap(); @@ -117,6 +119,147 @@ async fn v1_extract_status_unknown_404() { r.assert_status(StatusCode::NOT_FOUND); } +#[tokio::test] +async fn v1_extract_cancel_rejects_malformed_and_unknown_ids() { + let s = test_app(); + s.delete("/v1/extract/not-a-uuid") + .await + .assert_status(StatusCode::BAD_REQUEST); + s.delete("/v1/extract/00000000-0000-0000-0000-000000000000") + .await + .assert_status(StatusCode::NOT_FOUND); +} + +fn pending_slot(url: &str) -> UrlResult { + UrlResult { + url: url.into(), + status: ExtractStatus::Processing, + data: None, + error: None, + llm_usage: None, + basis: None, + basis_warnings: Vec::new(), + llm_input_hash: None, + } +} + +fn seeded_record(urls: &[&str], claimed_index: Option) -> ExtractRecord { + ExtractRecord { + status: ExtractStatus::Processing, + data: None, + per_url: urls.iter().map(|url| pending_slot(url)).collect(), + tokens_used: 0, + credits_used: 0, + error: None, + created_at: Instant::now(), + claimed_index, + } +} + +#[tokio::test] +async fn cancel_before_dispatch_is_terminal_ordered_and_idempotent() { + let config: AppConfig = toml::from_str("").unwrap(); + let state = AppState::new(config).unwrap(); + let id = Uuid::new_v4(); + state.extract_jobs.write().await.insert( + id, + seeded_record(&["https://a.example", "https://b.example"], None), + ); + let server = TestServer::new(create_app(state)); + + let first: Value = server.delete(&format!("/v1/extract/{id}")).await.json(); + assert_eq!(first["status"], "cancelled"); + assert_eq!(first["results"].as_array().unwrap().len(), 2); + assert_eq!(first["results"][0]["url"], "https://a.example"); + assert_eq!(first["results"][1]["url"], "https://b.example"); + for result in first["results"].as_array().unwrap() { + assert_eq!(result["status"], "cancelled"); + assert_eq!(result.as_object().unwrap().len(), 2); + } + + let repeated: Value = server.delete(&format!("/v1/extract/{id}")).await.json(); + assert_eq!(repeated, first, "repeated DELETE returns persisted state"); + let get: Value = server.get(&format!("/v1/extract/{id}")).await.json(); + assert_eq!(get, first, "GET and DELETE share the canonical serializer"); +} + +#[tokio::test] +async fn claimed_slot_holds_cancelling_barrier_until_its_result_persists() { + let config: AppConfig = toml::from_str("").unwrap(); + let state = AppState::new(config).unwrap(); + let id = Uuid::new_v4(); + state.extract_jobs.write().await.insert( + id, + seeded_record( + &["https://claimed.example", "https://pending.example"], + Some(0), + ), + ); + let server = TestServer::new(create_app(state.clone())); + + let cancelling: Value = server.delete(&format!("/v1/extract/{id}")).await.json(); + assert_eq!(cancelling["status"], "cancelling"); + assert_eq!(cancelling["results"][0]["status"], "processing"); + assert_eq!(cancelling["results"][1]["status"], "processing"); + let repeated: Value = server.delete(&format!("/v1/extract/{id}")).await.json(); + assert_eq!(repeated, cancelling); + + // Model the worker's atomic final write for the already claimed URL. + { + let mut jobs = state.extract_jobs.write().await; + let rec = jobs.get_mut(&id).unwrap(); + rec.per_url[0].status = ExtractStatus::Completed; + rec.per_url[0].data = Some(json!({"title": "persisted"})); + rec.tokens_used = 9; + rec.credits_used = 1; + rec.claimed_index = None; + } + + let terminal: Value = server.delete(&format!("/v1/extract/{id}")).await.json(); + assert_eq!(terminal["status"], "cancelled"); + assert_eq!(terminal["results"][0]["status"], "completed"); + assert_eq!(terminal["results"][0]["data"]["title"], "persisted"); + assert_eq!(terminal["results"][1]["status"], "cancelled"); + assert_eq!(terminal["tokensUsed"], 9); + assert_eq!(terminal["creditsUsed"], 1); +} + +#[tokio::test] +async fn delete_never_rewrites_completed_or_failed_terminal_state() { + let config: AppConfig = toml::from_str("").unwrap(); + let state = AppState::new(config).unwrap(); + let completed_id = Uuid::new_v4(); + let failed_id = Uuid::new_v4(); + let mut completed = seeded_record(&["https://done.example"], None); + completed.status = ExtractStatus::Completed; + completed.per_url[0].status = ExtractStatus::Completed; + completed.per_url[0].data = Some(json!({"ok": true})); + let mut failed = seeded_record(&["https://failed.example"], None); + failed.status = ExtractStatus::Failed; + failed.per_url[0].status = ExtractStatus::Failed; + failed.per_url[0].error = Some("failed".into()); + failed.error = Some("failed".into()); + { + let mut jobs = state.extract_jobs.write().await; + jobs.insert(completed_id, completed); + jobs.insert(failed_id, failed); + } + let server = TestServer::new(create_app(state)); + + let completed: Value = server + .delete(&format!("/v1/extract/{completed_id}")) + .await + .json(); + assert_eq!(completed["status"], "completed"); + assert_eq!(completed["results"][0]["data"]["ok"], true); + let failed: Value = server + .delete(&format!("/v1/extract/{failed_id}")) + .await + .json(); + assert_eq!(failed["status"], "failed"); + assert_eq!(failed["error"], "failed"); +} + #[tokio::test] async fn worker_seeds_preflight_failures_in_order() { // The worker processes entries in original order and surfaces preflight @@ -179,6 +322,136 @@ async fn worker_seeds_preflight_failures_in_order() { assert_eq!(rec.per_url[0].error.as_deref(), Some("bad-1")); } +#[tokio::test] +async fn worker_cancel_after_persisted_result_settles_claim_and_stops_next_dispatch() { + use crw_core::types::{OutputFormat, ScrapeRequest}; + use std::time::Duration; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + unsafe { + std::env::set_var("CRW_ALLOW_LOOPBACK_FOR_TESTS", "1"); + } + let mock = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/page-a")) + .respond_with( + ResponseTemplate::new(200).set_body_string("

A

"), + ) + .mount(&mock) + .await; + Mock::given(method("GET")) + .and(path("/page-b")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string("

B

") + .set_delay(Duration::from_millis(75)), + ) + .mount(&mock) + .await; + Mock::given(method("GET")) + .and(path("/page-c")) + .respond_with(ResponseTemplate::new(200).set_body_string("

C

")) + .mount(&mock) + .await; + Mock::given(method("POST")) + .and(path("/v1/messages")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "content": [{ + "type": "tool_use", + "id": "t1", + "name": "extract_data", + "input": { "title": "A" } + }], + "usage": { "input_tokens": 7, "output_tokens": 3 } + }))) + .mount(&mock) + .await; + + let mut config: AppConfig = toml::from_str( + r#" +[extraction.llm] +api_key = "test-key" +provider = "anthropic" +model = "claude-test" +"#, + ) + .unwrap(); + config.extraction.llm.as_mut().unwrap().base_url = Some(mock.uri()); + let state = AppState::new(config).unwrap(); + let entries = vec![ + PreparedUrl { + url: format!("{}/page-a", mock.uri()), + preflight_error: None, + }, + PreparedUrl { + url: format!("{}/page-b", mock.uri()), + preflight_error: None, + }, + PreparedUrl { + url: format!("{}/page-c", mock.uri()), + preflight_error: None, + }, + ]; + let template = ScrapeRequest { + formats: vec![OutputFormat::Json], + json_schema: Some(json!({ + "type": "object", + "properties": { "title": {"type": "string"} } + })), + ..Default::default() + }; + let id = state.start_extract_job(entries, template).await; + + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let jobs = state.extract_jobs.read().await; + if jobs[&id].claimed_index == Some(1) + && jobs[&id].per_url[0].status == ExtractStatus::Completed + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("second URL was never claimed after the first result persisted"); + let cancelling = state.cancel_extract_job(id).await.unwrap(); + assert_eq!(cancelling.status, ExtractStatus::Cancelling); + + let terminal = tokio::time::timeout(Duration::from_secs(2), async { + loop { + let rec = state.extract_jobs.read().await[&id].clone(); + if rec.status == ExtractStatus::Cancelled { + break rec; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("cancellation barrier never reached terminal"); + assert_eq!(terminal.per_url.len(), 3); + assert_eq!(terminal.per_url[0].status, ExtractStatus::Completed); + assert_eq!(terminal.per_url[0].data, Some(json!({"title": "A"}))); + assert_eq!(terminal.per_url[1].status, ExtractStatus::Completed); + assert_eq!(terminal.per_url[2].status, ExtractStatus::Cancelled); + assert!(terminal.tokens_used > 0); + assert_eq!(terminal.credits_used, 2); + + let requests = mock.received_requests().await.unwrap(); + assert!( + requests + .iter() + .any(|request| request.url.path() == "/page-a") + ); + assert!( + requests + .iter() + .all(|request| request.url.path() != "/page-c"), + "cancellation must prevent the next URL dispatch" + ); +} + #[tokio::test] async fn v1_extract_completes_end_to_end_with_mocked_llm() { use wiremock::matchers::{method, path}; diff --git a/docs/agent-onboarding/SKILL.md b/docs/agent-onboarding/SKILL.md index 8694938e..ebcb2491 100644 --- a/docs/agent-onboarding/SKILL.md +++ b/docs/agent-onboarding/SKILL.md @@ -130,6 +130,17 @@ Parameters: Returns: status and, when complete, a per-URL results array. +### crw_cancel_extract + +Idempotently request cancellation of an extract job. A claimed URL may finish +while status is `cancelling`; terminal `cancelled` preserves that result and +marks every untouched ordered slot `cancelled`. + +Parameters: +- `id` (required) — The extract job ID from `crw_extract` + +Returns the same canonical status envelope as `crw_check_extract_status`. + ### crw_parse_file Parse a local file (PDF) into markdown or structured output without fetching from the web. diff --git a/docs/docs/extract.md b/docs/docs/extract.md index 7db19605..a0513848 100644 --- a/docs/docs/extract.md +++ b/docs/docs/extract.md @@ -215,6 +215,36 @@ If the underlying page scrape is weak, the JSON extraction will also be weak. - Debugging extraction before confirming the underlying scrape succeeded - Using the async `/v1/extract` job for a single URL when a synchronous `/v1/scrape` with `formats: ["json"]` is simpler +## Multi-URL lifecycle and cancellation + +`POST /v1/extract` creates one ordered result slot per requested URL. Poll +`GET /v1/extract/{id}` or cancel idempotently with +`DELETE /v1/extract/{id}`. The lifecycle is: + +```text +processing -> completed | failed +processing -> cancelling -> cancelled +``` + +`cancelling` is not terminal: the one URL already claimed by the sequential +worker may finish and persist its result and measured usage. No later URL is +started. At the terminal barrier, every untouched slot becomes `cancelled` and +omits `data`, `error`, `llmUsage`, and `basis`. The result array always keeps the +original URL count and order. Repeated DELETE returns the persisted state; +DELETE after `completed`, `failed`, or `cancelled` does not rewrite it. + +```bash +curl -X DELETE https://api.fastcrw.com/v1/extract/JOB_ID \ + -H "Authorization: Bearer YOUR_API_KEY" +``` + +The TypeScript SDK exposes `startExtract`, `getExtract`, and `cancelExtract`; +Python exposes `start_extract`, `get_extract`, and `cancel_extract`. Their +convenience `extract` waiter treats `cancelling` as non-terminal, raises a typed +cancellation error with partial results on `cancelled`, and performs a +best-effort DELETE on timeout. Async starters send `Prefer: respond-async` for +managed and self-hosted parity. + ## When to use something else - Use [Scrape](#scraping) when prose or markdown is enough diff --git a/docs/docs/mcp-clients.md b/docs/docs/mcp-clients.md index e72c9187..a303716e 100644 --- a/docs/docs/mcp-clients.md +++ b/docs/docs/mcp-clients.md @@ -26,7 +26,7 @@ Use one of these three patterns: ## What tools you get -Embedded local mode (`crw-mcp`) exposes up to 8 tools: +Embedded local mode (`crw-mcp`) exposes up to 9 tools: - `crw_scrape` - `crw_crawl` @@ -34,10 +34,11 @@ Embedded local mode (`crw-mcp`) exposes up to 8 tools: - `crw_map` - `crw_extract` — structured extraction across URLs (async job) - `crw_check_extract_status` — poll an extract job +- `crw_cancel_extract` — cancel an extract job and return canonical status - `crw_parse_file` — parse a local PDF (base64) to markdown, no OCR (always present) - `crw_search` — only advertised when a search backend is configured; hidden otherwise -fastcrw.com cloud mode exposes all 8 tools: +fastcrw.com cloud mode exposes all 9 tools: - `crw_scrape` - `crw_crawl` @@ -45,6 +46,7 @@ fastcrw.com cloud mode exposes all 8 tools: - `crw_map` - `crw_extract` - `crw_check_extract_status` +- `crw_cancel_extract` - `crw_parse_file` - `crw_search` — always available (managed search backend) @@ -54,7 +56,7 @@ Browser automation mode (`crw-browse`, separate server — v0.4.0+) exposes: - `tree` — accessibility snapshot of the current page :::tip -If you only remember one rule, remember this one: local embedded mode is the easiest setup (up to 8 tools, with `crw_search` appearing automatically when a search backend is configured), and fastcrw.com cloud mode is the easiest way to get all 8 tools including always-on web search. +If you only remember one rule, remember this one: local embedded mode is the easiest setup (up to 9 tools, with `crw_search` appearing automatically when a search backend is configured), and fastcrw.com cloud mode is the easiest way to get all 9 tools including always-on web search. ::: ## Claude Code @@ -230,7 +232,7 @@ Edit `~/.codeium/windsurf/mcp_config.json`. ``` :::note -Windsurf has a total MCP tool limit. CRW stays lightweight: local embedded mode exposes up to 8 tools (`crw_search` is hidden unless a search backend is configured), and cloud mode always exposes all 8 tools. +Windsurf has a total MCP tool limit. CRW stays lightweight: local embedded mode exposes up to 9 tools (`crw_search` is hidden unless a search backend is configured), and cloud mode always exposes all 9 tools. ::: ## Cline @@ -365,7 +367,7 @@ Open Management (`⌘⇧M`) → **Providers** → **MCP Providers** → **Add MC | Command | `npx` | | Args | `crw-mcp` | -`crw_*` tools then appear in chat namespaced as `crw_scrape`, `crw_crawl`, `crw_check_crawl_status`, `crw_map`, `crw_extract`, `crw_check_extract_status`, `crw_parse_file` (and `crw_search` when a search backend is configured). +`crw_*` tools then appear in chat namespaced as `crw_scrape`, `crw_crawl`, `crw_check_crawl_status`, `crw_map`, `crw_extract`, `crw_check_extract_status`, `crw_cancel_extract`, `crw_parse_file` (and `crw_search` when a search backend is configured). ### Or edit the config file diff --git a/docs/docs/mcp.md b/docs/docs/mcp.md index 516a135d..5533ce70 100644 --- a/docs/docs/mcp.md +++ b/docs/docs/mcp.md @@ -10,8 +10,8 @@ CRW includes a built-in MCP (Model Context Protocol) server that gives any MCP-c | Mode | When | Tools | Description | |------|------|-------|-------------| -| **Embedded** (default) | No `--api-url` / `CRW_API_URL` set | scrape, crawl, check_crawl_status, map, extract, check_extract_status, parse_file + **search** (when a search backend is configured) | Self-contained. No server needed. The scraping engine runs inside the MCP process. `crw_search` is advertised only when a search backend is configured (e.g. the Docker compose sidecar). | -| **Proxy / Server** | `--api-url` / `CRW_API_URL` set | scrape, crawl, check_crawl_status, map, extract, check_extract_status, parse_file, search | Forwards tool calls to a remote CRW server — the [fastcrw.com](https://fastcrw.com) cloud **or your own self-hosted server**. `crw_search` is always advertised in proxy mode; it works whenever the server has a search backend configured (the Docker stack enables it by default). | +| **Embedded** (default) | No `--api-url` / `CRW_API_URL` set | scrape, crawl, check_crawl_status, map, extract, check_extract_status, cancel_extract, parse_file + **search** (when a search backend is configured) | Self-contained. No server needed. The scraping engine runs inside the MCP process. `crw_search` is advertised only when a search backend is configured (e.g. the Docker compose sidecar). | +| **Proxy / Server** | `--api-url` / `CRW_API_URL` set | scrape, crawl, check_crawl_status, map, extract, check_extract_status, cancel_extract, parse_file, search | Forwards tool calls to a remote CRW server — the [fastcrw.com](https://fastcrw.com) cloud **or your own self-hosted server**. `crw_search` is always advertised in proxy mode; it works whenever the server has a search backend configured (the Docker stack enables it by default). | ## Where to use what @@ -154,6 +154,7 @@ This yields a ~4.2 MB binary (vs ~17 MB for the default embedded build) because | `crw_map` | Discover all URLs on a site | `POST /v1/map` | All modes | | `crw_extract` | Extract structured JSON from URLs → async job ID | `POST /v1/extract` | All modes | | `crw_check_extract_status` | Poll extract status and get results | `GET /v1/extract/:id` | All modes | +| `crw_cancel_extract` | Idempotently cancel extract and return canonical status | `DELETE /v1/extract/:id` | All modes | | `crw_search` | Search the web → titles, URLs, descriptions | `POST /v1/search` | Always in proxy mode; embedded only when a search backend is configured | | `crw_parse_file` | Parse a local PDF (base64) → markdown | `POST /firecrawl/v2/parse` (multipart) | All modes | diff --git a/docs/openapi-3.0.json b/docs/openapi-3.0.json index 55d30575..25ae5aa6 100644 --- a/docs/openapi-3.0.json +++ b/docs/openapi-3.0.json @@ -406,8 +406,22 @@ "extract" ], "summary": "Start an async multi-URL structured extraction job", - "description": "Native extraction. Returns a per-URL array via GET /v1/extract/{id}. At least one of `prompt` or `schema` is required.", + "description": "Native extraction. Returns a fixed-cardinality per-URL array via GET /v1/extract/{id}. At least one of `prompt` or `schema` is required. SDK async starters send `Prefer: respond-async` for managed/self-hosted parity.", "operationId": "extractStart", + "parameters": [ + { + "name": "Prefer", + "in": "header", + "required": false, + "schema": { + "type": "string", + "enum": [ + "respond-async" + ] + }, + "description": "Request the canonical async lifecycle across managed and self-hosted deployments." + } + ], "requestBody": { "required": true, "content": { @@ -464,6 +478,46 @@ } } }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + } + } + }, + "delete": { + "tags": [ + "extract" + ], + "summary": "Cancel an extract job", + "description": "Idempotent. Processing becomes cancelling while a claimed URL settles, then cancelled. Completed, failed, and cancelled states are immutable.", + "operationId": "extractCancel", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Extract job id returned by POST /v1/extract" + } + ], + "responses": { + "200": { + "description": "Canonical job status + ordered per-URL results", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExtractStatus" + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, "404": { "$ref": "#/components/responses/NotFound" } @@ -1386,14 +1440,16 @@ "status": { "type": "string", "enum": [ + "processing", "completed", - "failed" + "failed", + "cancelled" ] }, "data": { "type": "object", "additionalProperties": true, - "description": "Extracted JSON object (present when status is completed)." + "description": "Extracted JSON object (present when status is completed). Untouched cancelled slots omit data, error, usage, and basis." }, "error": { "type": "string" @@ -1427,6 +1483,7 @@ "required": [ "id", "status", + "results", "expiresAt", "creditsUsed", "tokensUsed" @@ -1439,8 +1496,10 @@ "type": "string", "enum": [ "processing", + "cancelling", "completed", - "failed" + "failed", + "cancelled" ] }, "results": { @@ -1448,7 +1507,7 @@ "items": { "$ref": "#/components/schemas/ExtractUrlResult" }, - "description": "Per-URL results in original request order." + "description": "Fixed-cardinality per-URL slots in original request order. Pending slots are processing; untouched terminal slots are cancelled." }, "error": { "type": "string", diff --git a/docs/openapi.json b/docs/openapi.json index aebc81aa..2a9ef712 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -777,8 +777,22 @@ "extract" ], "summary": "Start an async multi-URL structured extraction job", - "description": "Native extraction. Returns a per-URL array via GET /v1/extract/{id}. At least one of `prompt` or `schema` is required.", + "description": "Native extraction. Returns a fixed-cardinality per-URL array via GET /v1/extract/{id}. At least one of `prompt` or `schema` is required. SDK async starters send `Prefer: respond-async` for managed/self-hosted parity.", "operationId": "extractStart", + "parameters": [ + { + "name": "Prefer", + "in": "header", + "required": false, + "schema": { + "type": "string", + "enum": [ + "respond-async" + ] + }, + "description": "Request the canonical async lifecycle across managed and self-hosted deployments." + } + ], "requestBody": { "required": true, "content": { @@ -835,6 +849,46 @@ } } }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "404": { + "$ref": "#/components/responses/NotFound" + } + } + }, + "delete": { + "tags": [ + "extract" + ], + "summary": "Cancel an extract job", + "description": "Idempotent. Processing becomes cancelling while a claimed URL settles, then cancelled. Completed, failed, and cancelled states are immutable.", + "operationId": "extractCancel", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Extract job id returned by POST /v1/extract" + } + ], + "responses": { + "200": { + "description": "Canonical job status + ordered per-URL results", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExtractStatus" + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, "404": { "$ref": "#/components/responses/NotFound" } @@ -2006,14 +2060,16 @@ "status": { "type": "string", "enum": [ + "processing", "completed", - "failed" + "failed", + "cancelled" ] }, "data": { "type": "object", "additionalProperties": true, - "description": "Extracted JSON object (present when status is completed)." + "description": "Extracted JSON object (present when status is completed). Untouched cancelled slots omit data, error, usage, and basis." }, "error": { "type": "string" @@ -2047,6 +2103,7 @@ "required": [ "id", "status", + "results", "expiresAt", "creditsUsed", "tokensUsed" @@ -2059,8 +2116,10 @@ "type": "string", "enum": [ "processing", + "cancelling", "completed", - "failed" + "failed", + "cancelled" ] }, "results": { @@ -2068,7 +2127,7 @@ "items": { "$ref": "#/components/schemas/ExtractUrlResult" }, - "description": "Per-URL results in original request order." + "description": "Fixed-cardinality per-URL slots in original request order. Pending slots are processing; untouched terminal slots are cancelled." }, "error": { "type": "string", diff --git a/mcp/crw-mcp/skills/SKILL.md b/mcp/crw-mcp/skills/SKILL.md index 8694938e..ebcb2491 100644 --- a/mcp/crw-mcp/skills/SKILL.md +++ b/mcp/crw-mcp/skills/SKILL.md @@ -130,6 +130,17 @@ Parameters: Returns: status and, when complete, a per-URL results array. +### crw_cancel_extract + +Idempotently request cancellation of an extract job. A claimed URL may finish +while status is `cancelling`; terminal `cancelled` preserves that result and +marks every untouched ordered slot `cancelled`. + +Parameters: +- `id` (required) — The extract job ID from `crw_extract` + +Returns the same canonical status envelope as `crw_check_extract_status`. + ### crw_parse_file Parse a local file (PDF) into markdown or structured output without fetching from the web. diff --git a/sdks/python/README.md b/sdks/python/README.md index 9587df1f..c1ad00d3 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -165,6 +165,15 @@ for r in results: if r["status"] == "completed": print(r["url"], r["data"]) +# Explicit typed lifecycle. start_extract always sends Prefer: respond-async. +accepted = client.start_extract( + ["https://a.example", "https://b.example"], + schema={"type": "object", "properties": {"title": {"type": "string"}}}, + basis=True, +) +status = client.get_extract(accepted["id"]) +client.cancel_extract(accepted["id"]) # idempotent + # Scrape many URLs in one async batch: pages = client.batch_scrape(["https://a.com", "https://b.com"], formats=["markdown"]) diff --git a/sdks/python/src/crw/__init__.py b/sdks/python/src/crw/__init__.py index 35e9338b..748f33a6 100644 --- a/sdks/python/src/crw/__init__.py +++ b/sdks/python/src/crw/__init__.py @@ -1,6 +1,24 @@ """CRW Python SDK — scrape, crawl, and map any website.""" from crw.client import CrwClient -from crw.exceptions import CrwError, CrwApiError, CrwBinaryNotFoundError, CrwTimeoutError +from crw.exceptions import ( + CrwApiError, + CrwBinaryNotFoundError, + CrwError, + CrwExtractCancelledError, + CrwTimeoutError, +) +from crw.types import Basis, ExtractAccepted, ExtractStatus, ExtractUrlResult -__all__ = ["CrwClient", "CrwError", "CrwApiError", "CrwBinaryNotFoundError", "CrwTimeoutError"] +__all__ = [ + "CrwClient", + "CrwError", + "CrwApiError", + "CrwBinaryNotFoundError", + "CrwTimeoutError", + "CrwExtractCancelledError", + "ExtractAccepted", + "ExtractStatus", + "ExtractUrlResult", + "Basis", +] diff --git a/sdks/python/src/crw/client.py b/sdks/python/src/crw/client.py index 886d6020..12a27c3a 100644 --- a/sdks/python/src/crw/client.py +++ b/sdks/python/src/crw/client.py @@ -7,11 +7,12 @@ import os import subprocess import time -from typing import Any +from typing import Any, cast from urllib.parse import quote, urlencode from crw._binary import ensure_binary -from crw.exceptions import CrwApiError, CrwError, CrwTimeoutError +from crw.exceptions import CrwApiError, CrwError, CrwExtractCancelledError, CrwTimeoutError +from crw.types import ExtractAccepted, ExtractStatus, ExtractUrlResult _REQUEST_ID = 0 @@ -447,7 +448,7 @@ def parse_file( args.update(kwargs) return self._tool_call("crw_parse_file", args) - def extract( + def _extract_body( self, urls: list[str], prompt: str | None = None, @@ -455,61 +456,135 @@ def extract( llm_api_key: str | None = None, llm_provider: str | None = None, llm_model: str | None = None, - poll_interval: float = 2.0, - timeout: float = 120.0, - ) -> list[dict]: - """Run native multi-URL structured extraction (HTTP mode only). - - Starts an async ``/v1/extract`` job, polls until completion, and returns - the per-URL ``results`` array (``[{url, status, data, error, llmUsage}]``) - in request order. Requires an LLM configured on the engine (or a BYOK key). - - Args: - urls: URLs to extract from (at least one required). - prompt: Free-text extraction instruction (required unless ``schema``). - schema: JSON Schema describing the desired output shape. - llm_api_key: BYOK LLM key (self-host/local engine only). - llm_provider: BYOK provider. - llm_model: BYOK model. - """ - if not self._api_url: - raise CrwError(_HTTP_ONLY_HINT.format(name="extract", reason="LLM extract job endpoint")) - + basis: bool | None = None, + ) -> dict[str, Any]: body: dict[str, Any] = {"urls": list(urls)} if prompt is not None: body["prompt"] = prompt if schema is not None: body["schema"] = schema + if basis is not None: + body["basis"] = basis if llm_api_key is not None: body["llmApiKey"] = llm_api_key if llm_provider is not None: body["llmProvider"] = llm_provider if llm_model is not None: body["llmModel"] = llm_model + return body + + def _start_extract_request(self, body: dict[str, Any]) -> dict: + return self._http_request( + "POST", + "/v1/extract", + body, + raw=True, + headers={"Prefer": "respond-async"}, + ) + + def start_extract( + self, + urls: list[str], + prompt: str | None = None, + schema: dict | None = None, + llm_api_key: str | None = None, + llm_provider: str | None = None, + llm_model: str | None = None, + basis: bool | None = None, + ) -> ExtractAccepted: + """Start extraction without waiting; always requests an async response.""" + if not self._api_url: + raise CrwError( + _HTTP_ONLY_HINT.format(name="start_extract", reason="LLM extract job endpoint") + ) + result = self._start_extract_request( + self._extract_body( + urls, prompt, schema, llm_api_key, llm_provider, llm_model, basis + ) + ) + if isinstance(result.get("id"), str) and result.get("status") == "processing": + return cast(ExtractAccepted, result) + raise CrwError(f"start_extract did not return an accepted job: {result}") + + def get_extract(self, job_id: str) -> ExtractStatus: + """Return the canonical extract lifecycle envelope.""" + if not self._api_url: + raise CrwError( + _HTTP_ONLY_HINT.format(name="get_extract", reason="LLM extract job endpoint") + ) + return cast( + ExtractStatus, + self._http_request( + "GET", f"/v1/extract/{quote(job_id, safe='')}", raw=True, check_success=False + ), + ) + + def cancel_extract(self, job_id: str) -> ExtractStatus: + """Idempotently request cancellation and return canonical status.""" + if not self._api_url: + raise CrwError( + _HTTP_ONLY_HINT.format(name="cancel_extract", reason="LLM extract job endpoint") + ) + return cast( + ExtractStatus, + self._http_request( + "DELETE", f"/v1/extract/{quote(job_id, safe='')}", raw=True, check_success=False + ), + ) - start = self._http_request("POST", "/v1/extract", body, raw=True) + def extract( + self, + urls: list[str], + prompt: str | None = None, + schema: dict | None = None, + llm_api_key: str | None = None, + llm_provider: str | None = None, + llm_model: str | None = None, + poll_interval: float = 2.0, + timeout: float = 120.0, + basis: bool | None = None, + ) -> list[ExtractUrlResult]: + """Wait for native multi-URL extraction, cancelling best-effort on timeout. + + ``cancelling`` is non-terminal. Terminal ``cancelled`` raises + :class:`CrwExtractCancelledError` with the status and partial results. + """ + if not self._api_url: + raise CrwError(_HTTP_ONLY_HINT.format(name="extract", reason="LLM extract job endpoint")) + + body = self._extract_body( + urls, prompt, schema, llm_api_key, llm_provider, llm_model, basis + ) + start = self._start_extract_request(body) job_id = start.get("id") if not job_id: # The managed API answers synchronously: the extraction is already # finished and the payload is in this first response, with no job to # poll. Only the async path hands back an `id`. Demanding one made # every managed extract() raise. + if start.get("status") == "cancelled": + raise CrwExtractCancelledError(cast(ExtractStatus, start)) if isinstance(start.get("results"), list): - return start["results"] + return cast(list[ExtractUrlResult], start["results"]) raise CrwError(f"extract returned neither a job id nor results: {start}") deadline = time.monotonic() + timeout while True: if time.monotonic() > deadline: + try: + self.cancel_extract(str(job_id)) + except Exception: + # Timeout remains caller-visible; DELETE is best effort. + pass raise CrwTimeoutError(f"Extract {job_id} timed out after {timeout}s") - status_result = self._http_request( - "GET", f"/v1/extract/{job_id}", raw=True, check_success=False - ) + status_result = self.get_extract(str(job_id)) status = status_result.get("status") if status == "completed": - return status_result.get("results", []) + return status_result["results"] if status == "failed": raise CrwError(f"Extract failed: {status_result.get('error', 'unknown')}") + if status == "cancelled": + raise CrwExtractCancelledError(status_result) time.sleep(poll_interval) def batch_scrape( @@ -694,17 +769,18 @@ def _http_request( *, raw: bool = False, check_success: bool = True, + headers: dict[str, str] | None = None, ) -> dict: import urllib.request assert self._api_url is not None url = f"{self._api_url.rstrip('/')}{path}" - headers = {"Content-Type": "application/json"} + request_headers = {"Content-Type": "application/json", **(headers or {})} if self._api_key: - headers["Authorization"] = f"Bearer {self._api_key}" + request_headers["Authorization"] = f"Bearer {self._api_key}" data = json.dumps(body).encode() if body else None - req = urllib.request.Request(url, data=data, headers=headers, method=method) + req = urllib.request.Request(url, data=data, headers=request_headers, method=method) result = _read_json_response(req) diff --git a/sdks/python/src/crw/exceptions.py b/sdks/python/src/crw/exceptions.py index f5d62476..97f07630 100644 --- a/sdks/python/src/crw/exceptions.py +++ b/sdks/python/src/crw/exceptions.py @@ -1,5 +1,10 @@ """CRW SDK exceptions.""" +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from crw.types import ExtractStatus, ExtractUrlResult + class CrwError(Exception): """Base exception for CRW SDK.""" @@ -13,6 +18,15 @@ class CrwTimeoutError(CrwError): """Operation timed out.""" +class CrwExtractCancelledError(CrwError): + """An extract waiter reached the immutable cancelled state.""" + + def __init__(self, status: "ExtractStatus"): + super().__init__(f"Extract {status['id']} was cancelled") + self.status = status + self.results: list["ExtractUrlResult"] = status["results"] + + class CrwApiError(CrwError): """API returned an error.""" diff --git a/sdks/python/src/crw/types.py b/sdks/python/src/crw/types.py new file mode 100644 index 00000000..40f719b9 --- /dev/null +++ b/sdks/python/src/crw/types.py @@ -0,0 +1,68 @@ +"""Typed native extract lifecycle contracts.""" + +from typing import Any, Literal, TypedDict + +ExtractJobState = Literal["processing", "cancelling", "completed", "failed", "cancelled"] +ExtractUrlState = Literal["processing", "completed", "failed", "cancelled"] +FieldStatus = Literal["supported", "unverified", "unsupported", "notFound"] + + +class _EvidenceCitationRequired(TypedDict): + url: str + sourceHash: str + sourceTextKind: str + + +class EvidenceCitation(_EvidenceCitationRequired, total=False): + title: str + excerpt: str + + +class _BasisRequired(TypedDict): + basisVersion: int + field: str + value: Any | None + status: FieldStatus + citations: list[EvidenceCitation] + + +class Basis(_BasisRequired, total=False): + confidence: Literal["low", "medium", "high"] + + +class BasisWarning(TypedDict): + field: str + code: str + + +class ExtractAccepted(TypedDict): + id: str + status: Literal["processing"] + urls: int + + +class _ExtractUrlResultRequired(TypedDict): + url: str + status: ExtractUrlState + + +class ExtractUrlResult(_ExtractUrlResultRequired, total=False): + data: Any + error: str + llmUsage: dict[str, Any] + basis: list[Basis] + basisWarnings: list[BasisWarning] + llmInputHash: str + + +class _ExtractStatusRequired(TypedDict): + id: str + status: ExtractJobState + results: list[ExtractUrlResult] + expiresAt: str + creditsUsed: int + tokensUsed: int + + +class ExtractStatus(_ExtractStatusRequired, total=False): + error: str diff --git a/sdks/python/tests/test_client_unit.py b/sdks/python/tests/test_client_unit.py index d3508161..3bf6d6b7 100644 --- a/sdks/python/tests/test_client_unit.py +++ b/sdks/python/tests/test_client_unit.py @@ -8,7 +8,7 @@ import pytest from crw.client import CLOUD_API_URL, CrwClient -from crw.exceptions import CrwApiError, CrwError, CrwTimeoutError +from crw.exceptions import CrwApiError, CrwError, CrwExtractCancelledError, CrwTimeoutError @pytest.fixture(autouse=True) @@ -346,6 +346,9 @@ class TestHttpOnlyMethods: "call", [ lambda c: c.extract(["https://example.com"], prompt="x"), + lambda c: c.start_extract(["https://example.com"], prompt="x"), + lambda c: c.get_extract("job-1"), + lambda c: c.cancel_extract("job-1"), lambda c: c.batch_scrape(["https://example.com"]), lambda c: c.capabilities(), lambda c: c.change_tracking_diff({"markdown": "a"}), @@ -377,6 +380,100 @@ def test_extract_polls_until_complete(self) -> None: ] # Native route, not the FC-legacy /v2/extract. assert req.call_args_list[0][0][:2] == ("POST", "/v1/extract") + assert req.call_args_list[0].kwargs["headers"] == {"Prefer": "respond-async"} + + def test_start_extract_prefer_managed_and_self_hosted_fixtures(self) -> None: + accepted = {"id": "job-1", "status": "processing", "urls": 1} + for client in ( + CrwClient(api_key="fc-test"), + CrwClient(api_url="http://localhost:3000"), + ): + with patch.object(client, "_http_request", return_value=accepted) as req: + result = client.start_extract( + ["https://example.com"], schema={"type": "object"}, basis=True + ) + assert result["id"] == "job-1" + assert req.call_args[0][:2] == ("POST", "/v1/extract") + assert req.call_args.kwargs["headers"] == {"Prefer": "respond-async"} + assert req.call_args[0][2]["basis"] is True + + def test_extract_preserves_managed_sync_fixture_with_prefer(self) -> None: + client = CrwClient(api_key="fc-test") + sync = { + "success": True, + "results": [ + {"url": "https://example.com", "status": "completed", "data": {"title": "Hi"}} + ], + } + with patch.object(client, "_http_request", return_value=sync) as req: + result = client.extract(["https://example.com"], prompt="title") + assert result[0]["status"] == "completed" + assert req.call_args.kwargs["headers"] == {"Prefer": "respond-async"} + + def test_get_and_cancel_extract_use_canonical_route(self) -> None: + status = { + "id": "job/one", + "status": "cancelled", + "results": [{"url": "https://example.com", "status": "cancelled"}], + "expiresAt": "2026-07-14T00:00:00Z", + "creditsUsed": 0, + "tokensUsed": 0, + } + client = CrwClient(api_url="http://localhost:3000") + with patch.object(client, "_http_request", return_value=status) as req: + assert client.get_extract("job/one")["status"] == "cancelled" + assert client.cancel_extract("job/one")["results"][0]["status"] == "cancelled" + assert req.call_args_list[0][0][:2] == ("GET", "/v1/extract/job%2Fone") + assert req.call_args_list[1][0][:2] == ("DELETE", "/v1/extract/job%2Fone") + + def test_extract_cancelling_then_typed_cancelled_error(self) -> None: + client = CrwClient(api_url="http://localhost:3000") + accepted = {"id": "job-1", "status": "processing", "urls": 2} + cancelling = { + "id": "job-1", + "status": "cancelling", + "results": [ + {"url": "https://a.example", "status": "completed", "data": {"title": "A"}}, + {"url": "https://b.example", "status": "processing"}, + ], + "expiresAt": "2026-07-14T00:00:00Z", + "creditsUsed": 1, + "tokensUsed": 9, + } + cancelled = { + **cancelling, + "status": "cancelled", + "results": [ + {"url": "https://a.example", "status": "completed", "data": {"title": "A"}}, + {"url": "https://b.example", "status": "cancelled"}, + ], + } + with patch.object(client, "_http_request", side_effect=[accepted, cancelling, cancelled]): + with patch("time.sleep"): + with pytest.raises(CrwExtractCancelledError) as exc: + client.extract( + ["https://a.example", "https://b.example"], + prompt="x", + poll_interval=0, + ) + assert exc.value.status["status"] == "cancelled" + assert exc.value.results[0]["status"] == "completed" + + def test_extract_timeout_best_effort_deletes(self) -> None: + client = CrwClient(api_url="http://localhost:3000") + accepted = {"id": "job-1", "status": "processing", "urls": 1} + cancelled = { + "id": "job-1", + "status": "cancelled", + "results": [{"url": "https://example.com", "status": "cancelled"}], + "expiresAt": "2026-07-14T00:00:00Z", + "creditsUsed": 0, + "tokensUsed": 0, + } + with patch.object(client, "_http_request", side_effect=[accepted, cancelled]) as req: + with pytest.raises(CrwTimeoutError): + client.extract(["https://example.com"], prompt="x", timeout=-1) + assert req.call_args_list[-1][0][:2] == ("DELETE", "/v1/extract/job-1") def test_batch_scrape_returns_pages(self) -> None: client = CrwClient(api_url="https://fastcrw.com/api") diff --git a/sdks/typescript/README.md b/sdks/typescript/README.md index f4871b13..96e7b58a 100644 --- a/sdks/typescript/README.md +++ b/sdks/typescript/README.md @@ -53,6 +53,7 @@ CRW_LOCAL=1 node app.js | `search(query, opts?)` | Web search (+ optional scrape) | both¹ | | `parseFile(bytes, opts?)` | PDF → markdown / structured JSON | both | | `extract({urls, schema?})` | Structured LLM extraction | HTTP | +| `startExtract(opts)` / `getExtract(id)` / `cancelExtract(id)` | Typed extract lifecycle | HTTP | | `batchScrape(urls, opts?)` | Scrape many URLs (async) | HTTP | | `capabilities()` | Feature-detect the engine | HTTP | | `changeTrackingDiff(cur, prev?)` | Diff vs a prior snapshot | HTTP | @@ -69,6 +70,14 @@ const results = await crw.extract({ // results: [{ url, status, data, error, llmUsage }] for (const r of results) if (r.status === "completed") console.log(r.url, r.data); +// Explicit lifecycle. startExtract always sends Prefer: respond-async. +const accepted = await crw.startExtract({ + urls: ["https://a.example", "https://b.example"], + schema: { type: "object", properties: { title: { type: "string" } } }, +}); +const status = await crw.getExtract(accepted.id); +await crw.cancelExtract(accepted.id); // idempotent + // Parse a PDF: import { readFileSync } from "node:fs"; const doc = await crw.parseFile(readFileSync("invoice.pdf"), { formats: ["markdown"] }); diff --git a/sdks/typescript/src/client.ts b/sdks/typescript/src/client.ts index de8b2548..e91af1d2 100644 --- a/sdks/typescript/src/client.ts +++ b/sdks/typescript/src/client.ts @@ -1,6 +1,6 @@ /** CRW client — cloud (default), self-hosted HTTP, or local subprocess mode. */ -import { CrwApiError, CrwError, CrwTimeoutError } from "./errors.js"; +import { CrwApiError, CrwError, CrwExtractCancelledError, CrwTimeoutError } from "./errors.js"; import { LocalTransport } from "./local.js"; import type { BatchResult, @@ -12,7 +12,9 @@ import type { CrawlResult, DiffResult, ExtractOptions, + ExtractAccepted, ExtractResult, + ExtractStatus, Json, MapOptions, ParseFileOptions, @@ -221,39 +223,91 @@ export class CrwClient { } /** - * Native multi-URL structured extraction (HTTP mode only). Starts an async - * `/v1/extract` job and polls until complete, returning a per-URL results - * array (`[{ url, status, data, error, llmUsage }]`) in request order. + * Start native multi-URL extraction without waiting. `Prefer: respond-async` + * is always sent so managed and self-hosted servers use the same lifecycle. + */ + async startExtract(opts: ExtractOptions): Promise { + if (!this.apiUrl) throw new CrwError(httpOnlyHint("startExtract", "LLM extract job endpoint")); + const start = await this.startExtractRequest(opts); + if (typeof start.id === "string" && start.status === "processing") { + return start as unknown as ExtractAccepted; + } + throw new CrwError(`startExtract did not return an accepted job: ${JSON.stringify(start)}`); + } + + /** Get the canonical lifecycle envelope for an extract job. */ + async getExtract(id: string): Promise { + if (!this.apiUrl) throw new CrwError(httpOnlyHint("getExtract", "LLM extract job endpoint")); + const result = await this.httpRequest("GET", `/v1/extract/${encodeURIComponent(id)}`, undefined, { + raw: true, + checkSuccess: false, + }); + return result as unknown as ExtractStatus; + } + + /** Idempotently request cancellation and return the canonical lifecycle envelope. */ + async cancelExtract(id: string): Promise { + if (!this.apiUrl) throw new CrwError(httpOnlyHint("cancelExtract", "LLM extract job endpoint")); + const result = await this.httpRequest("DELETE", `/v1/extract/${encodeURIComponent(id)}`, undefined, { + raw: true, + checkSuccess: false, + }); + return result as unknown as ExtractStatus; + } + + /** + * Convenience waiter over start/get/cancel. `cancelling` is non-terminal; + * terminal cancellation raises a typed error carrying partial results. */ async extract(opts: ExtractOptions): Promise { if (!this.apiUrl) throw new CrwError(httpOnlyHint("extract", "LLM extract job endpoint")); - const { urls, prompt, schema, basis, llmApiKey, llmProvider, llmModel, pollInterval = 2, timeout = 120 } = opts; - const body: Json = { urls: [...urls] }; - if (prompt !== undefined) body.prompt = prompt; - if (schema !== undefined) body.schema = schema; - if (basis !== undefined) body.basis = basis; - if (llmApiKey !== undefined) body.llmApiKey = llmApiKey; - if (llmProvider !== undefined) body.llmProvider = llmProvider; - if (llmModel !== undefined) body.llmModel = llmModel; - const start = await this.httpRequest("POST", "/v1/extract", body, { raw: true }); + const { pollInterval = 2, timeout = 120 } = opts; + const start = await this.startExtractRequest(opts); const jobId = start.id as string | undefined; - // The managed API answers synchronously: the extraction is already finished and - // the payload is in this first response, with no job to poll. Only the async - // path hands back an `id`. Demanding one made every managed extract() throw. + // Preserve the managed synchronous response during rollout if an older + // endpoint ignores Prefer. New managed/self-hosted fixtures return an id. if (!jobId) { - if (Array.isArray(start.results)) return start.results as Json[]; + if (start.status === "cancelled") { + throw new CrwExtractCancelledError(start as unknown as ExtractStatus); + } + if (Array.isArray(start.results)) return start.results as ExtractResult; throw new CrwError(`extract returned neither a job id nor results: ${JSON.stringify(start)}`); } const deadline = Date.now() + timeout * 1000; for (;;) { - if (Date.now() > deadline) throw new CrwTimeoutError(`Extract ${jobId} timed out after ${timeout}s`); - const status = await this.httpRequest("GET", `/v1/extract/${jobId}`, undefined, { raw: true, checkSuccess: false }); - if (status.status === "completed") return (status.results as Json[]) ?? []; + if (Date.now() > deadline) { + try { + await this.cancelExtract(jobId); + } catch { + // Timeout remains the caller-visible failure; DELETE is best effort. + } + throw new CrwTimeoutError(`Extract ${jobId} timed out after ${timeout}s`); + } + const status = await this.getExtract(jobId); + if (status.status === "completed") return status.results; if (status.status === "failed") throw new CrwError(`Extract failed: ${status.error ?? "unknown"}`); + if (status.status === "cancelled") throw new CrwExtractCancelledError(status); await sleep(pollInterval * 1000); } } + private async startExtractRequest(opts: ExtractOptions): Promise { + const { urls, prompt, schema, basis, llmApiKey, llmProvider, llmModel, pollInterval = 2, timeout = 120 } = opts; + void pollInterval; + void timeout; + const body: Json = { urls: [...urls] }; + if (prompt !== undefined) body.prompt = prompt; + if (schema !== undefined) body.schema = schema; + if (basis !== undefined) body.basis = basis; + if (llmApiKey !== undefined) body.llmApiKey = llmApiKey; + if (llmProvider !== undefined) body.llmProvider = llmProvider; + if (llmModel !== undefined) body.llmModel = llmModel; + return this.httpRequest("POST", "/v1/extract", body, { + raw: true, + headers: { Prefer: "respond-async" }, + }); + } + /** Scrape many URLs in one async batch job (HTTP mode only). */ async batchScrape(urls: string[], opts: BatchScrapeOptions = {}): Promise { if (!this.apiUrl) throw new CrwError(httpOnlyHint("batchScrape", "batch job endpoint")); @@ -321,11 +375,18 @@ export class CrwClient { method: string, path: string, body?: Json, - { raw = false, checkSuccess = true }: { raw?: boolean; checkSuccess?: boolean } = {}, + { + raw = false, + checkSuccess = true, + headers: extraHeaders, + }: { raw?: boolean; checkSuccess?: boolean; headers?: Record } = {}, ): Promise { if (this.apiUrl === null) throw new CrwError("internal: httpRequest in local mode"); const url = `${this.apiUrl.replace(/\/$/, "")}${path}`; - const headers: Record = { "Content-Type": "application/json" }; + const headers: Record = { + "Content-Type": "application/json", + ...extraHeaders, + }; if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`; const resp = await fetch(url, { method, headers, body: body ? JSON.stringify(body) : undefined }); const result = await this.readJson(resp); diff --git a/sdks/typescript/src/errors.ts b/sdks/typescript/src/errors.ts index f9e5ef07..b4069d06 100644 --- a/sdks/typescript/src/errors.ts +++ b/sdks/typescript/src/errors.ts @@ -1,5 +1,7 @@ /** CRW SDK error types. */ +import type { ExtractStatus } from "./types.js"; + export class CrwError extends Error { constructor(message: string) { super(message); @@ -23,6 +25,19 @@ export class CrwTimeoutError extends CrwError { } } +/** A convenience extract waiter reached the immutable cancelled state. */ +export class CrwExtractCancelledError extends CrwError { + readonly status: ExtractStatus; + readonly results: ExtractStatus["results"]; + + constructor(status: ExtractStatus) { + super(`Extract ${status.id} was cancelled`); + this.name = "CrwExtractCancelledError"; + this.status = status; + this.results = status.results; + } +} + export class CrwBinaryNotFoundError extends CrwError { constructor(message: string) { super(message); diff --git a/sdks/typescript/src/index.ts b/sdks/typescript/src/index.ts index da789cd2..80eb0443 100644 --- a/sdks/typescript/src/index.ts +++ b/sdks/typescript/src/index.ts @@ -1,5 +1,11 @@ export { CrwClient, CLOUD_API_URL, DASHBOARD_URL, DOCS_URL } from "./client.js"; -export { CrwError, CrwApiError, CrwTimeoutError, CrwBinaryNotFoundError } from "./errors.js"; +export { + CrwError, + CrwApiError, + CrwTimeoutError, + CrwExtractCancelledError, + CrwBinaryNotFoundError, +} from "./errors.js"; export type { ClientOptions, ScrapeOptions, @@ -8,6 +14,15 @@ export type { SearchOptions, ParseFileOptions, ExtractOptions, + ExtractJobState, + ExtractUrlState, + ExtractAccepted, + ExtractStatus, + ExtractUrlResult, + Basis, + BasisWarning, + EvidenceCitation, + FieldStatus, BatchScrapeOptions, ChangeTrackingOptions, ScrapeResult, diff --git a/sdks/typescript/src/types.ts b/sdks/typescript/src/types.ts index 76d656ac..19b2eb84 100644 --- a/sdks/typescript/src/types.ts +++ b/sdks/typescript/src/types.ts @@ -129,6 +129,46 @@ export interface ExtractOptions { timeout?: number; } +export type ExtractJobState = + | "processing" + | "cancelling" + | "completed" + | "failed" + | "cancelled"; + +export type ExtractUrlState = "processing" | "completed" | "failed" | "cancelled"; + +/** Response from an async extract admission request. */ +export interface ExtractAccepted { + id: string; + status: "processing"; + /** Count of URLs enqueued for fetch. */ + urls: number; +} + +/** One fixed, ordered result slot for a requested URL. */ +export interface ExtractUrlResult { + url: string; + status: ExtractUrlState; + data?: unknown; + error?: string; + llmUsage?: Json; + basis?: Basis[]; + basisWarnings?: BasisWarning[]; + llmInputHash?: string; +} + +/** Canonical GET/DELETE extract lifecycle envelope. */ +export interface ExtractStatus { + id: string; + status: ExtractJobState; + results: ExtractUrlResult[]; + error?: string; + expiresAt: string; + creditsUsed: number; + tokensUsed: number; +} + export interface BatchScrapeOptions { formats?: string[]; pollInterval?: number; @@ -148,7 +188,7 @@ export type CrawlResult = Json[]; export type SearchResult = Json | Json[]; export type ParseResult = Json; /** Native `/v1/extract` returns one result object per URL, in request order. */ -export type ExtractResult = Json[]; +export type ExtractResult = ExtractUrlResult[]; export type BatchResult = Json[]; export type Capabilities = Json; export type DiffResult = Json; diff --git a/sdks/typescript/test/client.test.ts b/sdks/typescript/test/client.test.ts index e9ea10e9..77a6c87e 100644 --- a/sdks/typescript/test/client.test.ts +++ b/sdks/typescript/test/client.test.ts @@ -1,6 +1,12 @@ import assert from "node:assert/strict"; import { afterEach, beforeEach, test } from "node:test"; -import { CLOUD_API_URL, CrwClient, CrwError } from "../dist/esm/index.js"; +import { + CLOUD_API_URL, + CrwClient, + CrwError, + CrwExtractCancelledError, + CrwTimeoutError, +} from "../dist/esm/index.js"; const origFetch = globalThis.fetch; const origEnv = { ...process.env }; @@ -66,6 +72,9 @@ test("HTTP-only methods throw in local mode", async () => { process.env.CRW_LOCAL = "1"; const c = new CrwClient(); await assert.rejects(() => c.extract({ urls: ["https://example.com"] }), /requires HTTP mode/); + await assert.rejects(() => c.startExtract({ urls: ["https://example.com"] }), /requires HTTP mode/); + await assert.rejects(() => c.getExtract("job-1"), /requires HTTP mode/); + await assert.rejects(() => c.cancelExtract("job-1"), /requires HTTP mode/); await assert.rejects(() => c.batchScrape(["https://example.com"]), /requires HTTP mode/); await assert.rejects(() => c.capabilities(), /requires HTTP mode/); await assert.rejects(() => c.changeTrackingDiff({ markdown: "a" }), /requires HTTP mode/); @@ -100,6 +109,7 @@ test("extract starts a /v1/extract job and returns per-URL results", async () => const postBody = JSON.parse(calls[0].init!.body as string); assert.deepEqual(postBody.urls, ["https://example.com"]); assert.equal(postBody.llmApiKey, "sk"); + assert.equal((calls[0].init!.headers as Record).Prefer, "respond-async"); assert.ok(calls[1].url.startsWith(`${CLOUD_API_URL}/v1/extract/job-1`)); assert.equal(results.length, 1); const r0 = results[0] as { url: string; data: { title: string } }; @@ -107,6 +117,115 @@ test("extract starts a /v1/extract job and returns per-URL results", async () => assert.equal(r0.data.title, "Hi"); }); +test("startExtract sends Prefer for managed and self-hosted fixtures", async () => { + const calls = mockFetch({ id: "job-1", status: "processing", urls: 1 }); + const managed = new CrwClient({ apiKey: "fc-test" }); + assert.equal((await managed.startExtract({ urls: ["https://example.com"], prompt: "x" })).id, "job-1"); + const selfHosted = new CrwClient({ apiUrl: "http://localhost:3000" }); + await selfHosted.startExtract({ urls: ["https://example.com"], schema: { type: "object" } }); + + assert.equal(calls[0].url, `${CLOUD_API_URL}/v1/extract`); + assert.equal(calls[1].url, "http://localhost:3000/v1/extract"); + for (const call of calls) { + assert.equal((call.init!.headers as Record).Prefer, "respond-async"); + } +}); + +test("extract preserves managed synchronous fixture while requesting async", async () => { + const calls = mockFetch({ + success: true, + results: [{ url: "https://example.com", status: "completed", data: { title: "Hi" } }], + }); + const c = new CrwClient({ apiKey: "fc-test" }); + const results = await c.extract({ urls: ["https://example.com"], prompt: "title" }); + assert.equal(results[0].status, "completed"); + assert.equal((calls[0].init!.headers as Record).Prefer, "respond-async"); +}); + +test("getExtract and cancelExtract use canonical status route", async () => { + const status = { + id: "job/one", + status: "cancelled", + results: [{ url: "https://example.com", status: "cancelled" }], + expiresAt: "2026-07-14T00:00:00Z", + creditsUsed: 0, + tokensUsed: 0, + }; + const calls = mockFetch(status); + const c = new CrwClient({ apiUrl: "http://localhost:3000" }); + assert.equal((await c.getExtract("job/one")).status, "cancelled"); + assert.equal((await c.cancelExtract("job/one")).results.length, 1); + assert.equal(calls[0].url, "http://localhost:3000/v1/extract/job%2Fone"); + assert.equal(calls[0].init!.method, "GET"); + assert.equal(calls[1].init!.method, "DELETE"); +}); + +test("extract treats cancelling as non-terminal then raises typed cancelled error", async () => { + const responses = [ + { id: "job-1", status: "processing", urls: 2 }, + { + id: "job-1", + status: "cancelling", + results: [ + { url: "https://a.example", status: "completed", data: { title: "A" } }, + { url: "https://b.example", status: "processing" }, + ], + expiresAt: "2026-07-14T00:00:00Z", + creditsUsed: 1, + tokensUsed: 9, + }, + { + id: "job-1", + status: "cancelled", + results: [ + { url: "https://a.example", status: "completed", data: { title: "A" } }, + { url: "https://b.example", status: "cancelled" }, + ], + expiresAt: "2026-07-14T00:00:00Z", + creditsUsed: 1, + tokensUsed: 9, + }, + ]; + globalThis.fetch = (async () => { + const body = responses.shift(); + return { ok: true, status: 200, statusText: "OK", text: async () => JSON.stringify(body) } as Response; + }) as typeof fetch; + const c = new CrwClient({ apiUrl: "http://localhost:3000" }); + await assert.rejects( + () => c.extract({ urls: ["https://a.example", "https://b.example"], prompt: "x", pollInterval: 0 }), + (error: unknown) => { + assert.ok(error instanceof CrwExtractCancelledError); + assert.equal(error.status.status, "cancelled"); + assert.equal(error.results[0].status, "completed"); + return true; + }, + ); +}); + +test("extract timeout performs best-effort DELETE", async () => { + const calls: Array<{ url: string; init?: RequestInit }> = []; + globalThis.fetch = (async (url: string, init?: RequestInit) => { + calls.push({ url: String(url), init }); + const body = init?.method === "POST" + ? { id: "job-1", status: "processing", urls: 1 } + : { + id: "job-1", + status: "cancelled", + results: [{ url: "https://example.com", status: "cancelled" }], + expiresAt: "2026-07-14T00:00:00Z", + creditsUsed: 0, + tokensUsed: 0, + }; + return { ok: true, status: 200, statusText: "OK", text: async () => JSON.stringify(body) } as Response; + }) as typeof fetch; + const c = new CrwClient({ apiUrl: "http://localhost:3000" }); + await assert.rejects( + () => c.extract({ urls: ["https://example.com"], prompt: "x", timeout: -1 }), + CrwTimeoutError, + ); + assert.equal(calls.at(-1)?.init?.method, "DELETE"); +}); + test("capabilities unwraps and uses GET /v1/capabilities", async () => { const calls = mockFetch({ version: "0.14.0" }); const c = new CrwClient({ apiKey: "fc-test" }); From 2f2ebf500b7ece81a2ac20737b9413c2ccbc7012 Mon Sep 17 00:00:00 2001 From: us Date: Tue, 14 Jul 2026 00:31:33 +0300 Subject: [PATCH 4/6] fix(extract): enforce lifecycle expiry invariants --- crates/crw-mcp-proto/src/lib.rs | 38 ++- crates/crw-server/src/routes/extract.rs | 21 +- crates/crw-server/src/routes/mcp.rs | 2 +- crates/crw-server/src/routes/v2/adapters.rs | 10 + crates/crw-server/src/routes/v2/extract.rs | 11 +- crates/crw-server/src/state.rs | 275 +++++++++++++++++--- crates/crw-server/tests/mcp.rs | 171 ++++++++++-- crates/crw-server/tests/v1_extract.rs | 60 ++++- 8 files changed, 495 insertions(+), 93 deletions(-) diff --git a/crates/crw-mcp-proto/src/lib.rs b/crates/crw-mcp-proto/src/lib.rs index 1290ba52..8137afc0 100644 --- a/crates/crw-mcp-proto/src/lib.rs +++ b/crates/crw-mcp-proto/src/lib.rs @@ -91,6 +91,7 @@ impl JsonRpcResponse { fn extract_accepted_output_schema() -> Value { json!({ "type": "object", + "additionalProperties": false, "properties": { "id": { "type": "string" }, "status": { "type": "string", "enum": ["processing"] }, @@ -103,12 +104,36 @@ fn extract_accepted_output_schema() -> Value { fn extract_status_output_schema() -> Value { json!({ "type": "object", + "additionalProperties": false, "properties": { "id": { "type": "string" }, - "status": { "type": "string" }, - "results": { "type": "array", "items": { "type": "object" } }, + "status": { + "type": "string", + "enum": ["processing", "cancelling", "completed", "failed", "cancelled"] + }, + "results": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "url": { "type": "string" }, + "status": { + "type": "string", + "enum": ["processing", "completed", "failed", "cancelled"] + }, + "data": { "type": "object", "additionalProperties": true }, + "error": { "type": "string" }, + "llmUsage": { "type": "object" }, + "basis": { "type": "array", "items": { "type": "object" } }, + "basisWarnings": { "type": "array", "items": { "type": "object" } }, + "llmInputHash": { "type": "string" } + }, + "required": ["url", "status"] + } + }, "error": { "type": "string" }, - "expiresAt": { "type": "string" }, + "expiresAt": { "type": "string", "format": "date-time" }, "creditsUsed": { "type": "integer" }, "tokensUsed": { "type": "integer" } }, @@ -872,9 +897,10 @@ mod tests { /// Phase 1 trim + Phase 3 annotations/titles + the two native extract tools the /// full 8-tool list was ~8017 bytes (~2673 est-tok). The canonical lifecycle /// adds one cancel tool plus required output schemas for start/status/cancel; - /// after minimizing those schemas, the 9-tool list is ~9621 bytes (~3207 - /// est-tok). The ceiling keeps ~1.3% headroom so further growth still fails. - const TOOLS_LIST_TOKEN_CEILING: usize = 3250; + /// after closing lifecycle statuses and typing every per-URL result field, + /// the 9-tool list is ~10705 bytes (~3569 est-tok). The ceiling keeps ~2% + /// headroom so further growth still fails. + const TOOLS_LIST_TOKEN_CEILING: usize = 3650; #[test] fn tools_list_token_budget() { diff --git a/crates/crw-server/src/routes/extract.rs b/crates/crw-server/src/routes/extract.rs index 533fa8da..1f61e7d1 100644 --- a/crates/crw-server/src/routes/extract.rs +++ b/crates/crw-server/src/routes/extract.rs @@ -17,7 +17,7 @@ use crw_core::evidence::{Basis, BasisWarning}; use crw_core::types::{ExtractOptions, LlmUsage, OutputFormat, ScrapeRequest}; use crate::error::AppError; -use crate::routes::v2::adapters::expires_at_rfc3339; +use crate::routes::v2::adapters::system_time_rfc3339; use crate::state::{AppState, ExtractRecord, PreparedUrl, UrlResult}; /// Native extract request. camelCase like every other v1 public type. @@ -269,12 +269,8 @@ pub struct ExtractStatusResponse { } /// The one canonical HTTP/MCP serializer for extract lifecycle state. -pub(crate) fn serialize_extract_status( - state: &AppState, - id: Uuid, - rec: ExtractRecord, -) -> ExtractStatusResponse { - let expires_at = expires_at_rfc3339(rec.created_at, state.config.crawler.job_ttl_secs); +pub(crate) fn serialize_extract_status(id: Uuid, rec: ExtractRecord) -> ExtractStatusResponse { + let expires_at = system_time_rfc3339(rec.expires_at); ExtractStatusResponse { id: id.to_string(), status: rec.status.as_str().to_string(), @@ -294,13 +290,8 @@ pub(crate) async fn get_extract_status( state: &AppState, id: Uuid, ) -> Result { - let rec = { - let jobs = state.extract_jobs.read().await; - jobs.get(&id) - .cloned() - .ok_or_else(|| CrwError::NotFound(format!("Extract job {id} not found")))? - }; - Ok(serialize_extract_status(state, id, rec)) + let rec = state.get_extract_job(id).await?; + Ok(serialize_extract_status(id, rec)) } pub(crate) async fn cancel_extract_status( @@ -308,7 +299,7 @@ pub(crate) async fn cancel_extract_status( id: Uuid, ) -> Result { let rec = state.cancel_extract_job(id).await?; - Ok(serialize_extract_status(state, id, rec)) + Ok(serialize_extract_status(id, rec)) } pub async fn get_extract( diff --git a/crates/crw-server/src/routes/mcp.rs b/crates/crw-server/src/routes/mcp.rs index 277199e7..48d44f1d 100644 --- a/crates/crw-server/src/routes/mcp.rs +++ b/crates/crw-server/src/routes/mcp.rs @@ -124,7 +124,7 @@ pub async fn call_tool(state: &AppState, tool_name: &str, args: Value) -> Result let id = state .start_extract_job(prepared.entries, prepared.template) .await; - Ok(json!({"success": true, "id": id.to_string(), "status": "processing", "urls": urls})) + Ok(json!({"id": id.to_string(), "status": "processing", "urls": urls})) } "crw_check_extract_status" => { use crate::routes::extract::get_extract_status; diff --git a/crates/crw-server/src/routes/v2/adapters.rs b/crates/crw-server/src/routes/v2/adapters.rs index 114a50f7..a403f4a4 100644 --- a/crates/crw-server/src/routes/v2/adapters.rs +++ b/crates/crw-server/src/routes/v2/adapters.rs @@ -234,6 +234,16 @@ pub fn expires_at_rfc3339(created_at: Instant, job_ttl_secs: u64) -> String { rfc3339_utc(now + remaining) } +/// Format a persisted absolute job expiry. Unlike `expires_at_rfc3339`, this +/// does not derive a new wall-clock value at read time. +pub fn system_time_rfc3339(expires_at: SystemTime) -> String { + let unix_secs = expires_at + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or(0); + rfc3339_utc(unix_secs) +} + /// Format a Unix-epoch second count as `YYYY-MM-DDTHH:MM:SS.000Z` (UTC). /// Hand-rolled (Howard Hinnant's `civil_from_days`) to avoid a chrono/time /// dependency. diff --git a/crates/crw-server/src/routes/v2/extract.rs b/crates/crw-server/src/routes/v2/extract.rs index d81ba825..0392add5 100644 --- a/crates/crw-server/src/routes/v2/extract.rs +++ b/crates/crw-server/src/routes/v2/extract.rs @@ -14,7 +14,7 @@ use uuid::Uuid; use crw_core::error::CrwError; use crw_core::types::{OutputFormat, ScrapeRequest}; -use super::adapters::expires_at_rfc3339; +use super::adapters::system_time_rfc3339; use crate::error::AppError; use crate::state::{AppState, ExtractStatus, PreparedUrl}; @@ -113,13 +113,8 @@ pub async fn get_extract( State(state): State, Path(id): Path, ) -> Result, AppError> { - let rec = { - let jobs = state.extract_jobs.read().await; - jobs.get(&id) - .cloned() - .ok_or_else(|| CrwError::NotFound(format!("Extract job {id} not found")))? - }; - let expires_at = expires_at_rfc3339(rec.created_at, state.config.crawler.job_ttl_secs); + let rec = state.get_extract_job(id).await?; + let expires_at = system_time_rfc3339(rec.expires_at); Ok(Json(V2ExtractStatusResponse { success: !matches!(rec.status, ExtractStatus::Failed), status: rec.status.as_str().to_string(), diff --git a/crates/crw-server/src/state.rs b/crates/crw-server/src/state.rs index b64c7d0a..e2295e61 100644 --- a/crates/crw-server/src/state.rs +++ b/crates/crw-server/src/state.rs @@ -12,7 +12,7 @@ use crw_search::SearxngClient; use futures::stream::StreamExt; use std::collections::HashMap; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime}; use tokio::sync::{RwLock, watch}; use uuid::Uuid; @@ -172,12 +172,19 @@ pub struct ExtractRecord { pub credits_used: u32, pub error: Option, pub created_at: Instant, + /// Absolute wall-clock expiry captured once at admission. Serializers use + /// this persisted value so repeated lifecycle envelopes never drift. + pub expires_at: SystemTime, /// The one URL currently dispatched by the sequential worker. Cancellation /// cannot cross its terminal barrier until this slot has persisted. pub claimed_index: Option, } impl ExtractRecord { + fn is_expired(&self, ttl: Duration) -> bool { + self.created_at.elapsed() >= ttl + } + /// Complete the cancellation barrier. Call only while holding the extract /// jobs write lock and after observing that no URL remains claimed. fn finish_cancellation(&mut self) { @@ -197,6 +204,38 @@ impl ExtractRecord { } self.status = ExtractStatus::Cancelled; } + + /// Commit the worker's final job-level write. DELETE races this method on + /// the same map lock, so exactly one transition can win and terminal state + /// is never rewritten by the loser. + fn finish_processing(&mut self) { + if self.status != ExtractStatus::Processing { + if self.status == ExtractStatus::Cancelling { + self.finish_cancellation(); + } + return; + } + + let any_ok = self + .per_url + .iter() + .any(|result| result.status == ExtractStatus::Completed); + if any_ok { + self.status = ExtractStatus::Completed; + self.data + .get_or_insert_with(|| serde_json::Value::Object(Default::default())); + } else { + self.status = ExtractStatus::Failed; + self.error = self + .per_url + .iter() + .rev() + .find_map(|result| result.error.clone()); + } + // Preserve the existing one-credit floor for a naturally finished + // all-failed job. Cancelled jobs retain only measured usage. + self.credits_used = self.credits_used.max(1); + } } /// Shared application state. @@ -369,11 +408,9 @@ impl AppState { } drop(jobs); - // Prune terminal extract jobs past TTL (keep processing and the - // cancellation barrier in-flight). - let mut ejobs = cleanup_state.extract_jobs.write().await; - ejobs - .retain(|_id, rec| !rec.status.is_terminal() || rec.created_at.elapsed() < ttl); + // TTL is authoritative for every extract lifecycle state. A + // stalled processing/cancelling job must not live forever. + cleanup_state.prune_expired_extract_jobs(ttl).await; } }); @@ -675,6 +712,10 @@ impl AppState { .collect(); { let mut jobs = self.extract_jobs.write().await; + let created_at = Instant::now(); + let wall_now = SystemTime::now(); + let expires_at = + wall_now.checked_add(Duration::from_secs(self.config.crawler.job_ttl_secs)); jobs.insert( id, ExtractRecord { @@ -684,7 +725,8 @@ impl AppState { tokens_used: 0, credits_used: 0, error: None, - created_at: Instant::now(), + created_at, + expires_at: expires_at.unwrap_or(wall_now), claimed_index: None, }, ); @@ -693,6 +735,7 @@ impl AppState { let renderer = self.renderer.clone(); let config = self.config.clone(); let extract_jobs = self.extract_jobs.clone(); + let finalizer = self.clone(); tokio::spawn(async move { // `/v2/extract` is a multi-URL background job — `Batch` traffic, so its @@ -826,38 +869,7 @@ impl AppState { } } - let mut jobs = extract_jobs.write().await; - if let Some(rec) = jobs.get_mut(&id) { - match rec.status { - ExtractStatus::Processing => { - let any_ok = rec - .per_url - .iter() - .any(|result| result.status == ExtractStatus::Completed); - if any_ok { - rec.status = ExtractStatus::Completed; - rec.data.get_or_insert_with(|| { - serde_json::Value::Object(Default::default()) - }); - } else { - rec.status = ExtractStatus::Failed; - rec.error = rec - .per_url - .iter() - .rev() - .find_map(|result| result.error.clone()); - } - // Preserve the existing one-credit floor for a - // naturally finished all-failed job. Cancelled - // jobs retain only actually measured usage. - rec.credits_used = rec.credits_used.max(1); - } - ExtractStatus::Cancelling => rec.finish_cancellation(), - ExtractStatus::Completed - | ExtractStatus::Failed - | ExtractStatus::Cancelled => {} - } - } + finalizer.finalize_extract_job(id).await; }) .await; }); @@ -869,6 +881,11 @@ impl AppState { /// calls are idempotent; terminal jobs are never rewritten. pub async fn cancel_extract_job(&self, id: Uuid) -> CrwResult { let mut jobs = self.extract_jobs.write().await; + let ttl = Duration::from_secs(self.config.crawler.job_ttl_secs); + if jobs.get(&id).is_some_and(|rec| rec.is_expired(ttl)) { + jobs.remove(&id); + return Err(CrwError::NotFound(format!("Extract job {id} not found"))); + } let rec = jobs .get_mut(&id) .ok_or_else(|| CrwError::NotFound(format!("Extract job {id} not found")))?; @@ -880,4 +897,184 @@ impl AppState { } Ok(rec.clone()) } + + /// TTL-aware canonical lookup shared by v1, v2, and MCP handlers. A write + /// lock makes expiry observation and removal one atomic operation. + pub async fn get_extract_job(&self, id: Uuid) -> CrwResult { + let mut jobs = self.extract_jobs.write().await; + let ttl = Duration::from_secs(self.config.crawler.job_ttl_secs); + if jobs.get(&id).is_some_and(|rec| rec.is_expired(ttl)) { + jobs.remove(&id); + return Err(CrwError::NotFound(format!("Extract job {id} not found"))); + } + jobs.get(&id) + .cloned() + .ok_or_else(|| CrwError::NotFound(format!("Extract job {id} not found"))) + } + + async fn finalize_extract_job(&self, id: Uuid) { + let mut jobs = self.extract_jobs.write().await; + if let Some(rec) = jobs.get_mut(&id) { + rec.finish_processing(); + } + } + + async fn prune_expired_extract_jobs(&self, ttl: Duration) -> usize { + let mut jobs = self.extract_jobs.write().await; + let before = jobs.len(); + jobs.retain(|_id, rec| !rec.is_expired(ttl)); + before - jobs.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::routes::extract::serialize_extract_status; + use serde_json::json; + use tokio::sync::oneshot; + + fn completed_record(created_at: Instant) -> ExtractRecord { + ExtractRecord { + status: ExtractStatus::Processing, + data: Some(json!({"last": 3})), + per_url: (1..=3) + .map(|index| UrlResult { + url: format!("https://example.com/{index}"), + status: ExtractStatus::Completed, + data: Some(json!({"index": index})), + error: None, + llm_usage: None, + basis: None, + basis_warnings: Vec::new(), + llm_input_hash: None, + }) + .collect(), + tokens_used: 30, + credits_used: 3, + error: None, + created_at, + expires_at: SystemTime::now() + Duration::from_secs(3_600), + claimed_index: None, + } + } + + fn spawn_final_write( + state: AppState, + id: Uuid, + ) -> (oneshot::Receiver<()>, tokio::task::JoinHandle<()>) { + let (ready_tx, ready_rx) = oneshot::channel(); + let handle = tokio::spawn(async move { + let _ = ready_tx.send(()); + state.finalize_extract_job(id).await; + }); + (ready_rx, handle) + } + + fn spawn_delete( + state: AppState, + id: Uuid, + ) -> (oneshot::Receiver<()>, tokio::task::JoinHandle<()>) { + let (ready_tx, ready_rx) = oneshot::channel(); + let handle = tokio::spawn(async move { + let _ = ready_tx.send(()); + state.cancel_extract_job(id).await.unwrap(); + }); + (ready_rx, handle) + } + + async fn run_final_write_delete_race(final_write_first: bool) -> (AppState, Uuid) { + let config: AppConfig = toml::from_str("").unwrap(); + let state = AppState::new(config).unwrap(); + let id = Uuid::new_v4(); + state + .extract_jobs + .write() + .await + .insert(id, completed_record(Instant::now())); + + // Hold the exact lock used by both operations, then enqueue them in a + // known order. Tokio's fair write lock releases to the first waiter, + // making both legal race winners deterministic rather than probabilistic. + let gate = state.extract_jobs.write().await; + let ((first_ready, first), (second_ready, second)) = if final_write_first { + ( + spawn_final_write(state.clone(), id), + spawn_delete(state.clone(), id), + ) + } else { + ( + spawn_delete(state.clone(), id), + spawn_final_write(state.clone(), id), + ) + }; + first_ready.await.unwrap(); + second_ready.await.unwrap(); + drop(gate); + first.await.unwrap(); + second.await.unwrap(); + (state, id) + } + + #[tokio::test] + async fn final_write_versus_delete_race_covers_both_winners_and_freezes_terminal_state() { + for (final_write_first, expected) in [ + (true, ExtractStatus::Completed), + (false, ExtractStatus::Cancelled), + ] { + let (state, id) = run_final_write_delete_race(final_write_first).await; + let terminal = state.get_extract_job(id).await.unwrap(); + assert_eq!(terminal.status, expected); + assert_eq!(terminal.per_url.len(), 3); + assert_eq!( + terminal + .per_url + .iter() + .map(|result| result.url.as_str()) + .collect::>(), + [ + "https://example.com/1", + "https://example.com/2", + "https://example.com/3" + ] + ); + assert!( + terminal + .per_url + .iter() + .all(|result| result.status == ExtractStatus::Completed) + ); + + let frozen = serde_json::to_value(serialize_extract_status(id, terminal)).unwrap(); + state.finalize_extract_job(id).await; + let repeated_delete = state.cancel_extract_job(id).await.unwrap(); + let repeated = + serde_json::to_value(serialize_extract_status(id, repeated_delete)).unwrap(); + assert_eq!(repeated, frozen, "terminal envelope must be immutable"); + } + } + + #[tokio::test] + async fn cleanup_removes_expired_nonterminal_extract_jobs() { + let config: AppConfig = toml::from_str("").unwrap(); + let state = AppState::new(config).unwrap(); + let processing_id = Uuid::new_v4(); + let cancelling_id = Uuid::new_v4(); + let old = Instant::now() - Duration::from_secs(5); + let mut cancelling = completed_record(old); + cancelling.status = ExtractStatus::Cancelling; + { + let mut jobs = state.extract_jobs.write().await; + jobs.insert(processing_id, completed_record(old)); + jobs.insert(cancelling_id, cancelling); + } + + assert_eq!( + state + .prune_expired_extract_jobs(Duration::from_secs(1)) + .await, + 2 + ); + assert!(state.extract_jobs.read().await.is_empty()); + } } diff --git a/crates/crw-server/tests/mcp.rs b/crates/crw-server/tests/mcp.rs index c4db2d2e..1d6a7251 100644 --- a/crates/crw-server/tests/mcp.rs +++ b/crates/crw-server/tests/mcp.rs @@ -8,7 +8,7 @@ use crw_core::types::{ use crw_server::app::create_app; use crw_server::state::{AppState, ExtractRecord, ExtractStatus, UrlResult}; use serde_json::{Value, json}; -use std::time::Instant; +use std::time::{Duration, Instant, SystemTime}; use uuid::Uuid; fn test_app() -> TestServer { @@ -30,6 +30,29 @@ fn test_app_with_search() -> TestServer { TestServer::new(app) } +fn pending_extract_record(created_at: Instant) -> ExtractRecord { + ExtractRecord { + status: ExtractStatus::Processing, + data: None, + per_url: vec![UrlResult { + url: "https://example.com".into(), + status: ExtractStatus::Processing, + data: None, + error: None, + llm_usage: None, + basis: None, + basis_warnings: Vec::new(), + llm_input_hash: None, + }], + tokens_used: 0, + credits_used: 0, + error: None, + created_at, + expires_at: SystemTime::now() + Duration::from_secs(3_600), + claimed_index: None, + } +} + fn mcp_request( method: &str, id: serde_json::Value, @@ -158,28 +181,11 @@ async fn mcp_extract_status_and_cancel_share_canonical_http_serializer() { let config: AppConfig = toml::from_str("").unwrap(); let state = AppState::new(config).unwrap(); let id = Uuid::new_v4(); - state.extract_jobs.write().await.insert( - id, - ExtractRecord { - status: ExtractStatus::Processing, - data: None, - per_url: vec![UrlResult { - url: "https://example.com".into(), - status: ExtractStatus::Processing, - data: None, - error: None, - llm_usage: None, - basis: None, - basis_warnings: Vec::new(), - llm_input_hash: None, - }], - tokens_used: 0, - credits_used: 0, - error: None, - created_at: Instant::now(), - claimed_index: None, - }, - ); + state + .extract_jobs + .write() + .await + .insert(id, pending_extract_record(Instant::now())); let server = TestServer::new(create_app(state)); let status: Value = server @@ -198,6 +204,13 @@ async fn mcp_extract_status_and_cancel_share_canonical_http_serializer() { assert_eq!(status["results"][0]["status"], "processing"); assert!(status["expiresAt"].is_string()); assert!(status.get("success").is_none()); + let schema = tool_output_schema("crw_check_extract_status").unwrap(); + let validator = jsonschema::validator_for(&schema).expect("extract schema compiles"); + let errors: Vec = validator + .iter_errors(status) + .map(|error| error.to_string()) + .collect(); + assert!(errors.is_empty(), "serializer/schema drift: {errors:#?}"); let cancelled: Value = server .post("/mcp") @@ -214,6 +227,118 @@ async fn mcp_extract_status_and_cancel_share_canonical_http_serializer() { assert_eq!(cancelled["results"][0]["status"], "cancelled"); } +#[tokio::test] +async fn mcp_extract_output_schemas_match_openapi_lifecycle_components() { + let openapi: Value = + serde_json::from_str(include_str!("../openapi/openapi.json")).expect("valid OpenAPI"); + let components = &openapi["components"]["schemas"]; + + let accepted = tool_output_schema("crw_extract").unwrap(); + let openapi_accepted = &components["ExtractAccepted"]; + assert_eq!(accepted["required"], openapi_accepted["required"]); + assert_eq!( + accepted["properties"]["status"], + openapi_accepted["properties"]["status"] + ); + assert_eq!( + accepted["properties"] + .as_object() + .unwrap() + .keys() + .collect::>(), + openapi_accepted["properties"] + .as_object() + .unwrap() + .keys() + .collect::>() + ); + + let status = tool_output_schema("crw_check_extract_status").unwrap(); + assert_eq!(status, tool_output_schema("crw_cancel_extract").unwrap()); + let openapi_status = &components["ExtractStatus"]; + assert_eq!(status["required"], openapi_status["required"]); + assert_eq!( + status["properties"]["status"], + openapi_status["properties"]["status"] + ); + + let result = &status["properties"]["results"]["items"]; + let openapi_result = &components["ExtractUrlResult"]; + assert_eq!(result["required"], openapi_result["required"]); + assert_eq!( + result["properties"] + .as_object() + .unwrap() + .keys() + .collect::>(), + openapi_result["properties"] + .as_object() + .unwrap() + .keys() + .collect::>() + ); + for field in [ + "url", + "status", + "data", + "error", + "llmUsage", + "basis", + "basisWarnings", + "llmInputHash", + ] { + assert_eq!( + result["properties"][field]["type"], openapi_result["properties"][field]["type"], + "type drift for ExtractUrlResult.{field}" + ); + } + assert_eq!( + result["properties"]["status"], + openapi_result["properties"]["status"] + ); +} + +#[tokio::test] +async fn mcp_extract_status_and_cancel_treat_expired_ids_as_not_found() { + let mut config: AppConfig = toml::from_str("").unwrap(); + config.crawler.job_ttl_secs = 1; + let state = AppState::new(config).unwrap(); + let ids = [Uuid::new_v4(), Uuid::new_v4()]; + let expired = pending_extract_record(Instant::now() - Duration::from_secs(2)); + { + let mut jobs = state.extract_jobs.write().await; + jobs.insert(ids[0], expired.clone()); + jobs.insert(ids[1], expired); + } + let server = TestServer::new(create_app(state.clone())); + + for (tool, id) in [ + ("crw_check_extract_status", ids[0]), + ("crw_cancel_extract", ids[1]), + ] { + let response: Value = server + .post("/mcp") + .content_type("application/json") + .json(&mcp_request( + "tools/call", + json!(301), + json!({"name": tool, "arguments": {"id": id}}), + )) + .await + .json(); + assert_eq!(response["result"]["isError"], true, "{tool}"); + assert!( + response["result"]["content"][0]["text"] + .as_str() + .unwrap() + .contains("not found"), + "{tool}" + ); + assert!(response["result"].get("structuredContent").is_none()); + } + assert!(state.extract_jobs.read().await.is_empty()); +} + #[tokio::test] async fn mcp_unknown_method_error_32601() { let server = test_app(); diff --git a/crates/crw-server/tests/v1_extract.rs b/crates/crw-server/tests/v1_extract.rs index e3ab5522..0e2c280a 100644 --- a/crates/crw-server/tests/v1_extract.rs +++ b/crates/crw-server/tests/v1_extract.rs @@ -11,7 +11,7 @@ use crw_core::config::AppConfig; use crw_server::app::create_app; use crw_server::state::{AppState, ExtractRecord, ExtractStatus, PreparedUrl, UrlResult}; use serde_json::{Value, json}; -use std::time::Instant; +use std::time::{Duration, Instant, SystemTime}; use uuid::Uuid; fn test_app() -> TestServer { @@ -130,6 +130,37 @@ async fn v1_extract_cancel_rejects_malformed_and_unknown_ids() { .assert_status(StatusCode::NOT_FOUND); } +#[tokio::test] +async fn expired_extract_ids_are_immediate_404_for_v1_get_delete_and_v2_get() { + let mut config: AppConfig = toml::from_str("").unwrap(); + config.crawler.job_ttl_secs = 1; + let state = AppState::new(config).unwrap(); + let ids = [Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()]; + let mut expired = seeded_record(&["https://expired.example"], None); + expired.created_at = Instant::now() - Duration::from_secs(2); + { + let mut jobs = state.extract_jobs.write().await; + for id in ids { + jobs.insert(id, expired.clone()); + } + } + let server = TestServer::new(create_app(state.clone())); + + server + .get(&format!("/v1/extract/{}", ids[0])) + .await + .assert_status(StatusCode::NOT_FOUND); + server + .delete(&format!("/v1/extract/{}", ids[1])) + .await + .assert_status(StatusCode::NOT_FOUND); + server + .get(&format!("/v2/extract/{}", ids[2])) + .await + .assert_status(StatusCode::NOT_FOUND); + assert!(state.extract_jobs.read().await.is_empty()); +} + fn pending_slot(url: &str) -> UrlResult { UrlResult { url: url.into(), @@ -152,6 +183,7 @@ fn seeded_record(urls: &[&str], claimed_index: Option) -> ExtractRecord { credits_used: 0, error: None, created_at: Instant::now(), + expires_at: SystemTime::now() + Duration::from_secs(3_600), claimed_index, } } @@ -183,6 +215,32 @@ async fn cancel_before_dispatch_is_terminal_ordered_and_idempotent() { assert_eq!(get, first, "GET and DELETE share the canonical serializer"); } +#[tokio::test] +async fn terminal_envelope_uses_the_persisted_expiry_without_recomputation() { + let config: AppConfig = toml::from_str("").unwrap(); + let state = AppState::new(config).unwrap(); + let id = Uuid::new_v4(); + let mut record = seeded_record(&["https://done.example"], None); + record.status = ExtractStatus::Completed; + record.per_url[0].status = ExtractStatus::Completed; + record.per_url[0].data = Some(json!({"done": true})); + state.extract_jobs.write().await.insert(id, record); + let server = TestServer::new(create_app(state.clone())); + + let first: Value = server.get(&format!("/v1/extract/{id}")).await.json(); + // Move the monotonic admission time without changing the persisted wall + // expiry. The old read-time derivation would shift expiresAt by 10 seconds. + state + .extract_jobs + .write() + .await + .get_mut(&id) + .unwrap() + .created_at = Instant::now() - Duration::from_secs(10); + let repeated: Value = server.delete(&format!("/v1/extract/{id}")).await.json(); + assert_eq!(repeated, first); +} + #[tokio::test] async fn claimed_slot_holds_cancelling_barrier_until_its_result_persists() { let config: AppConfig = toml::from_str("").unwrap(); From 8a9f9bee9815ff1c601ffff9eb7fd4c6c15d82ad Mon Sep 17 00:00:00 2001 From: us Date: Wed, 15 Jul 2026 16:38:11 +0300 Subject: [PATCH 5/6] fix(extract): don't relabel a finished job as cancelled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A DELETE that landed in the narrow window after the worker persisted the last URL but before finalize ran marked the whole job cancelled unconditionally, even though every per-URL result was completed with real data — a response that contradicted itself. Only report cancelled when at least one in-flight URL was actually stopped; otherwise settle the job by its outcomes, as the naturally finished job it is. Lockfile re-synced to the workspace version. --- Cargo.lock | 293 ++++++++++++++++++++++++++------- crates/crw-server/src/state.rs | 48 ++++-- 2 files changed, 266 insertions(+), 75 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ebdbba3..2f79926d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -448,9 +448,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.2.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" dependencies = [ "find-msvc-tools", "jobserver", @@ -693,6 +693,16 @@ dependencies = [ "url", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -736,6 +746,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" version = "0.5.16" @@ -778,7 +794,7 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crw-browse" -version = "0.24.0" +version = "0.23.0" dependencies = [ "anyhow", "async-trait", @@ -808,7 +824,7 @@ dependencies = [ [[package]] name = "crw-cli" -version = "0.24.0" +version = "0.23.0" dependencies = [ "anyhow", "axum", @@ -843,7 +859,7 @@ dependencies = [ [[package]] name = "crw-core" -version = "0.24.0" +version = "0.23.0" dependencies = [ "config", "crw-mcp-proto", @@ -862,7 +878,7 @@ dependencies = [ [[package]] name = "crw-crawl" -version = "0.24.0" +version = "0.23.0" dependencies = [ "crw-core", "crw-diff", @@ -888,7 +904,7 @@ dependencies = [ [[package]] name = "crw-diff" -version = "0.24.0" +version = "0.23.0" dependencies = [ "crw-core", "hex", @@ -903,7 +919,7 @@ dependencies = [ [[package]] name = "crw-extract" -version = "0.24.0" +version = "0.23.0" dependencies = [ "crw-core", "ego-tree", @@ -937,7 +953,7 @@ dependencies = [ [[package]] name = "crw-mcp" -version = "0.24.0" +version = "0.23.0" dependencies = [ "base64", "clap", @@ -953,7 +969,7 @@ dependencies = [ [[package]] name = "crw-mcp-proto" -version = "0.24.0" +version = "0.23.0" dependencies = [ "jsonschema", "serde", @@ -963,7 +979,7 @@ dependencies = [ [[package]] name = "crw-renderer" -version = "0.24.0" +version = "0.23.0" dependencies = [ "async-trait", "axum", @@ -992,7 +1008,7 @@ dependencies = [ [[package]] name = "crw-search" -version = "0.24.0" +version = "0.23.0" dependencies = [ "crw-core", "futures", @@ -1010,7 +1026,7 @@ dependencies = [ [[package]] name = "crw-server" -version = "0.24.0" +version = "0.23.0" dependencies = [ "axum", "axum-test", @@ -1851,6 +1867,76 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-net" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni", + "rand 0.10.2", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.2", + "ring", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni", + "moka", + "ndk-context", + "once_cell", + "parking_lot", + "rand 0.10.2", + "resolv-conf", + "smallvec", + "system-configuration", + "thiserror 2.0.18", + "tokio", + "tracing", +] + [[package]] name = "htmd" version = "0.5.4" @@ -1908,9 +1994,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.1.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http", @@ -1918,9 +2004,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", @@ -2198,11 +2284,27 @@ dependencies = [ "rustversion", ] +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + [[package]] name = "ipnet" version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +dependencies = [ + "serde", +] [[package]] name = "is_terminal_polyfill" @@ -2218,9 +2320,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.32" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "961d16382652bfdd8c6f68b223b26a8c93e0d475c672f414411db31c6c5c900e" +checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" dependencies = [ "defmt", "jiff-static", @@ -2234,9 +2336,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.32" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" +checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" dependencies = [ "proc-macro2", "quote", @@ -2245,9 +2347,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.8" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" +checksum = "6142247df1a93c2b3587402a19710be3e6e942f1581a1702e76408f2c21d6590" [[package]] name = "jiff-tzdb-platform" @@ -2606,9 +2708,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.2" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -2652,6 +2754,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -2775,6 +2883,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -2853,9 +2965,9 @@ checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pdf-inspector" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a814c9c6a4f520b34789e4519969b4c77325d2a757f13655091ef5f603e25d4f" +checksum = "156c914888fd0efdfc53911208b74c0adf00e9322b264ec5bfe10fd320f8ebae" dependencies = [ "env_logger", "log", @@ -2980,7 +3092,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared 0.11.3", - "rand 0.8.7", + "rand 0.8.6", ] [[package]] @@ -3113,6 +3225,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "pretty_assertions" version = "1.4.1" @@ -3157,7 +3280,7 @@ dependencies = [ "bit-vec", "bitflags 2.13.0", "num-traits", - "rand 0.9.5", + "rand 0.9.4", "rand_chacha", "rand_xorshift", "regex-syntax", @@ -3288,18 +3411,18 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.7" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "rand_core 0.6.4", ] [[package]] name = "rand" -version = "0.9.5" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", "rand_core 0.9.5", @@ -3450,9 +3573,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.13.0" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -3462,9 +3585,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.15" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -3491,6 +3614,7 @@ dependencies = [ "futures-core", "futures-util", "h2", + "hickory-resolver", "http", "http-body", "http-body-util", @@ -3500,6 +3624,7 @@ dependencies = [ "js-sys", "log", "mime_guess", + "once_cell", "percent-encoding", "pin-project-lite", "quinn", @@ -3531,6 +3656,12 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + [[package]] name = "ring" version = "0.17.14" @@ -3649,9 +3780,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ "aws-lc-rs", "once_cell", @@ -3689,7 +3820,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "jni", "log", @@ -3818,7 +3949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags 2.13.0", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -3987,9 +4118,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.7" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", @@ -4106,9 +4237,9 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" -version = "0.6.5" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.61.2", @@ -4116,9 +4247,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.9" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "stable_deref_trait" @@ -4261,6 +4392,27 @@ dependencies = [ "syn", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -4342,9 +4494,9 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.10" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", ] @@ -4400,9 +4552,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.12.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -4686,7 +4838,7 @@ dependencies = [ "http", "httparse", "log", - "rand 0.9.5", + "rand 0.9.4", "rustls", "rustls-pki-types", "sha1", @@ -4843,9 +4995,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.5" +version = "1.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" +checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -5039,6 +5191,12 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi-util" version = "0.1.11" @@ -5089,6 +5247,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -5191,9 +5360,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.4" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] @@ -5291,18 +5460,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "75726053136156d419e285b9b7eddaaea9e3fea6ce32eed44a89901f0bd98de1" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71" dependencies = [ "proc-macro2", "quote", @@ -5371,6 +5540,6 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.23" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/crates/crw-server/src/state.rs b/crates/crw-server/src/state.rs index e2295e61..26e26576 100644 --- a/crates/crw-server/src/state.rs +++ b/crates/crw-server/src/state.rs @@ -191,6 +191,7 @@ impl ExtractRecord { if self.status != ExtractStatus::Cancelling || self.claimed_index.is_some() { return; } + let mut cancelled_any = false; for result in &mut self.per_url { if result.status == ExtractStatus::Processing { result.status = ExtractStatus::Cancelled; @@ -200,22 +201,25 @@ impl ExtractRecord { result.basis = None; result.basis_warnings.clear(); result.llm_input_hash = None; + cancelled_any = true; } } - self.status = ExtractStatus::Cancelled; - } - - /// Commit the worker's final job-level write. DELETE races this method on - /// the same map lock, so exactly one transition can win and terminal state - /// is never rewritten by the loser. - fn finish_processing(&mut self) { - if self.status != ExtractStatus::Processing { - if self.status == ExtractStatus::Cancelling { - self.finish_cancellation(); - } - return; + if cancelled_any { + // A genuine cancel: at least one in-flight URL was actually stopped. + self.status = ExtractStatus::Cancelled; + } else { + // Every URL had already reached a terminal state before the cancel + // landed. Reporting "cancelled" would contradict the per-URL results + // (all completed/failed with real data), so settle it as the + // naturally finished job it actually is. + self.complete_from_outcomes(); } + } + /// Set the terminal job status from the per-URL outcomes of a job that ran + /// to the end: completed if any URL succeeded, otherwise failed, with the + /// one-credit floor a naturally finished job carries. + fn complete_from_outcomes(&mut self) { let any_ok = self .per_url .iter() @@ -236,6 +240,20 @@ impl ExtractRecord { // all-failed job. Cancelled jobs retain only measured usage. self.credits_used = self.credits_used.max(1); } + + /// Commit the worker's final job-level write. DELETE races this method on + /// the same map lock, so exactly one transition can win and terminal state + /// is never rewritten by the loser. + fn finish_processing(&mut self) { + if self.status != ExtractStatus::Processing { + if self.status == ExtractStatus::Cancelling { + self.finish_cancellation(); + } + return; + } + + self.complete_from_outcomes(); + } } /// Shared application state. @@ -1018,9 +1036,13 @@ mod tests { #[tokio::test] async fn final_write_versus_delete_race_covers_both_winners_and_freezes_terminal_state() { + // Whichever of the final write or the DELETE wins, a job whose URLs all + // completed settles as Completed: a late cancel that stopped nothing + // in-flight must not relabel a finished job (the per-URL results below + // are all Completed in both branches). for (final_write_first, expected) in [ (true, ExtractStatus::Completed), - (false, ExtractStatus::Cancelled), + (false, ExtractStatus::Completed), ] { let (state, id) = run_final_write_delete_race(final_write_first).await; let terminal = state.get_extract_job(id).await.unwrap(); From c663b429ce8279d2be19b792f0a01bc81c8caea4 Mon Sep 17 00:00:00 2001 From: us Date: Wed, 15 Jul 2026 16:49:45 +0300 Subject: [PATCH 6/6] fix(mcp): keep the success envelope on extract lifecycle tools The canonical cancellation work dropped `success` from crw_extract, crw_check_extract_status and crw_cancel_extract to mirror the native /v1 HTTP shape, but every other MCP tool returns `success` and clients already read it, so removing it silently broke those consumers. Re-add `success` on the MCP surface only (injected from the shared HTTP type, false only when every URL failed) and relax the MCP/OpenAPI parity check to ignore that one envelope field while still enforcing the data-field contract. --- crates/crw-mcp-proto/src/lib.rs | 6 ++-- crates/crw-server/src/routes/mcp.rs | 20 +++++++++-- crates/crw-server/tests/mcp.rs | 51 +++++++++++++++++++++-------- 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/crates/crw-mcp-proto/src/lib.rs b/crates/crw-mcp-proto/src/lib.rs index 8137afc0..981ab2c8 100644 --- a/crates/crw-mcp-proto/src/lib.rs +++ b/crates/crw-mcp-proto/src/lib.rs @@ -93,11 +93,12 @@ fn extract_accepted_output_schema() -> Value { "type": "object", "additionalProperties": false, "properties": { + "success": { "type": "boolean" }, "id": { "type": "string" }, "status": { "type": "string", "enum": ["processing"] }, "urls": { "type": "integer", "minimum": 0 } }, - "required": ["id", "status", "urls"] + "required": ["success", "id", "status", "urls"] }) } @@ -106,6 +107,7 @@ fn extract_status_output_schema() -> Value { "type": "object", "additionalProperties": false, "properties": { + "success": { "type": "boolean" }, "id": { "type": "string" }, "status": { "type": "string", @@ -137,7 +139,7 @@ fn extract_status_output_schema() -> Value { "creditsUsed": { "type": "integer" }, "tokensUsed": { "type": "integer" } }, - "required": ["id", "status", "results", "expiresAt", "creditsUsed", "tokensUsed"] + "required": ["success", "id", "status", "results", "expiresAt", "creditsUsed", "tokensUsed"] }) } diff --git a/crates/crw-server/src/routes/mcp.rs b/crates/crw-server/src/routes/mcp.rs index 48d44f1d..fb3c81ca 100644 --- a/crates/crw-server/src/routes/mcp.rs +++ b/crates/crw-server/src/routes/mcp.rs @@ -21,6 +21,20 @@ pub async fn validate_url(url: &str) -> Result<(), String> { crw_core::url_safety::validate_safe_url_resolved(&parsed).await } +/// Serialize an extract lifecycle response for MCP, re-adding the `success` +/// field MCP clients rely on. The shared HTTP `/v1` type omits it, so it is +/// injected only here; `success` is false only when every URL failed. +fn extract_status_value( + response: crate::routes::extract::ExtractStatusResponse, +) -> Result { + let success = response.status != "failed"; + let mut value = serde_json::to_value(response).map_err(|e| format!("serialize error: {e}"))?; + if let Some(obj) = value.as_object_mut() { + obj.insert("success".to_string(), Value::Bool(success)); + } + Ok(value) +} + pub async fn call_tool(state: &AppState, tool_name: &str, args: Value) -> Result { match tool_name { "crw_scrape" => { @@ -124,7 +138,7 @@ pub async fn call_tool(state: &AppState, tool_name: &str, args: Value) -> Result let id = state .start_extract_job(prepared.entries, prepared.template) .await; - Ok(json!({"id": id.to_string(), "status": "processing", "urls": urls})) + Ok(json!({"success": true, "id": id.to_string(), "status": "processing", "urls": urls})) } "crw_check_extract_status" => { use crate::routes::extract::get_extract_status; @@ -138,7 +152,7 @@ pub async fn call_tool(state: &AppState, tool_name: &str, args: Value) -> Result let response = get_extract_status(state, id) .await .map_err(|e| format!("{e}"))?; - serde_json::to_value(response).map_err(|e| format!("serialize error: {e}")) + extract_status_value(response) } "crw_cancel_extract" => { use crate::routes::extract::cancel_extract_status; @@ -152,7 +166,7 @@ pub async fn call_tool(state: &AppState, tool_name: &str, args: Value) -> Result let response = cancel_extract_status(state, id) .await .map_err(|e| format!("{e}"))?; - serde_json::to_value(response).map_err(|e| format!("serialize error: {e}")) + extract_status_value(response) } "crw_parse_file" => { use base64::Engine; diff --git a/crates/crw-server/tests/mcp.rs b/crates/crw-server/tests/mcp.rs index 1d6a7251..5a07d237 100644 --- a/crates/crw-server/tests/mcp.rs +++ b/crates/crw-server/tests/mcp.rs @@ -203,7 +203,9 @@ async fn mcp_extract_status_and_cancel_share_canonical_http_serializer() { assert_eq!(status["status"], "processing"); assert_eq!(status["results"][0]["status"], "processing"); assert!(status["expiresAt"].is_string()); - assert!(status.get("success").is_none()); + // MCP keeps the `success` envelope every other MCP tool returns; it is false + // only for a job whose every URL failed (here the job is still processing). + assert_eq!(status["success"], true); let schema = tool_output_schema("crw_check_extract_status").unwrap(); let validator = jsonschema::validator_for(&schema).expect("extract schema compiles"); let errors: Vec = validator @@ -225,6 +227,8 @@ async fn mcp_extract_status_and_cancel_share_canonical_http_serializer() { let cancelled = &cancelled["result"]["structuredContent"]; assert_eq!(cancelled["status"], "cancelled"); assert_eq!(cancelled["results"][0]["status"], "cancelled"); + // A cancelled job is not a failed one, so it still reports success: true. + assert_eq!(cancelled["success"], true); } #[tokio::test] @@ -233,30 +237,51 @@ async fn mcp_extract_output_schemas_match_openapi_lifecycle_components() { serde_json::from_str(include_str!("../openapi/openapi.json")).expect("valid OpenAPI"); let components = &openapi["components"]["schemas"]; + // MCP responses carry the `success` envelope every MCP tool returns, which + // the native HTTP `/v1` (openapi) shape deliberately omits. Compare the data + // contract by ignoring that one envelope field on the MCP side. + let required_without_success = |schema: &Value| -> Vec { + schema["required"] + .as_array() + .unwrap() + .iter() + .filter_map(|v| v.as_str()) + .filter(|s| *s != "success") + .map(str::to_string) + .collect() + }; + let keys_without_success = |schema: &Value| -> Vec { + schema["properties"] + .as_object() + .unwrap() + .keys() + .filter(|k| *k != "success") + .cloned() + .collect() + }; + let accepted = tool_output_schema("crw_extract").unwrap(); let openapi_accepted = &components["ExtractAccepted"]; - assert_eq!(accepted["required"], openapi_accepted["required"]); + assert_eq!( + required_without_success(&accepted), + required_without_success(openapi_accepted) + ); assert_eq!( accepted["properties"]["status"], openapi_accepted["properties"]["status"] ); assert_eq!( - accepted["properties"] - .as_object() - .unwrap() - .keys() - .collect::>(), - openapi_accepted["properties"] - .as_object() - .unwrap() - .keys() - .collect::>() + keys_without_success(&accepted), + keys_without_success(openapi_accepted) ); let status = tool_output_schema("crw_check_extract_status").unwrap(); assert_eq!(status, tool_output_schema("crw_cancel_extract").unwrap()); let openapi_status = &components["ExtractStatus"]; - assert_eq!(status["required"], openapi_status["required"]); + assert_eq!( + required_without_success(&status), + required_without_success(openapi_status) + ); assert_eq!( status["properties"]["status"], openapi_status["properties"]["status"]