From 315cb4a369b2923e44a0781397e776afecd60000 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Mon, 26 Oct 2026 10:32:18 +0100 Subject: [PATCH] feat(client-contract): identify clients, order for reading, and pin the base URL and role keys The client-identity rule (rules/client-identity.json, 18 vectors) has every API request carry X-DFO-Client: / (), such as dfo-python/0.1.0 (CPython 3.14) or dfo-ts/0.1.0 (browser). A caller's headers or header provider naming it raise header_invalid, and no request to another origin, a signed storage URL included, carries it. Scenario records keep no request headers, so a client's replay checks that each path and origin_path request carries the header and each url request does not. The column-order rule becomes reading-order (rules/reading-order.json, 10 vectors with rows). Columns run: the filled reading slots, then the other columns whose values differ between rows, then the constant columns, with stat_var last. A stack puts part first and judges the groups over the stacked table. Each member's rows are sorted by the filled slots without value and the time period, then by the varying columns (stat_var among them when it varies), then by the time period, so each series' years read together. Keys sort ascending with nulls last, ties keep served order, and members keep member order. The validator reads served Parquet once per file through served_table and checks every scenario's columns and every vector's rows. 28 scenarios change their columns only; no body file changes, so digests stay the server's. A base URL is valid when it is an absolute http or https URL with a host and no userinfo (an empty one included), query, fragment, dot segment, character outside visible ASCII, or IPv4 host in a form other than four decimal numbers. A relative URL is valid only in a TypeScript client inside a browser page, which resolves it against the page's address, and a valid value is used less its trailing slashes. Seven new vectors in rules/base-url.json pin these cases. The User-Agent sentence now says TypeScript sends dfo-ts/ outside a browser page or browser worker, in Node, Deno, Bun or a server runtime such as Cloudflare Workers. The role-keys rule (rules/role-keys.json, 13 vectors) names the keys of entities=: place for the subject, posted as observationAbout, and origin and destination for flows, posted as themselves. Spec.roles and specRoles(spec) list a spec's roles under those keys, a flow spec takes both directions, and a key outside the spec's roles raises invalid_argument naming the typed key and the spec's roles, so observationAbout and column names such as financingRecipient are refused. A spec that names no role posts place as observationAbout and leaves the rest to the server. --- python/client-contract/CHANGELOG.md | 5 +- python/client-contract/README.md | 11 +- .../src/dfo_client_contract/client-codes.json | 2 +- .../src/dfo_client_contract/contract.py | 4 +- .../src/dfo_client_contract/rules/README.md | 39 +- .../dfo_client_contract/rules/base-url.json | 55 + .../rules/client-identity.json | 174 ++ .../rules/column-order.json | 529 ------- .../rules/reading-order.json | 1399 +++++++++++++++++ .../dfo_client_contract/rules/role-keys.json | 355 +++++ .../rules/rules.schema.json | 500 +++++- .../scenarios/answer-definite-fetch.json | 4 +- .../answer-folded-parts-unnamed.json | 6 +- .../answer-operations-not-a-list.json | 4 +- .../answer-several-readings-folded.json | 6 +- .../scenarios/choose-candidate-two-parts.json | 2 +- .../scenarios/fetch-null-member.json | 4 +- .../fetch-ref-artifact-unlisted.json | 4 +- .../fetch-ref-expired-file-selector.json | 4 +- ...etch-ref-expired-rebuild-same-version.json | 4 +- .../fetch-ref-signed-url-refused.json | 4 +- .../scenarios/fetch-ref.json | 4 +- .../scenarios/fetch-retry-after-504.json | 4 +- .../fetch-spec-accepted-running.json | 2 +- .../scenarios/fetch-spec-async-build.json | 2 +- .../scenarios/fetch-spec-entity-dropped.json | 8 +- .../scenarios/fetch-spec-entity-rebound.json | 4 +- .../scenarios/fetch-spec-frame-layout.json | 2 +- .../scenarios/fetch-spec-group-narrowed.json | 2 +- .../fetch-spec-group-origin-rebound.json | 2 +- ...tch-spec-operation-failed-resubmitted.json | 2 +- .../fetch-spec-poll-pacing-changes.json | 2 +- .../fetch-spec-poll-rate-limited.json | 2 +- .../fetch-spec-poll-refused-backoff.json | 2 +- .../fetch-spec-poll-refused-retry-after.json | 2 +- .../scenarios/fetch-spec-poll-refused.json | 2 +- .../scenarios/fetch-strict-info-notice.json | 4 +- .../scenarios/mint-async-build.json | 2 +- .../scenarios/operation-resume.json | 2 +- python/client-contract/tests/test_rules.py | 153 +- tools/README.md | 2 +- tools/capture_client_contract.py | 43 +- tools/tests/test_capture_client_contract.py | 6 +- tools/tests/test_validate_client_contract.py | 83 +- tools/validate_client_contract.py | 248 ++- 45 files changed, 2963 insertions(+), 737 deletions(-) create mode 100644 python/client-contract/src/dfo_client_contract/rules/client-identity.json delete mode 100644 python/client-contract/src/dfo_client_contract/rules/column-order.json create mode 100644 python/client-contract/src/dfo_client_contract/rules/reading-order.json create mode 100644 python/client-contract/src/dfo_client_contract/rules/role-keys.json diff --git a/python/client-contract/CHANGELOG.md b/python/client-contract/CHANGELOG.md index 2bafe50..3b161a4 100644 --- a/python/client-contract/CHANGELOG.md +++ b/python/client-contract/CHANGELOG.md @@ -24,7 +24,10 @@ All notable changes to the `dfo-spec-kit-client-contract` Python package will be - `tools/validate_client_contract.py`, which checks the contract against its schemas, the served document and `registries/codes.json`, checks `client-codes.json` against them and against the reasons the key-lookup rule raises, and checks each scenario's `expect` against the outcomes its exchanges, `actions.json` and `client-codes.json` give, each Search request against the input its steps derive, a sentence lookup's `entity_groups` included, each build's `input.scope` against the narrowing its step names over the spec's bound roles, each build's options against its layout, each Search request's whole body, each download against its artifact's digest, each recorded scenario's commit against `served/SOURCE.json`, each problem document's code against `registries/codes.json` or, for the website proxy's, `proxy-codes.json`, each proxy problem document's content type, members and `retryable: true`, each 202 against an operation that reads `done`, and each absolute URL against the storage host, each ref example's members against the dataset members and downloaded files of the scenario it names, and the scenarios directory against a 16,000,000-byte budget for records and bodies together. A record that ends a run of retries while a retry remains does not fit. The outcomes include each long build's operation id, poll waits and resubmissions, and a `VersionUnavailableError`'s newest version. - The contract README states that a POST for a build identical to one pending or running returns that operation, so a client that stops waiting posts the same build to join it, and that `fetch` waits for a build with no whole-call deadline by default, with a download that delivers no byte for 30 s a retryable `network_error`. - A fetch of an answer or a spec leaves out party totals by default: its build posts `without_party_totals: true` in `options`, so the server drops the group totals of the identity's open party roles, and a role the caller binds keeps what it binds. Fetch steps take `party_totals: true` for the client option, `party_totals` in Python and `partyTotals` in TypeScript, false by default, which posts no `without_party_totals`. The wire flag and the ref format are unchanged, and a ref records `without_party_totals: true` for a member fetched with the default. `fetch-spec-party-totals-left-out` and `fetch-spec-party-totals-kept` pin both on DAC2a ODA to Kenya in 2022 with the origin open. A mint and a resumed operation post the plain rendering. -- The `column-order` rule and its vectors: a client presents a dataset's columns with the reading columns first, the time period, provider, recipient, subject, purpose, value, unit, flow type and provenance, each slot filled by the long layout's label or the frame layout's code, the flow type by CRS `crsTypeOfFlow` or else DAC2a `dac2aAidType`, then the rest in the server's order with `stat_var` last. Scenario outcomes give `columns` in that order, and the validator checks it. +- The `reading-order` rule and its vectors, which carry rows: a client presents a dataset's columns with the reading columns first, the time period, provider, recipient, subject, purpose, value, unit, flow type and provenance, each slot filled by the long layout's label or the frame layout's code, the flow type by CRS `crsTypeOfFlow` or else DAC2a `dac2aAidType`, then the columns whose values differ between rows, then those that hold one value, each group in the server's order, with `stat_var` last. It sorts each member's rows by the filled slots without `value` and the time period, then the varying columns, then the time period, ascending with nulls last and ties in served order, so each series' years read together. Scenario outcomes give `columns` in that order, and the validator checks it against the served files. +- The `client-identity` rule and its vectors: every request a client sends to the API carries `X-DFO-Client: / ()`, such as `dfo-python/0.1.0 (CPython 3.14)` or `dfo-ts/0.1.0 (Node 22.18)`, and `browser` as the runtime in a browser, with `pattern` matching every well-formed value. A request to another origin, a signed storage URL included, carries none. A caller's headers or header provider naming `X-DFO-Client` in any letter case raises `InvalidRequestError` `header_invalid`, whose condition now says so, and `User-Agent` stays as each client sends it. A scenario replay checks the header on each `path` and `origin_path` request and its absence on each `url` request. +- The `role-keys` rule and its vectors: a caller narrows a spec with `entities=` keyed by `place`, `origin` and `destination`, and a client posts `place` as the wire role `observationAbout`. `Spec.roles`, `specRoles(spec)` and the clients' errors name the caller keys, and a flow spec takes both directions. A key outside the spec's roles raises `InvalidRequestError` `invalid_argument` naming the key and the spec's roles, so `observationAbout` on GDP is refused naming `place` and `financingRecipient` on a flow spec naming `origin` and `destination`. A spec that names no role posts `place` as `observationAbout` and other keys as typed, and refuses `observationAbout` naming `place`. - The client rules and their vectors: the credentials path and file format, the key lookup order, with an empty `api_key` refused as `key_empty` and an empty profile key as `profile_without_key`, the key format check, the base URL order, `base_url_invalid` for an invalid value, and the cache directory and layout. +- The `base-url` rule states the whole validity rule, and seven new vectors pin it: no query, fragment, `.` or `..` segment, character outside visible ASCII, empty userinfo or non-decimal IPv4 host; trailing slashes dropped; a relative URL valid only in a TypeScript client inside a browser page. [unreleased]: https://github.com/DevelopmentFinanceObservatory/spec-kit/commits/main/python diff --git a/python/client-contract/README.md b/python/client-contract/README.md index 1a1734a..74a5ee3 100644 --- a/python/client-contract/README.md +++ b/python/client-contract/README.md @@ -118,7 +118,7 @@ An answer's gaps follow from its parts. A candidates part is a `choice` gap, its ## Client codes -`client-codes.json` lists, sorted by code, each code a client sets on an error it raises when no problem document carries one: the `code`, the error `class` it raises, `retryable`, and a one-line `condition`. `retryable` is the flag both clients generate onto the error: `deadline_exceeded`, `download_failed` and `network_error` are retryable, `digest_mismatch` and the rest are not, and `refusal_without_code` is retryable as its status is, which its row gives as `by_status`. Both clients set an error's `code` from this list, so every error a client raises carries a code. A refusal keeps its problem document's `code`, the website proxy's included. A refusal whose body carries no code raises `refusal_without_code` with the class its HTTP status maps to, which its row gives as `by_status`. No other row does. A problem document without a code still gives the error its `retryable` and `trace_id`, so a 503 whose body says `retryable: false` raises at once, which `search-refused-without-code` pins. Every refusal's trace id is the response's trace id, as the trace-id rule under Scenarios gives it. `search-refused-without-code` pins the body's winning over the header, and `search-unauthorized` the header's for a refusal with no body. No client code is a code in `actions.json` or `registries/codes.json`, and each class is one the scenario schema names. Each reason the key-lookup rule raises, `key_empty`, `profile_not_found`, `profile_without_key` and `credentials_file_invalid`, is a row of class `AuthenticationError`, as are `key_invalid`, a key outside visible ASCII, and `credentials_file_unreadable`, a credentials file that exists but cannot be read. `header_invalid`, a header value with a control character, `insecure_transport`, a key over plain http to a non-loopback host, and `base_url_invalid`, a base URL that is empty, not absolute http or https, or carries userinfo, are `InvalidRequestError` rows. `response_too_large`, a decoded body past its cap, is a `ServerError` row. None of these six is retryable. The list holds no warning. `warning-codes.json` lists, sorted by code, each code a client sets on a warning it issues itself, with its class and condition: `ref_digest_changed`, a `DfoWarning`, for a ref member whose `content_version` serves a file with another digest than the ref's, because the server rendered the same data again, where the client returns the data; `route_deprecated`, a `DfoDeprecationWarning`, for a `Deprecation` or `Sunset` header; `server_warning`, a `DfoWarning`, for an `x-dfo-warning` header; and `key_checksum_mismatch`, `key_not_base62` and `key_unknown_prefix`, each a `DfoWarning`, for a key that fails the key-format rule with that reason. Both clients generate their warning codes from it and issue no other. No warning code is a code in another table or in `registries/codes.json`. A `SearchWarning` carries the code of the one way the readings differ from the question, a departure's or a warning's code or the verb of an operation Search did not run, and a null code when they differ in several ways. A `DataWarning` carries the manifest warning's code. +`client-codes.json` lists, sorted by code, each code a client sets on an error it raises when no problem document carries one: the `code`, the error `class` it raises, `retryable`, and a one-line `condition`. `retryable` is the flag both clients generate onto the error: `deadline_exceeded`, `download_failed` and `network_error` are retryable, `digest_mismatch` and the rest are not, and `refusal_without_code` is retryable as its status is, which its row gives as `by_status`. Both clients set an error's `code` from this list, so every error a client raises carries a code. A refusal keeps its problem document's `code`, the website proxy's included. A refusal whose body carries no code raises `refusal_without_code` with the class its HTTP status maps to, which its row gives as `by_status`. No other row does. A problem document without a code still gives the error its `retryable` and `trace_id`, so a 503 whose body says `retryable: false` raises at once, which `search-refused-without-code` pins. Every refusal's trace id is the response's trace id, as the trace-id rule under Scenarios gives it. `search-refused-without-code` pins the body's winning over the header, and `search-unauthorized` the header's for a refusal with no body. No client code is a code in `actions.json` or `registries/codes.json`, and each class is one the scenario schema names. Each reason the key-lookup rule raises, `key_empty`, `profile_not_found`, `profile_without_key` and `credentials_file_invalid`, is a row of class `AuthenticationError`, as are `key_invalid`, a key outside visible ASCII, and `credentials_file_unreadable`, a credentials file that exists but cannot be read. `header_invalid`, a header value with a control character or a caller's header named `X-DFO-Client`, `insecure_transport`, a key over plain http to a non-loopback host, and `base_url_invalid`, a base URL that is empty, not absolute http or https, or carries userinfo, are `InvalidRequestError` rows. `response_too_large`, a decoded body past its cap, is a `ServerError` row. None of these six is retryable. The list holds no warning. `warning-codes.json` lists, sorted by code, each code a client sets on a warning it issues itself, with its class and condition: `ref_digest_changed`, a `DfoWarning`, for a ref member whose `content_version` serves a file with another digest than the ref's, because the server rendered the same data again, where the client returns the data; `route_deprecated`, a `DfoDeprecationWarning`, for a `Deprecation` or `Sunset` header; `server_warning`, a `DfoWarning`, for an `x-dfo-warning` header; and `key_checksum_mismatch`, `key_not_base62` and `key_unknown_prefix`, each a `DfoWarning`, for a key that fails the key-format rule with that reason. Both clients generate their warning codes from it and issue no other. No warning code is a code in another table or in `registries/codes.json`. A `SearchWarning` carries the code of the one way the readings differ from the question, a departure's or a warning's code or the verb of an operation Search did not run, and a null code when they differ in several ways. A `DataWarning` carries the manifest warning's code. ## Scenarios @@ -138,15 +138,16 @@ A step that acts on an earlier result follows one: `choose` follows `search` or An exchange's request names its target in one of three ways. `path` is a path the client composes under the base URL, such as `/search/resolve`. `origin_path` is a root-relative URL the server returned, such as an artifact's `url`, or the path and query of an absolute one on the API's origin, which resolves against the base URL's origin, whatever path that URL has. `url` is an absolute URL outside the API, such as the signed storage URL a files route redirects to, which the client fetches with a bare request carrying no client header and no key. `json` holds the request body members that must match; a recorded scenario lists every member the client sent. The response gives `status`, the `headers` a client reads, and a `json` body or a `body_file`, a binary body in `scenarios/`. A scenario's own body files are named `.` and `-.`; a synthetic scenario may instead name a body file its parent names. -A `newest_version` outcome gives the `content_version` the step found, or null. An answer outcome gives `kind`, `status`, `sentence`, `complete`, the distinct `specs` of its definite parts, and its `gaps`, each with the `part` index, the gap `kind` and the number of `options`. A dataset outcome gives `rows`, `columns`, `complete`, its `members`, each with `parts`, `spec_id`, `content_version` and `sentence`, and the `missing` gaps. For one member the columns are the artifact's columns in the order the column-order rule gives; a dataset of several members adds a `part` column and gives `part_rows`, one entry per member in member order with the `part` value of that member's rows and their number. A fetch of an answer also gives `search_warning`, whether the fetch issued a `SearchWarning`. An error outcome gives the error `class` and its `code`: a refusal's code, a warning's code, which for a `SearchWarning` may be null, or for any other error the client raises itself its code in `client-codes.json`. Every error outcome gives its `trace_id`: a refusal's, as the refusal rules below give it; a failed operation's, its error document's `trace_id`, else the `x-dfo-trace-id` header of the response that carried the failed resource; a `VersionUnavailableError`'s, its first refusal's, or null when nothing refused. An error the client raises itself follows one rule. An error that describes a response carries that response's trace id, and an error that describes the caller's input carries null. The trace id is the one a user quotes to find the fault in the server's logs. The errors that describe a response are `unexpected_response`, `contract_violation` on a body, `answer_declined`, `answer_incomplete`, and a strict fetch's `SearchWarning` or `DataWarning`. A response's trace id is the top-level `trace_id` member of its body when the body is a JSON object holding one as a non-empty string, a problem document or any other body, else its `x-dfo-trace-id` header, else null. A body `trace_id` that is empty or not a string reads as absent, so the header applies. No scenario holds one, and the validator's tests pin it. A Search answer's `diagnostics.trace_id` is not read, and `answer-timed-out` pins the header's winning over a `diagnostics.trace_id` that differs from it. `answer_declined`, `answer_incomplete`, a `SearchWarning`, and a `contract_violation` a choice raises on an option describe the held answer, so they carry the trace id of the Search response that answered it. The errors that describe the caller's input, `choice_invalid`, `invalid_argument` and the key-lookup reasons, carry null, and so do `digest_mismatch` and `download_failed`, which describe a storage-host fetch, and `deadline_exceeded`, `network_error` and `offline`, which describe no response. `search-no-json-body`, `fetch-spec-strict-warning`, `answer-timed-out`, `answer-empty-parts`, `fetch-spec-files-route-no-content` and `choose-option-two-part-query` pin a trace id on an error that describes a response, `operation-failed-without-error` the header of the response that carried a failed resource with no error, and `choose-refused-shared-label` a null one on a choice whose answer's response carries a trace id. A `VersionUnavailableError` also gives `newest_version`, the version the lookup found, or null. A dataset or error outcome of a step that made a long build gives `long_builds`, one entry per build the server ran on, in order: the `operation_id` the client polled last, the `poll_waits` in seconds before each poll, and the number of `resubmissions`. Any outcome of a step that retried a refused request gives `retry_waits`, the seconds before each retry in order, and any outcome of a step whose API responses issued warnings gives `warnings`, their codes from `warning-codes.json` in order. +A `newest_version` outcome gives the `content_version` the step found, or null. An answer outcome gives `kind`, `status`, `sentence`, `complete`, the distinct `specs` of its definite parts, and its `gaps`, each with the `part` index, the gap `kind` and the number of `options`. A dataset outcome gives `rows`, `columns`, `complete`, its `members`, each with `parts`, `spec_id`, `content_version` and `sentence`, and the `missing` gaps. For one member the columns are the artifact's columns in the order the reading-order rule gives; a dataset of several members adds a `part` column and gives `part_rows`, one entry per member in member order with the `part` value of that member's rows and their number. A fetch of an answer also gives `search_warning`, whether the fetch issued a `SearchWarning`. An error outcome gives the error `class` and its `code`: a refusal's code, a warning's code, which for a `SearchWarning` may be null, or for any other error the client raises itself its code in `client-codes.json`. Every error outcome gives its `trace_id`: a refusal's, as the refusal rules below give it; a failed operation's, its error document's `trace_id`, else the `x-dfo-trace-id` header of the response that carried the failed resource; a `VersionUnavailableError`'s, its first refusal's, or null when nothing refused. An error the client raises itself follows one rule. An error that describes a response carries that response's trace id, and an error that describes the caller's input carries null. The trace id is the one a user quotes to find the fault in the server's logs. The errors that describe a response are `unexpected_response`, `contract_violation` on a body, `answer_declined`, `answer_incomplete`, and a strict fetch's `SearchWarning` or `DataWarning`. A response's trace id is the top-level `trace_id` member of its body when the body is a JSON object holding one as a non-empty string, a problem document or any other body, else its `x-dfo-trace-id` header, else null. A body `trace_id` that is empty or not a string reads as absent, so the header applies. No scenario holds one, and the validator's tests pin it. A Search answer's `diagnostics.trace_id` is not read, and `answer-timed-out` pins the header's winning over a `diagnostics.trace_id` that differs from it. `answer_declined`, `answer_incomplete`, a `SearchWarning`, and a `contract_violation` a choice raises on an option describe the held answer, so they carry the trace id of the Search response that answered it. The errors that describe the caller's input, `choice_invalid`, `invalid_argument` and the key-lookup reasons, carry null, and so do `digest_mismatch` and `download_failed`, which describe a storage-host fetch, and `deadline_exceeded`, `network_error` and `offline`, which describe no response. `search-no-json-body`, `fetch-spec-strict-warning`, `answer-timed-out`, `answer-empty-parts`, `fetch-spec-files-route-no-content` and `choose-option-two-part-query` pin a trace id on an error that describes a response, `operation-failed-without-error` the header of the response that carried a failed resource with no error, and `choose-refused-shared-label` a null one on a choice whose answer's response carries a trace id. A `VersionUnavailableError` also gives `newest_version`, the version the lookup found, or null. A dataset or error outcome of a step that made a long build gives `long_builds`, one entry per build the server ran on, in order: the `operation_id` the client polled last, the `poll_waits` in seconds before each poll, and the number of `resubmissions`. Any outcome of a step that retried a refused request gives `retry_waits`, the seconds before each retry in order, and any outcome of a step whose API responses issued warnings gives `warnings`, their codes from `warning-codes.json` in order. -Each outcome follows from the exchanges and `actions.json`, and `tools/validate_client_contract.py` fails a record whose `expect` says otherwise. A record whose outcomes it cannot derive, because its exchanges do not fit its steps or a malformed member raises while it reads them, is reported as that record's problem, and the other records are checked. It runs the steps over the exchanges by the rules below: a refusal followed by the same request is a retry where the retry rules allow one, except on a poll, where the long-build rules count retries, and a record that ends a run of retries while the rules allow another does not fit, since the client makes it, a refusal's class is its code's action or else its status, the class of an error the client raises itself is its code's row in `client-codes.json`, a gap's kind is its reason's `gap_kind`, option and candidate counts are those the bodies hold, a dataset's members, `complete`, `missing` and `search_warning` follow from the answer and the manifests, each POST carries its body, each build posts the body the rules give for the spec it builds, so the builds' order and spec_ids are checked, each Search request posts the whole body `{"schema_version": 1, "input": ...}` with the input the rules give: the question, a choice's input, or a member's identity, each poll names the operation the client holds, each files route is the one the artifact's `url` names, each signed download is the URL the files route's `location` names, each retry waits the wait the retry rules give, and each response's warning headers issue the warnings the warning rules give. A dataset's `columns` are derived: the validator reads each downloaded Parquet file's columns from its footer, in the order the server wrote them, and orders them as the column-order rule gives. A dataset's `rows` and `part_rows` row counts are the Parquet bodies' and are not derived, except that `part_rows` is given exactly when the dataset has several members, each entry's `part` is its member's value and their rows sum to `rows`. +Each outcome follows from the exchanges and `actions.json`, and `tools/validate_client_contract.py` fails a record whose `expect` says otherwise. A record whose outcomes it cannot derive, because its exchanges do not fit its steps or a malformed member raises while it reads them, is reported as that record's problem, and the other records are checked. It runs the steps over the exchanges by the rules below: a refusal followed by the same request is a retry where the retry rules allow one, except on a poll, where the long-build rules count retries, and a record that ends a run of retries while the rules allow another does not fit, since the client makes it, a refusal's class is its code's action or else its status, the class of an error the client raises itself is its code's row in `client-codes.json`, a gap's kind is its reason's `gap_kind`, option and candidate counts are those the bodies hold, a dataset's members, `complete`, `missing` and `search_warning` follow from the answer and the manifests, each POST carries its body, each build posts the body the rules give for the spec it builds, so the builds' order and spec_ids are checked, each Search request posts the whole body `{"schema_version": 1, "input": ...}` with the input the rules give: the question, a choice's input, or a member's identity, each poll names the operation the client holds, each files route is the one the artifact's `url` names, each signed download is the URL the files route's `location` names, each retry waits the wait the retry rules give, and each response's warning headers issue the warnings the warning rules give. A dataset's `columns` are derived: the validator reads each downloaded Parquet file with pyarrow, its columns in the order the server wrote them and which of them vary between rows, and orders them as the reading-order rule gives. The outcome holds no rows, so the rule's vectors pin the row order. A dataset's `rows` and `part_rows` row counts are the Parquet bodies' and are not derived, except that `part_rows` is given exactly when the dataset has several members, each entry's `part` is its member's value and their rows sum to `rows`. A recorded scenario's request and response bodies validate against their route's schemas in `served/openapi.json`. A required member stays required when its schema admits null: the body carries it, as null or a value. A member the schema leaves optional may be absent or null, and a client reads both as null. A client replays a scenario by serving the exchanges in order: - A request matches the next exchange on its method and target, and on each listed `json` member, compared as parsed JSON values. +- Each `path` and `origin_path` request carries the `X-DFO-Client` header the client-identity rule gives, and each `url` request carries none. Records keep no request headers, so the replay checks the header on every request it serves. - A request with no exchange left, or one that matches none, fails the scenario, and so does an exchange the call never makes. - `path` resolves against the base URL the replay gives the client, and `origin_path` against that base URL's origin, so a replay can run under any base URL form. - A replay substitutes its own base URL's origin for the origin of the record's `origin.base_url`, a synthetic record's parent's, wherever a response names it, in its JSON body and its headers. `fetch-spec-same-origin-artifact-url` names the recorded origin in its manifest's artifact `url`s, and the validator replays every scenario under its recorded base URL and again under `https://replay.example.org` with the same path, so a record that holds only under its recorded host fails. @@ -159,7 +160,7 @@ The records also fix these client rules: - A fetch builds each distinct `spec_id` once, in part order, and finishes one member (build, the sentence when the client lacks it, files, download) before the next. - A fetch of a spec posts its narrowing as the build's `input.scope`, and posts no `scope` when it narrows nothing. A mint takes a layout and no narrowing, so it posts no `scope`. `window` posts `{"kind": "range", "start_year", "end_year"}` for a pair of years, or `{"kind": "all"}` or `{"kind": "latest"}`, and `facets` posts each axis as `{"ref": ...}`, and `facets: {}` names no axis and posts no `facets`. The server's `scope.entities` replaces the spec's whole entity binding, so `entities` posts a whole map: the roles the spec's `definition.identity.entities` binds, with each role the caller names replaced by its refs and each role the caller binds to null dropped. Naming `origin` alone on a spec bound to `origin` and `destination` keeps `destination`. A fetch that names no role, `entities: {}` included, posts no `entities`, so a fetch whose narrowing is all empty posts no `scope`; `fetch-spec-empty-narrowing` pins it. - A built `spec_id` that differs from the posted one, or a spec with no sentence, takes its sentence from Search: the client posts the built definition's `identity` as one `dataset_resubmit` identity entry, with the member's first part as `variable_text`, or `"dataset"` when no answer part names it, and checks the answer's `spec_id` equals the built one. The entry's `entity_groups` holds the spec's `resolved.entity_groups` for each role the built identity binds to the same entities as the spec's definition, so a window or facets narrowing keeps every group and a role the narrowing rebound or dropped loses its group. The entry leaves `entity_groups` out when it keeps no group, as for a resumed operation, whose spec the client does not hold. The server reads a role posted without its group as its members, so a narrowed spec of flows to the least developed countries would read "44 recipients" in place of "the 44 Least Developed Countries". -- A dataset of several members puts the `part` column first, then the members' columns, gathered in member order with each column once and ordered by the column-order rule. A column that members hold in different types widens as pyarrow's permissive promotion widens it, and a pair that cannot widen raises `ContractError` `contract_violation`. A member file that already holds a `part` column raises `contract_violation` when members stack. Rows and columns are not derived, so no scenario pins these. The `part` value on a member's rows is the member's `parts` joined by `"; "`, so a member that answers two folded parts reads `"health ODA disbursements; education ODA disbursements"`, or the member's sentence when `parts` is empty. A definite part whose `variable_text` is absent or empty adds nothing to its member's `parts`, which `answer-folded-parts-unnamed` pins. +- A dataset of several members puts the `part` column first, then the members' columns, gathered in member order with each column once and ordered by the reading-order rule, which also sorts each member's rows. A column that members hold in different types widens as pyarrow's permissive promotion widens it, and a pair that cannot widen raises `ContractError` `contract_violation`. A member file that already holds a `part` column raises `contract_violation` when members stack. Rows and columns are not derived, so no scenario pins these. The `part` value on a member's rows is the member's `parts` joined by `"; "`, so a member that answers two folded parts reads `"health ODA disbursements; education ODA disbursements"`, or the member's sentence when `parts` is empty. A definite part whose `variable_text` is absent or empty adds nothing to its member's `parts`, which `answer-folded-parts-unnamed` pins. - A choice names its part by index, by variable text, or by a word that one gap's options mention, and its option, a candidate among them, by index or label. A boolean part or option is neither an index nor a label. A candidate's label is the `text` of its spec's `resolved.sentence_plan` tokens in the `heading` segment, a token whose `segment` is not a string counted in it, joined and stripped of surrounding white space, or else its `resolved.rendered_sentence`. A candidate with neither a heading nor a sentence has a null label; no scenario pins it, and the validator's tests do. A no_data option's label is matched among the options that mention the word the part was named by. With no part named, exactly one gap must await a choice: a `choice` gap, or any gap with options. - A choice raises `InvalidRequestError` `choice_invalid` and makes no request when no gap, or more than one, awaits it, when the part is named by a boolean, is unknown, names more than one part or already has a reading, when the gap offers no options, or when the option is a boolean, is out of range or matches no label or several. No scenario can hold a boolean part or option, and the validator's tests pin both. It raises the same when an entry it would post cannot be built: a part with no `variable_text`, or a part of a status the client does not know that has no `restatement`. A chosen option whose `data_query` holds other than one part raises `ContractError`, and so does a chosen option with no `data_query`, in a one-part answer as in a several-part one, before any request. `choose-refused-shared-label`, `choose-refused-unknown-candidate-label`, `choose-refused-shared-candidate-label`, `choose-refused-part-without-variable-text`, `choose-refused-unknown-status`, `choose-option-two-part-query` and `choose-option-without-data-query` pin these, and `choose-candidate-by-label` pins a candidate chosen by its label. - Every pick of a candidate asks Search for the picked answer, and no client builds it locally. In a one-part answer the pick posts `dataset_resubmit`, without `raw_query`, with one identity entry: the candidate's identity and `entity_groups`, and the part's `variable_text`, `separate` and `operations` as below. The picked answer, its sentence with any "Assumed: …" clause, and its assumptions are Search's response. A choice of a no_data option in a one-part answer posts the option's `data_query` as the input. @@ -216,7 +217,7 @@ A record's `origin.base_url` is the base URL the run was given, a local app's in ## Rules -`rules/README.md` states the shared rules for credentials, key lookup and format, the base URL, disk caching, and the order a client presents a dataset's columns in. A rule applies where the client and runtime implement its capability. Python implements disk caching; TypeScript currently keeps no disk cache, so `cache-dir` and `cache-layout` apply to Python. Credentials-file rules apply to runtimes with local credentials support. Each rule's vectors are in `rules/.json`. +`rules/README.md` states the shared rules for credentials, key lookup and format, the base URL, disk caching, the order a client presents a dataset's columns and rows in, the keys a caller narrows a spec's roles by, and the `X-DFO-Client` header that identifies the client to the API. A rule applies where the client and runtime implement its capability. Python implements disk caching; TypeScript currently keeps no disk cache, so `cache-dir` and `cache-layout` apply to Python. Credentials-file rules apply to runtimes with local credentials support. Each rule's vectors are in `rules/.json`. ## Validation diff --git a/python/client-contract/src/dfo_client_contract/client-codes.json b/python/client-contract/src/dfo_client_contract/client-codes.json index 025fdf5..a0e8c9f 100644 --- a/python/client-contract/src/dfo_client_contract/client-codes.json +++ b/python/client-contract/src/dfo_client_contract/client-codes.json @@ -65,7 +65,7 @@ "code": "header_invalid", "class": "InvalidRequestError", "retryable": false, - "condition": "A header value holds a control character." + "condition": "A header value holds a control character, or a caller's header names X-DFO-Client." }, { "code": "insecure_transport", diff --git a/python/client-contract/src/dfo_client_contract/contract.py b/python/client-contract/src/dfo_client_contract/contract.py index 57b5484..944da09 100644 --- a/python/client-contract/src/dfo_client_contract/contract.py +++ b/python/client-contract/src/dfo_client_contract/contract.py @@ -16,7 +16,9 @@ "base-url", "cache-dir", "cache-layout", - "column-order", + "reading-order", + "client-identity", + "role-keys", ) diff --git a/python/client-contract/src/dfo_client_contract/rules/README.md b/python/client-contract/src/dfo_client_contract/rules/README.md index 1ecfdf9..3c52496 100644 --- a/python/client-contract/src/dfo_client_contract/rules/README.md +++ b/python/client-contract/src/dfo_client_contract/rules/README.md @@ -1,6 +1,6 @@ # Client rules -Clients implement these rules for the capabilities available in their runtime. Python implements every rule. TypeScript implements the key, credentials and base-URL rules in Node, and the column-order rule in every runtime; browser runtimes use the key sources available there. TypeScript currently keeps no disk cache, so `cache-dir` and `cache-layout` apply to Python. Clients implementing the same capability use the same rules and formats. Each rule's vectors are in `.json`, validated by `rules.schema.json`. In a vector, `env` lists the environment variables set, and an empty value counts as unset everywhere below. +Clients implement these rules for the capabilities available in their runtime. Python implements every rule. TypeScript implements the key, credentials and base-URL rules in Node, and the reading-order, client-identity and role-keys rules in every runtime; browser runtimes use the key sources available there. TypeScript currently keeps no disk cache, so `cache-dir` and `cache-layout` apply to Python. Clients implementing the same capability use the same rules and formats. Each rule's vectors are in `.json`, validated by `rules.schema.json`. In a vector, `env` lists the environment variables set, and an empty value counts as unset everywhere below. ## Credentials path (`credentials-path`) @@ -42,7 +42,7 @@ A client resolves its base URL when it is built. The first source found wins: 1. `DFO_BASE_URL`; 1. the public default, `https://developmentfinance.org/api/v1`. -Only the chosen source is checked. A base URL is valid when it is an absolute `http` or `https` URL with a host and no userinfo. An invalid one raises `base_url_invalid`, so an empty `base_url` argument raises it where an empty `DFO_BASE_URL` counts as unset. A valid value is used as given; the rule fixes the source and the validity, and leaves the URL's form to the client. A key sent over plain `http` to a host that is not loopback is refused by `insecure_transport`, which this rule does not decide. +Only the chosen source is checked. A base URL is valid when it is an absolute `http` or `https` URL with a host and none of these: userinfo, an empty one as in `https://@host/api` included; a query; a fragment; a `.` or `..` path segment; a character outside visible ASCII, a control character, a space or a non-ASCII character; or a host written as an IPv4 address in any form but four decimal numbers, such as `127.1`, `0x7f.1` or `2130706433`. An invalid one raises `base_url_invalid`, so an empty `base_url` argument raises it where an empty `DFO_BASE_URL` counts as unset. A relative URL is invalid, except in a TypeScript client inside a browser page, which resolves a relative base URL against the page's address. A valid value is used as given, less its trailing slashes, so the client's base URL reads `https://developmentfinance.org/api/v1` for `https://developmentfinance.org/api/v1/`; the rule fixes the source and the validity, and leaves the rest of the URL's form to the client. A key sent over plain `http` to a host that is not loopback is refused by `insecure_transport`, which this rule does not decide. ## Key format (`key-format`) @@ -70,9 +70,9 @@ Inside the cache directory: The key leaves out a rendering's `without_party_totals`: a cut that leaves rows out answers a version of its own, and one that leaves none out answers the plain version's file. A client writes an entry to a temporary name in the same directory and renames it into place, so a reader sees a complete entry. Concurrent writers of one content-addressed artifact write identical bytes. Manifest updates can replace another complete manifest and lose a cache candidate, so cache availability is best-effort. A read re-hashes an artifact, and a mismatch counts as a miss. -## Column order (`column-order`) +## Reading order (`reading-order`) -A client presents a dataset's columns in reading order, in every table it returns. `reading_columns` in `column-order.json` lists nine slots in order: the time period, the provider, the recipient, the subject, the purpose, the value, the unit, the flow type and the provenance. Each slot names the long layout's label column, then the frame layout's code column, and the flow type slot names the CRS pair, then the DAC2a aid type pair. The first name of a slot that the dataset holds fills the slot, and a slot whose names it holds none of stays empty. The filled slots come first, in slot order, then every other column in the order the server wrote it, with `stat_var` last. +A client presents a dataset's columns and rows in reading order, in every table it returns. `reading_columns` in `reading-order.json` lists nine slots in order: the time period, the provider, the recipient, the subject, the purpose, the value, the unit, the flow type and the provenance. Each slot names the long layout's label column, then the frame layout's code column, and the flow type slot names the CRS pair, then the DAC2a aid type pair. The first name of a slot that the dataset holds fills the slot, and a slot whose names it holds none of stays empty. | Slot | Long layout | Frame layout | | ----------- | ------------------------------------------------ | ------------------------------------------------------ | @@ -86,4 +86,33 @@ A client presents a dataset's columns in reading order, in every table it return | Flow type | `crsTypeOfFlow_label`, else `dac2aAidType_label` | `stat_var_crsTypeOfFlow`, else `stat_var_dac2aAidType` | | Provenance | `provenance_label` | `provenance` | -A CRS series in the long layout, 26 columns, reads `time_period`, `financingProvider_label`, `financingRecipient_label`, `crsPurpose_label`, `value`, `unit_label`, `crsTypeOfFlow_label` and `provenance_label`, then its 18 other columns, `stat_var` last. A DAC2a series leads with `dac2aAidType_label` in the flow type slot. A dataset of several members puts `part` first, then orders the members' columns, gathered in member order with each column once, by the same rule. A vector gives one dataset's `columns`, or a stacked dataset's `members`, each member's columns in the server's order; in a stack of a CRS member and a DAC2a member, `crsTypeOfFlow_label` fills the flow type slot and `dac2aAidType_label` falls among the other columns. The order moves columns and changes no value, and a client hashes a downloaded file before it orders the columns, so the digest and the ref keep the server's file. +The columns come in four groups: + +1. the filled slots, in slot order; +1. the other columns whose values differ between rows, in the order the server wrote them; +1. the other columns that hold one value in every row, null included, in the order the server wrote them; +1. `stat_var`, last whether or not it varies. + +A table with no row or one row has no column that varies. A dataset of several members puts `part` first and judges the groups over the stacked table, where a member's rows hold null in a column that member lacks. So a CRS series of Germany's basic health ODA to Nigeria, split by co-operation modality, reads `time_period`, `financingProvider_label`, `financingRecipient_label`, `crsPurpose_label`, `value`, `unit_label`, `crsTypeOfFlow_label` and `provenance_label`, then `crsCooperationModality` and `crsCooperationModality_label`, which tell its rows apart, then its 15 constant columns, `stat_var` last. + +The rows of each member are sorted by the filled slots in slot order, leaving out `value`, a measure, and the time period, then by the varying columns in presented order, `stat_var` after them when it varies, then by the time period, so each series' years read together. The keys of that series are the provider, recipient, purpose, unit, flow type and provenance labels, then `crsCooperationModality` and `crsCooperationModality_label`, then the time period, so it reads every A02 year, then every C01 year. Each key sorts ascending, numbers by value and strings by Unicode code point, with nulls last. Rows whose keys tie keep the order the server wrote them. The members follow one another in member order. + +A vector gives the dataset's `members`, each with its `part` when there are several, its `columns` and its `rows` as the server wrote them, and the `columns` and `rows` a client presents. The order moves columns and rows and changes no value. A client hashes a downloaded file before it orders anything, so the digest and the ref keep the server's file. + +## Client identity (`client-identity`) + +Every request a client sends to the API carries `X-DFO-Client: / ()`. `` is `dfo-python` or `dfo-ts`, and `` is the package version as the package states it. `` names the runtime and its major and minor version. In Python it is the implementation as `platform.python_implementation()` names it, then the interpreter's major and minor version, such as `CPython 3.14`. In TypeScript it is `Node`, `Deno` or `Bun` with the major and minor of that runtime's version, such as `Node 22.18`, and `browser` in any other runtime, a page or a worker. `pattern` in `client-identity.json` matches every well-formed value. A vector gives the `client`, the `version`, the `runtime` as its `name` and full `version`, or `browser` with no version, and the `value` the client sends. + +An API request is one to the base URL's origin, a `path` or `origin_path` request in a scenario. A request to any other origin, a signed storage URL included, carries no `X-DFO-Client`. In a browser, the files-route request the TypeScript client sends with `redirect: "follow"` carries the API key alone, because the browser forwards every other header to the storage host it is redirected to. + +A caller cannot replace or remove the header. A client checks the caller's `headers` argument before any request, and the headers a header provider returns each time the client calls it. A name equal to `X-DFO-Client` in any letter case raises `InvalidRequestError` `header_invalid`, whatever its value, the empty string included. A vector gives `via`, `argument` or `provider`, the caller's `headers`, and the `verdict`, `accept` or `refuse` with the `reason`. Every other header is the caller's to set, `User-Agent` included. Python sends `dfo-python/`, TypeScript sends `dfo-ts/` whenever it runs outside a browser page or browser worker (Node, Deno, Bun, or a server runtime such as Cloudflare Workers) and none inside one, where the browser forbids setting it, and a caller's `User-Agent` replaces the client's. + +## Role keys (`role-keys`) + +A caller narrows a spec by its entity roles with `entities=`, keyed by the caller keys `keys` in `role-keys.json` lists: `place`, the place a measure is about, and the flow directions `origin` and `destination`. A client posts `place` as the wire role `observationAbout` and each direction as itself. `Spec.roles` in Python, `specRoles(spec)` in TypeScript, and every error a client raises name the caller key. Each role has one caller spelling, so `observationAbout` is a wire role and no caller key. + +The roles a spec takes are those its `definition.identity` binds in `entities` or lists in `open_roles`, in that order, each under its caller key with the dcids the spec binds it to, or null when it binds none. A flow spec, one that names `origin` or `destination`, takes both, since a direction it neither binds nor opens reads as the total over it. The direction it leaves unnamed comes last. A spec that names no role takes `{}`. + +A client checks each key the caller types before any request. A key outside the spec's roles raises `InvalidRequestError` `invalid_argument`, and the message names the key and the spec's roles. So GDP for Kenya refuses `origin` and `observationAbout` naming `place`, and a flow spec refuses `place` and a column name such as `financingRecipient` naming `origin` and `destination`. A spec that names no role leaves the check to the server, since the client cannot tell which roles it takes. Such a spec posts `place` as `observationAbout` and any other key as typed, and still refuses `observationAbout` naming `place`. + +The server replaces a spec's whole binding with the posted one, so a client posts the spec's own `identity.entities` with each role the caller names replaced by its dcids and each role the caller binds to null dropped. A vector gives the spec's `identity` and its `roles`. A vector that narrows adds the caller's `entities`, then either `posted`, the wire entities the build posts, or the `error` with its `reason` and the keys its message `names`, the typed key first. diff --git a/python/client-contract/src/dfo_client_contract/rules/base-url.json b/python/client-contract/src/dfo_client_contract/rules/base-url.json index 4fde759..c53ffa5 100644 --- a/python/client-contract/src/dfo_client_contract/rules/base-url.json +++ b/python/client-contract/src/dfo_client_contract/rules/base-url.json @@ -117,6 +117,61 @@ }, "source": "argument", "url": "https://arg.example.org/api/v1" + }, + { + "name": "query-raises", + "base_url": "https://developmentfinance.org/api/v1?key=1", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "fragment-raises", + "base_url": "https://developmentfinance.org/api/v1#top", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "dot-segment-raises", + "base_url": "https://developmentfinance.org/api/../v1", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "control-character-raises", + "base_url": "https://developmentfinance.org/api\n/v1", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "non-decimal-ipv4-raises", + "base_url": "http://0x7f.1:8083/api", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "empty-userinfo-raises", + "base_url": "https://@developmentfinance.org/api/v1", + "env": {}, + "error": { + "reason": "base_url_invalid" + } + }, + { + "name": "trailing-slash-is-dropped", + "base_url": "https://developmentfinance.org/api/v1/", + "env": {}, + "source": "argument", + "url": "https://developmentfinance.org/api/v1" } ] } diff --git a/python/client-contract/src/dfo_client_contract/rules/client-identity.json b/python/client-contract/src/dfo_client_contract/rules/client-identity.json new file mode 100644 index 0000000..1d7756c --- /dev/null +++ b/python/client-contract/src/dfo_client_contract/rules/client-identity.json @@ -0,0 +1,174 @@ +{ + "format_version": 1, + "rule": "client-identity", + "header": "X-DFO-Client", + "pattern": "^(dfo-python|dfo-ts)/[0-9A-Za-z.+-]+ \\(([A-Za-z]+ [0-9]+\\.[0-9]+|browser)\\)$", + "vectors": [ + { + "name": "python-cpython", + "client": "dfo-python", + "version": "0.1.0", + "runtime": { + "name": "CPython", + "version": "3.14.0" + }, + "value": "dfo-python/0.1.0 (CPython 3.14)" + }, + { + "name": "python-pre-release-interpreter", + "client": "dfo-python", + "version": "0.1.0", + "runtime": { + "name": "CPython", + "version": "3.15.0a1" + }, + "value": "dfo-python/0.1.0 (CPython 3.15)" + }, + { + "name": "python-pypy", + "client": "dfo-python", + "version": "0.1.0", + "runtime": { + "name": "PyPy", + "version": "3.11.13" + }, + "value": "dfo-python/0.1.0 (PyPy 3.11)" + }, + { + "name": "python-pre-release-package", + "client": "dfo-python", + "version": "0.2.0rc1", + "runtime": { + "name": "CPython", + "version": "3.12.11" + }, + "value": "dfo-python/0.2.0rc1 (CPython 3.12)" + }, + { + "name": "python-local-version", + "client": "dfo-python", + "version": "0.2.0.dev3+g41c47f8", + "runtime": { + "name": "CPython", + "version": "3.13.5" + }, + "value": "dfo-python/0.2.0.dev3+g41c47f8 (CPython 3.13)" + }, + { + "name": "ts-node", + "client": "dfo-ts", + "version": "0.1.0", + "runtime": { + "name": "Node", + "version": "22.18.0" + }, + "value": "dfo-ts/0.1.0 (Node 22.18)" + }, + { + "name": "ts-deno", + "client": "dfo-ts", + "version": "0.1.0", + "runtime": { + "name": "Deno", + "version": "2.1.4" + }, + "value": "dfo-ts/0.1.0 (Deno 2.1)" + }, + { + "name": "ts-bun", + "client": "dfo-ts", + "version": "0.1.0", + "runtime": { + "name": "Bun", + "version": "1.2.19" + }, + "value": "dfo-ts/0.1.0 (Bun 1.2)" + }, + { + "name": "ts-browser", + "client": "dfo-ts", + "version": "0.1.0", + "runtime": { + "name": "browser" + }, + "value": "dfo-ts/0.1.0 (browser)" + }, + { + "name": "ts-pre-release-package", + "client": "dfo-ts", + "version": "0.2.0-rc.1", + "runtime": { + "name": "Node", + "version": "24.4.1" + }, + "value": "dfo-ts/0.2.0-rc.1 (Node 24.4)" + }, + { + "name": "header-argument-naming-it-refused", + "via": "argument", + "headers": { + "X-DFO-Client": "my-app/1.0" + }, + "verdict": "refuse", + "reason": "header_invalid" + }, + { + "name": "lower-case-name-refused", + "via": "argument", + "headers": { + "x-dfo-client": "my-app/1.0" + }, + "verdict": "refuse", + "reason": "header_invalid" + }, + { + "name": "empty-value-refused", + "via": "argument", + "headers": { + "X-DFO-CLIENT": "" + }, + "verdict": "refuse", + "reason": "header_invalid" + }, + { + "name": "provider-naming-it-refused", + "via": "provider", + "headers": { + "Accept-Language": "en", + "X-Dfo-Client": "dfo-python/0.1.0 (CPython 3.14)" + }, + "verdict": "refuse", + "reason": "header_invalid" + }, + { + "name": "other-headers-accepted", + "via": "argument", + "headers": { + "X-Request-Id": "f3a9" + }, + "verdict": "accept" + }, + { + "name": "user-agent-accepted", + "via": "argument", + "headers": { + "User-Agent": "my-app/2.0" + }, + "verdict": "accept" + }, + { + "name": "longer-name-accepted", + "via": "provider", + "headers": { + "X-DFO-Client-Id": "notebook-7" + }, + "verdict": "accept" + }, + { + "name": "no-headers-accepted", + "via": "argument", + "headers": {}, + "verdict": "accept" + } + ] +} diff --git a/python/client-contract/src/dfo_client_contract/rules/column-order.json b/python/client-contract/src/dfo_client_contract/rules/column-order.json deleted file mode 100644 index 47f3542..0000000 --- a/python/client-contract/src/dfo_client_contract/rules/column-order.json +++ /dev/null @@ -1,529 +0,0 @@ -{ - "format_version": 1, - "rule": "column-order", - "reading_columns": [ - [ - "time_period" - ], - [ - "financingProvider_label", - "financingProvider" - ], - [ - "financingRecipient_label", - "financingRecipient" - ], - [ - "observationAbout_label", - "observationAbout" - ], - [ - "crsPurpose_label", - "stat_var_crsPurpose" - ], - [ - "value" - ], - [ - "unit_label", - "unit" - ], - [ - "crsTypeOfFlow_label", - "stat_var_crsTypeOfFlow", - "dac2aAidType_label", - "stat_var_dac2aAidType" - ], - [ - "provenance_label", - "provenance" - ] - ], - "vectors": [ - { - "name": "long-oda-series", - "columns": [ - "stat_var", - "financingProvider", - "financingProvider_label", - "financingProvider_kind", - "financingRecipient", - "financingRecipient_label", - "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", - "crsPurpose", - "crsPurpose_label", - "crsTypeOfFinance", - "crsTypeOfFinance_label", - "crsTypeOfFlow", - "crsTypeOfFlow_label", - "time_period", - "value", - "unit", - "unit_label", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "provenance_label" - ], - "ordered": [ - "time_period", - "financingProvider_label", - "financingRecipient_label", - "crsPurpose_label", - "value", - "unit_label", - "crsTypeOfFlow_label", - "provenance_label", - "financingProvider", - "financingProvider_kind", - "financingRecipient", - "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", - "crsPurpose", - "crsTypeOfFinance", - "crsTypeOfFinance_label", - "crsTypeOfFlow", - "unit", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "stat_var" - ] - }, - { - "name": "long-one-recipient-role", - "columns": [ - "stat_var", - "stat_var_label", - "financingRecipient", - "financingRecipient_label", - "financingRecipient_kind", - "time_period", - "value", - "unit", - "unit_label", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "provenance_label" - ], - "ordered": [ - "time_period", - "financingRecipient_label", - "value", - "unit_label", - "provenance_label", - "stat_var_label", - "financingRecipient", - "financingRecipient_kind", - "unit", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "stat_var" - ] - }, - { - "name": "long-observation-about", - "columns": [ - "stat_var", - "stat_var_label", - "observationAbout", - "observationAbout_label", - "time_period", - "value", - "unit", - "unit_label", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "provenance_label" - ], - "ordered": [ - "time_period", - "observationAbout_label", - "value", - "unit_label", - "provenance_label", - "stat_var_label", - "observationAbout", - "unit", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "stat_var" - ] - }, - { - "name": "long-dac2a-series", - "columns": [ - "stat_var", - "financingProvider", - "financingProvider_label", - "financingProvider_kind", - "financingRecipient", - "financingRecipient_label", - "financingRecipient_kind", - "dac2aAidType", - "dac2aAidType_label", - "time_period", - "value", - "unit", - "unit_label", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "provenance_label" - ], - "ordered": [ - "time_period", - "financingProvider_label", - "financingRecipient_label", - "value", - "unit_label", - "dac2aAidType_label", - "provenance_label", - "financingProvider", - "financingProvider_kind", - "financingRecipient", - "financingRecipient_kind", - "dac2aAidType", - "unit", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "stat_var" - ] - }, - { - "name": "frame-layout-codes", - "columns": [ - "stat_var", - "financingProvider", - "financingRecipient", - "time_period", - "value", - "unit", - "measurementMethod", - "provenance", - "status", - "stat_var_crsCooperationModality", - "stat_var_crsPurpose", - "stat_var_crsTypeOfFinance", - "stat_var_crsTypeOfFlow" - ], - "ordered": [ - "time_period", - "financingProvider", - "financingRecipient", - "stat_var_crsPurpose", - "value", - "unit", - "stat_var_crsTypeOfFlow", - "provenance", - "measurementMethod", - "status", - "stat_var_crsCooperationModality", - "stat_var_crsTypeOfFinance", - "stat_var" - ] - }, - { - "name": "label-beats-code", - "columns": [ - "stat_var", - "unit", - "unit_label", - "value", - "provenance" - ], - "ordered": [ - "value", - "unit_label", - "provenance", - "unit", - "stat_var" - ] - }, - { - "name": "ordered-columns-keep-their-order", - "columns": [ - "time_period", - "financingProvider_label", - "financingRecipient_label", - "crsPurpose_label", - "value", - "unit_label", - "crsTypeOfFlow_label", - "provenance_label", - "financingProvider", - "financingProvider_kind", - "financingRecipient", - "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", - "crsPurpose", - "crsTypeOfFinance", - "crsTypeOfFinance_label", - "crsTypeOfFlow", - "unit", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "stat_var" - ], - "ordered": [ - "time_period", - "financingProvider_label", - "financingRecipient_label", - "crsPurpose_label", - "value", - "unit_label", - "crsTypeOfFlow_label", - "provenance_label", - "financingProvider", - "financingProvider_kind", - "financingRecipient", - "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", - "crsPurpose", - "crsTypeOfFinance", - "crsTypeOfFinance_label", - "crsTypeOfFlow", - "unit", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "stat_var" - ] - }, - { - "name": "no-reading-columns", - "columns": [ - "stat_var", - "x", - "y" - ], - "ordered": [ - "x", - "y", - "stat_var" - ] - }, - { - "name": "no-stat-var", - "columns": [ - "value", - "time_period", - "status" - ], - "ordered": [ - "time_period", - "value", - "status" - ] - }, - { - "name": "stack-crs-and-observation-about", - "members": [ - [ - "stat_var", - "financingProvider", - "financingProvider_label", - "financingProvider_kind", - "financingRecipient", - "financingRecipient_label", - "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", - "crsPurpose", - "crsPurpose_label", - "crsTypeOfFinance", - "crsTypeOfFinance_label", - "crsTypeOfFlow", - "crsTypeOfFlow_label", - "time_period", - "value", - "unit", - "unit_label", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "provenance_label" - ], - [ - "stat_var", - "stat_var_label", - "observationAbout", - "observationAbout_label", - "time_period", - "value", - "unit", - "unit_label", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "provenance_label" - ] - ], - "ordered": [ - "part", - "time_period", - "financingProvider_label", - "financingRecipient_label", - "observationAbout_label", - "crsPurpose_label", - "value", - "unit_label", - "crsTypeOfFlow_label", - "provenance_label", - "financingProvider", - "financingProvider_kind", - "financingRecipient", - "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", - "crsPurpose", - "crsTypeOfFinance", - "crsTypeOfFinance_label", - "crsTypeOfFlow", - "unit", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "stat_var_label", - "observationAbout", - "stat_var" - ] - }, - { - "name": "stack-crs-and-dac2a", - "members": [ - [ - "stat_var", - "financingProvider", - "financingProvider_label", - "financingProvider_kind", - "financingRecipient", - "financingRecipient_label", - "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", - "crsPurpose", - "crsPurpose_label", - "crsTypeOfFinance", - "crsTypeOfFinance_label", - "crsTypeOfFlow", - "crsTypeOfFlow_label", - "time_period", - "value", - "unit", - "unit_label", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "provenance_label" - ], - [ - "stat_var", - "financingProvider", - "financingProvider_label", - "financingProvider_kind", - "financingRecipient", - "financingRecipient_label", - "financingRecipient_kind", - "dac2aAidType", - "dac2aAidType_label", - "time_period", - "value", - "unit", - "unit_label", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "provenance_label" - ] - ], - "ordered": [ - "part", - "time_period", - "financingProvider_label", - "financingRecipient_label", - "crsPurpose_label", - "value", - "unit_label", - "crsTypeOfFlow_label", - "provenance_label", - "financingProvider", - "financingProvider_kind", - "financingRecipient", - "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", - "crsPurpose", - "crsTypeOfFinance", - "crsTypeOfFinance_label", - "crsTypeOfFlow", - "unit", - "currency", - "price_basis", - "base_year", - "measurementMethod", - "measurementMethod_label", - "provenance", - "dac2aAidType", - "dac2aAidType_label", - "stat_var" - ] - } - ] -} diff --git a/python/client-contract/src/dfo_client_contract/rules/reading-order.json b/python/client-contract/src/dfo_client_contract/rules/reading-order.json new file mode 100644 index 0000000..d4bb978 --- /dev/null +++ b/python/client-contract/src/dfo_client_contract/rules/reading-order.json @@ -0,0 +1,1399 @@ +{ + "format_version": 1, + "rule": "reading-order", + "reading_columns": [ + [ + "time_period" + ], + [ + "financingProvider_label", + "financingProvider" + ], + [ + "financingRecipient_label", + "financingRecipient" + ], + [ + "observationAbout_label", + "observationAbout" + ], + [ + "crsPurpose_label", + "stat_var_crsPurpose" + ], + [ + "value" + ], + [ + "unit_label", + "unit" + ], + [ + "crsTypeOfFlow_label", + "stat_var_crsTypeOfFlow", + "dac2aAidType_label", + "stat_var_dac2aAidType" + ], + [ + "provenance_label", + "provenance" + ] + ], + "vectors": [ + { + "name": "varying-column-follows-reading-columns", + "members": [ + { + "columns": [ + "stat_var", + "financingProvider", + "financingProvider_label", + "financingRecipient", + "financingRecipient_label", + "crsCooperationModality", + "crsCooperationModality_label", + "crsPurpose", + "crsPurpose_label", + "crsTypeOfFlow", + "crsTypeOfFlow_label", + "time_period", + "value", + "unit", + "unit_label", + "currency", + "provenance", + "provenance_label" + ], + "rows": [ + [ + "dc/sv/oda_health_grant", + "country/DEU", + "Germany", + "country/NGA", + "Nigeria", + "A02", + "Sector budget support", + "CRSPurposeCode12220", + "Basic health care", + "CRSTypeOfFlow10", + "ODA", + 2021, + 5.0, + "USDollar", + "US dollars", + "USD", + "provenance/CRS", + "OECD CRS" + ], + [ + "dc/sv/oda_health_loan", + "country/DEU", + "Germany", + "country/NGA", + "Nigeria", + "C01", + "Project-type interventions", + "CRSPurposeCode12220", + "Basic health care", + "CRSTypeOfFlow10", + "ODA", + 2020, + 7.0, + "USDollar", + "US dollars", + "USD", + "provenance/CRS", + "OECD CRS" + ], + [ + "dc/sv/oda_health_grant", + "country/DEU", + "Germany", + "country/NGA", + "Nigeria", + "A02", + "Sector budget support", + "CRSPurposeCode12220", + "Basic health care", + "CRSTypeOfFlow10", + "ODA", + 2020, + 4.0, + "USDollar", + "US dollars", + "USD", + "provenance/CRS", + "OECD CRS" + ], + [ + "dc/sv/oda_health_loan", + "country/DEU", + "Germany", + "country/NGA", + "Nigeria", + "C01", + "Project-type interventions", + "CRSPurposeCode12220", + "Basic health care", + "CRSTypeOfFlow10", + "ODA", + 2021, + 8.0, + "USDollar", + "US dollars", + "USD", + "provenance/CRS", + "OECD CRS" + ] + ] + } + ], + "columns": [ + "time_period", + "financingProvider_label", + "financingRecipient_label", + "crsPurpose_label", + "value", + "unit_label", + "crsTypeOfFlow_label", + "provenance_label", + "crsCooperationModality", + "crsCooperationModality_label", + "financingProvider", + "financingRecipient", + "crsPurpose", + "crsTypeOfFlow", + "unit", + "currency", + "provenance", + "stat_var" + ], + "rows": [ + [ + 2020, + "Germany", + "Nigeria", + "Basic health care", + 4.0, + "US dollars", + "ODA", + "OECD CRS", + "A02", + "Sector budget support", + "country/DEU", + "country/NGA", + "CRSPurposeCode12220", + "CRSTypeOfFlow10", + "USDollar", + "USD", + "provenance/CRS", + "dc/sv/oda_health_grant" + ], + [ + 2021, + "Germany", + "Nigeria", + "Basic health care", + 5.0, + "US dollars", + "ODA", + "OECD CRS", + "A02", + "Sector budget support", + "country/DEU", + "country/NGA", + "CRSPurposeCode12220", + "CRSTypeOfFlow10", + "USDollar", + "USD", + "provenance/CRS", + "dc/sv/oda_health_grant" + ], + [ + 2020, + "Germany", + "Nigeria", + "Basic health care", + 7.0, + "US dollars", + "ODA", + "OECD CRS", + "C01", + "Project-type interventions", + "country/DEU", + "country/NGA", + "CRSPurposeCode12220", + "CRSTypeOfFlow10", + "USDollar", + "USD", + "provenance/CRS", + "dc/sv/oda_health_loan" + ], + [ + 2021, + "Germany", + "Nigeria", + "Basic health care", + 8.0, + "US dollars", + "ODA", + "OECD CRS", + "C01", + "Project-type interventions", + "country/DEU", + "country/NGA", + "CRSPurposeCode12220", + "CRSTypeOfFlow10", + "USDollar", + "USD", + "provenance/CRS", + "dc/sv/oda_health_loan" + ] + ] + }, + { + "name": "all-constant-table", + "members": [ + { + "columns": [ + "stat_var", + "stat_var_label", + "observationAbout", + "observationAbout_label", + "time_period", + "value", + "unit", + "unit_label", + "currency", + "base_year", + "provenance", + "provenance_label" + ], + "rows": [ + [ + "dc/sv/gdp", + "GDP", + "country/KEN", + "Kenya", + 2023, + 108.0, + "USDollar", + "US dollars", + "USD", + null, + "provenance/WDI", + "World Bank WDI" + ], + [ + "dc/sv/gdp", + "GDP", + "country/KEN", + "Kenya", + 2022, + 113.4, + "USDollar", + "US dollars", + "USD", + null, + "provenance/WDI", + "World Bank WDI" + ] + ] + } + ], + "columns": [ + "time_period", + "observationAbout_label", + "value", + "unit_label", + "provenance_label", + "stat_var_label", + "observationAbout", + "unit", + "currency", + "base_year", + "provenance", + "stat_var" + ], + "rows": [ + [ + 2022, + "Kenya", + 113.4, + "US dollars", + "World Bank WDI", + "GDP", + "country/KEN", + "USDollar", + "USD", + null, + "provenance/WDI", + "dc/sv/gdp" + ], + [ + 2023, + "Kenya", + 108.0, + "US dollars", + "World Bank WDI", + "GDP", + "country/KEN", + "USDollar", + "USD", + null, + "provenance/WDI", + "dc/sv/gdp" + ] + ] + }, + { + "name": "one-row-table-is-constant", + "members": [ + { + "columns": [ + "stat_var", + "financingProvider", + "financingProvider_label", + "financingRecipient", + "financingRecipient_label", + "crsCooperationModality", + "crsCooperationModality_label", + "crsPurpose", + "crsPurpose_label", + "crsTypeOfFlow", + "crsTypeOfFlow_label", + "time_period", + "value", + "unit", + "unit_label", + "currency", + "provenance", + "provenance_label" + ], + "rows": [ + [ + "dc/sv/oda_health_grant", + "country/DEU", + "Germany", + "country/NGA", + "Nigeria", + "A02", + "Sector budget support", + "CRSPurposeCode12220", + "Basic health care", + "CRSTypeOfFlow10", + "ODA", + 2021, + 5.0, + "USDollar", + "US dollars", + "USD", + "provenance/CRS", + "OECD CRS" + ] + ] + } + ], + "columns": [ + "time_period", + "financingProvider_label", + "financingRecipient_label", + "crsPurpose_label", + "value", + "unit_label", + "crsTypeOfFlow_label", + "provenance_label", + "financingProvider", + "financingRecipient", + "crsCooperationModality", + "crsCooperationModality_label", + "crsPurpose", + "crsTypeOfFlow", + "unit", + "currency", + "provenance", + "stat_var" + ], + "rows": [ + [ + 2021, + "Germany", + "Nigeria", + "Basic health care", + 5.0, + "US dollars", + "ODA", + "OECD CRS", + "country/DEU", + "country/NGA", + "A02", + "Sector budget support", + "CRSPurposeCode12220", + "CRSTypeOfFlow10", + "USDollar", + "USD", + "provenance/CRS", + "dc/sv/oda_health_grant" + ] + ] + }, + { + "name": "empty-table", + "members": [ + { + "columns": [ + "stat_var", + "financingProvider", + "financingProvider_label", + "financingRecipient", + "financingRecipient_label", + "crsCooperationModality", + "crsCooperationModality_label", + "crsPurpose", + "crsPurpose_label", + "crsTypeOfFlow", + "crsTypeOfFlow_label", + "time_period", + "value", + "unit", + "unit_label", + "currency", + "provenance", + "provenance_label" + ], + "rows": [] + } + ], + "columns": [ + "time_period", + "financingProvider_label", + "financingRecipient_label", + "crsPurpose_label", + "value", + "unit_label", + "crsTypeOfFlow_label", + "provenance_label", + "financingProvider", + "financingRecipient", + "crsCooperationModality", + "crsCooperationModality_label", + "crsPurpose", + "crsTypeOfFlow", + "unit", + "currency", + "provenance", + "stat_var" + ], + "rows": [] + }, + { + "name": "frame-layout-codes", + "members": [ + { + "columns": [ + "stat_var", + "financingProvider", + "financingRecipient", + "time_period", + "value", + "unit", + "measurementMethod", + "provenance", + "status", + "stat_var_crsCooperationModality", + "stat_var_crsPurpose", + "stat_var_crsTypeOfFinance", + "stat_var_crsTypeOfFlow" + ], + "rows": [ + [ + "dc/sv/oda_health_grant", + "country/DEU", + "country/NGA", + 2021, + 5.0, + "USDollar", + "Gross", + "provenance/CRS", + null, + "A02", + "CRSPurposeCode12220", + "CRSTypeOfFinance110", + "CRSTypeOfFlow10" + ], + [ + "dc/sv/oda_health_grant", + "country/DEU", + "country/NGA", + 2020, + 4.0, + "USDollar", + "Gross", + "provenance/CRS", + "provisional", + "A02", + "CRSPurposeCode12220", + "CRSTypeOfFinance110", + "CRSTypeOfFlow10" + ] + ] + } + ], + "columns": [ + "time_period", + "financingProvider", + "financingRecipient", + "stat_var_crsPurpose", + "value", + "unit", + "stat_var_crsTypeOfFlow", + "provenance", + "status", + "measurementMethod", + "stat_var_crsCooperationModality", + "stat_var_crsTypeOfFinance", + "stat_var" + ], + "rows": [ + [ + 2020, + "country/DEU", + "country/NGA", + "CRSPurposeCode12220", + 4.0, + "USDollar", + "CRSTypeOfFlow10", + "provenance/CRS", + "provisional", + "Gross", + "A02", + "CRSTypeOfFinance110", + "dc/sv/oda_health_grant" + ], + [ + 2021, + "country/DEU", + "country/NGA", + "CRSPurposeCode12220", + 5.0, + "USDollar", + "CRSTypeOfFlow10", + "provenance/CRS", + null, + "Gross", + "A02", + "CRSTypeOfFinance110", + "dc/sv/oda_health_grant" + ] + ] + }, + { + "name": "null-sorts-last", + "members": [ + { + "columns": [ + "stat_var", + "financingProvider", + "financingProvider_label", + "financingProvider_kind", + "financingRecipient", + "financingRecipient_label", + "dac2aAidType", + "dac2aAidType_label", + "time_period", + "value", + "unit", + "unit_label", + "provenance", + "provenance_label" + ], + "rows": [ + [ + "dc/sv/oda_dac2a", + null, + null, + "unallocated", + "country/KEN", + "Kenya", + "DAC2aAidType206", + "ODA: Total Net", + 2022, + 1.0, + "USDollar", + "US dollars", + "provenance/DAC2a", + "OECD DAC2a" + ], + [ + "dc/sv/oda_dac2a", + "country/FRA", + "France", + "official", + "country/KEN", + "Kenya", + "DAC2aAidType206", + "ODA: Total Net", + 2022, + 2.0, + "USDollar", + "US dollars", + "provenance/DAC2a", + "OECD DAC2a" + ], + [ + "dc/sv/oda_dac2a", + "country/DEU", + "Germany", + "official", + "country/KEN", + "Kenya", + "DAC2aAidType206", + "ODA: Total Net", + 2022, + 3.0, + "USDollar", + "US dollars", + "provenance/DAC2a", + "OECD DAC2a" + ], + [ + "dc/sv/oda_dac2a", + "country/DEU", + "Germany", + "official", + "country/KEN", + "Kenya", + "DAC2aAidType206", + "ODA: Total Net", + 2021, + 4.0, + "USDollar", + "US dollars", + "provenance/DAC2a", + "OECD DAC2a" + ] + ] + } + ], + "columns": [ + "time_period", + "financingProvider_label", + "financingRecipient_label", + "value", + "unit_label", + "dac2aAidType_label", + "provenance_label", + "financingProvider", + "financingProvider_kind", + "financingRecipient", + "dac2aAidType", + "unit", + "provenance", + "stat_var" + ], + "rows": [ + [ + 2022, + "France", + "Kenya", + 2.0, + "US dollars", + "ODA: Total Net", + "OECD DAC2a", + "country/FRA", + "official", + "country/KEN", + "DAC2aAidType206", + "USDollar", + "provenance/DAC2a", + "dc/sv/oda_dac2a" + ], + [ + 2021, + "Germany", + "Kenya", + 4.0, + "US dollars", + "ODA: Total Net", + "OECD DAC2a", + "country/DEU", + "official", + "country/KEN", + "DAC2aAidType206", + "USDollar", + "provenance/DAC2a", + "dc/sv/oda_dac2a" + ], + [ + 2022, + "Germany", + "Kenya", + 3.0, + "US dollars", + "ODA: Total Net", + "OECD DAC2a", + "country/DEU", + "official", + "country/KEN", + "DAC2aAidType206", + "USDollar", + "provenance/DAC2a", + "dc/sv/oda_dac2a" + ], + [ + 2022, + null, + "Kenya", + 1.0, + "US dollars", + "ODA: Total Net", + "OECD DAC2a", + null, + "unallocated", + "country/KEN", + "DAC2aAidType206", + "USDollar", + "provenance/DAC2a", + "dc/sv/oda_dac2a" + ] + ] + }, + { + "name": "stat-var-varying-keys-series", + "members": [ + { + "columns": [ + "stat_var", + "stat_var_label", + "observationAbout", + "observationAbout_label", + "time_period", + "value", + "unit", + "unit_label", + "currency", + "base_year", + "provenance", + "provenance_label" + ], + "rows": [ + [ + "dc/sv/gdp_ppp", + "GDP", + "country/KEN", + "Kenya", + 2022, + 120.0, + "USDollar", + "US dollars", + "USD", + null, + "provenance/WDI", + "World Bank WDI" + ], + [ + "dc/sv/gdp", + "GDP", + "country/KEN", + "Kenya", + 2021, + 110.0, + "USDollar", + "US dollars", + "USD", + null, + "provenance/WDI", + "World Bank WDI" + ], + [ + "dc/sv/gdp_ppp", + "GDP", + "country/KEN", + "Kenya", + 2021, + 118.0, + "USDollar", + "US dollars", + "USD", + null, + "provenance/WDI", + "World Bank WDI" + ], + [ + "dc/sv/gdp", + "GDP", + "country/KEN", + "Kenya", + 2022, + 113.4, + "USDollar", + "US dollars", + "USD", + null, + "provenance/WDI", + "World Bank WDI" + ] + ] + } + ], + "columns": [ + "time_period", + "observationAbout_label", + "value", + "unit_label", + "provenance_label", + "stat_var_label", + "observationAbout", + "unit", + "currency", + "base_year", + "provenance", + "stat_var" + ], + "rows": [ + [ + 2021, + "Kenya", + 110.0, + "US dollars", + "World Bank WDI", + "GDP", + "country/KEN", + "USDollar", + "USD", + null, + "provenance/WDI", + "dc/sv/gdp" + ], + [ + 2022, + "Kenya", + 113.4, + "US dollars", + "World Bank WDI", + "GDP", + "country/KEN", + "USDollar", + "USD", + null, + "provenance/WDI", + "dc/sv/gdp" + ], + [ + 2021, + "Kenya", + 118.0, + "US dollars", + "World Bank WDI", + "GDP", + "country/KEN", + "USDollar", + "USD", + null, + "provenance/WDI", + "dc/sv/gdp_ppp" + ], + [ + 2022, + "Kenya", + 120.0, + "US dollars", + "World Bank WDI", + "GDP", + "country/KEN", + "USDollar", + "USD", + null, + "provenance/WDI", + "dc/sv/gdp_ppp" + ] + ] + }, + { + "name": "ties-keep-served-order", + "members": [ + { + "columns": [ + "stat_var", + "stat_var_label", + "observationAbout", + "observationAbout_label", + "time_period", + "value", + "unit", + "unit_label", + "currency", + "base_year", + "provenance", + "provenance_label" + ], + "rows": [ + [ + "dc/sv/gdp", + "GDP", + "country/KEN", + "Kenya", + 2022, + 2.0, + "USDollar", + "US dollars", + "USD", + null, + "provenance/WDI", + "World Bank WDI" + ], + [ + "dc/sv/gdp", + "GDP", + "country/KEN", + "Kenya", + 2022, + 1.0, + "USDollar", + "US dollars", + "USD", + null, + "provenance/WDI", + "World Bank WDI" + ], + [ + "dc/sv/gdp", + "GDP", + "country/KEN", + "Kenya", + 2021, + 3.0, + "USDollar", + "US dollars", + "USD", + null, + "provenance/WDI", + "World Bank WDI" + ] + ] + } + ], + "columns": [ + "time_period", + "observationAbout_label", + "value", + "unit_label", + "provenance_label", + "stat_var_label", + "observationAbout", + "unit", + "currency", + "base_year", + "provenance", + "stat_var" + ], + "rows": [ + [ + 2021, + "Kenya", + 3.0, + "US dollars", + "World Bank WDI", + "GDP", + "country/KEN", + "USDollar", + "USD", + null, + "provenance/WDI", + "dc/sv/gdp" + ], + [ + 2022, + "Kenya", + 2.0, + "US dollars", + "World Bank WDI", + "GDP", + "country/KEN", + "USDollar", + "USD", + null, + "provenance/WDI", + "dc/sv/gdp" + ], + [ + 2022, + "Kenya", + 1.0, + "US dollars", + "World Bank WDI", + "GDP", + "country/KEN", + "USDollar", + "USD", + null, + "provenance/WDI", + "dc/sv/gdp" + ] + ] + }, + { + "name": "stack-crs-and-dac2a", + "members": [ + { + "part": "health ODA", + "columns": [ + "stat_var", + "financingProvider", + "financingProvider_label", + "financingRecipient", + "financingRecipient_label", + "crsCooperationModality", + "crsCooperationModality_label", + "crsPurpose", + "crsPurpose_label", + "crsTypeOfFlow", + "crsTypeOfFlow_label", + "time_period", + "value", + "unit", + "unit_label", + "currency", + "provenance", + "provenance_label" + ], + "rows": [ + [ + "dc/sv/oda_health_grant", + "country/DEU", + "Germany", + "country/NGA", + "Nigeria", + "A02", + "Sector budget support", + "CRSPurposeCode12220", + "Basic health care", + "CRSTypeOfFlow10", + "ODA", + 2021, + 5.0, + "USDollar", + "US dollars", + "USD", + "provenance/CRS", + "OECD CRS" + ], + [ + "dc/sv/oda_health_grant", + "country/DEU", + "Germany", + "country/NGA", + "Nigeria", + "A02", + "Sector budget support", + "CRSPurposeCode12220", + "Basic health care", + "CRSTypeOfFlow10", + "ODA", + 2020, + 4.0, + "USDollar", + "US dollars", + "USD", + "provenance/CRS", + "OECD CRS" + ] + ] + }, + { + "part": "total ODA", + "columns": [ + "stat_var", + "financingProvider", + "financingProvider_label", + "financingProvider_kind", + "financingRecipient", + "financingRecipient_label", + "dac2aAidType", + "dac2aAidType_label", + "time_period", + "value", + "unit", + "unit_label", + "provenance", + "provenance_label" + ], + "rows": [ + [ + "dc/sv/oda_dac2a", + "country/DEU", + "Germany", + "official", + "country/KEN", + "Kenya", + "DAC2aAidType206", + "ODA: Total Net", + 2021, + 40.0, + "USDollar", + "US dollars", + "provenance/DAC2a", + "OECD DAC2a" + ], + [ + "dc/sv/oda_dac2a", + "country/DEU", + "Germany", + "official", + "country/KEN", + "Kenya", + "DAC2aAidType206", + "ODA: Total Net", + 2020, + 30.0, + "USDollar", + "US dollars", + "provenance/DAC2a", + "OECD DAC2a" + ] + ] + } + ], + "columns": [ + "part", + "time_period", + "financingProvider_label", + "financingRecipient_label", + "crsPurpose_label", + "value", + "unit_label", + "crsTypeOfFlow_label", + "provenance_label", + "financingRecipient", + "crsCooperationModality", + "crsCooperationModality_label", + "crsPurpose", + "crsTypeOfFlow", + "currency", + "provenance", + "financingProvider_kind", + "dac2aAidType", + "dac2aAidType_label", + "financingProvider", + "unit", + "stat_var" + ], + "rows": [ + [ + "health ODA", + 2020, + "Germany", + "Nigeria", + "Basic health care", + 4.0, + "US dollars", + "ODA", + "OECD CRS", + "country/NGA", + "A02", + "Sector budget support", + "CRSPurposeCode12220", + "CRSTypeOfFlow10", + "USD", + "provenance/CRS", + null, + null, + null, + "country/DEU", + "USDollar", + "dc/sv/oda_health_grant" + ], + [ + "health ODA", + 2021, + "Germany", + "Nigeria", + "Basic health care", + 5.0, + "US dollars", + "ODA", + "OECD CRS", + "country/NGA", + "A02", + "Sector budget support", + "CRSPurposeCode12220", + "CRSTypeOfFlow10", + "USD", + "provenance/CRS", + null, + null, + null, + "country/DEU", + "USDollar", + "dc/sv/oda_health_grant" + ], + [ + "total ODA", + 2020, + "Germany", + "Kenya", + null, + 30.0, + "US dollars", + null, + "OECD DAC2a", + "country/KEN", + null, + null, + null, + null, + null, + "provenance/DAC2a", + "official", + "DAC2aAidType206", + "ODA: Total Net", + "country/DEU", + "USDollar", + "dc/sv/oda_dac2a" + ], + [ + "total ODA", + 2021, + "Germany", + "Kenya", + null, + 40.0, + "US dollars", + null, + "OECD DAC2a", + "country/KEN", + null, + null, + null, + null, + null, + "provenance/DAC2a", + "official", + "DAC2aAidType206", + "ODA: Total Net", + "country/DEU", + "USDollar", + "dc/sv/oda_dac2a" + ] + ] + }, + { + "name": "stack-constant-per-member-varies-in-stack", + "members": [ + { + "part": "GDP in dollars", + "columns": [ + "stat_var", + "stat_var_label", + "observationAbout", + "observationAbout_label", + "time_period", + "value", + "unit", + "unit_label", + "currency", + "base_year", + "provenance", + "provenance_label" + ], + "rows": [ + [ + "dc/sv/gdp", + "GDP", + "country/KEN", + "Kenya", + 2022, + 113.4, + "USDollar", + "US dollars", + "USD", + null, + "provenance/WDI", + "World Bank WDI" + ] + ] + }, + { + "part": "GDP in shillings", + "columns": [ + "stat_var", + "stat_var_label", + "observationAbout", + "observationAbout_label", + "time_period", + "value", + "unit", + "unit_label", + "currency", + "base_year", + "provenance", + "provenance_label" + ], + "rows": [ + [ + "dc/sv/gdp", + "GDP", + "country/KEN", + "Kenya", + 2022, + 15.1, + "USDollar", + "US dollars", + "KES", + null, + "provenance/WDI", + "World Bank WDI" + ] + ] + } + ], + "columns": [ + "part", + "time_period", + "observationAbout_label", + "value", + "unit_label", + "provenance_label", + "currency", + "stat_var_label", + "observationAbout", + "unit", + "base_year", + "provenance", + "stat_var" + ], + "rows": [ + [ + "GDP in dollars", + 2022, + "Kenya", + 113.4, + "US dollars", + "World Bank WDI", + "USD", + "GDP", + "country/KEN", + "USDollar", + null, + "provenance/WDI", + "dc/sv/gdp" + ], + [ + "GDP in shillings", + 2022, + "Kenya", + 15.1, + "US dollars", + "World Bank WDI", + "KES", + "GDP", + "country/KEN", + "USDollar", + null, + "provenance/WDI", + "dc/sv/gdp" + ] + ] + } + ] +} diff --git a/python/client-contract/src/dfo_client_contract/rules/role-keys.json b/python/client-contract/src/dfo_client_contract/rules/role-keys.json new file mode 100644 index 0000000..1ac9f25 --- /dev/null +++ b/python/client-contract/src/dfo_client_contract/rules/role-keys.json @@ -0,0 +1,355 @@ +{ + "format_version": 1, + "rule": "role-keys", + "keys": { + "place": "observationAbout", + "origin": "origin", + "destination": "destination" + }, + "flow_keys": [ + "origin", + "destination" + ], + "vectors": [ + { + "name": "subject-place", + "identity": { + "entities": { + "observationAbout": [ + "country/KEN" + ] + } + }, + "roles": { + "place": [ + "country/KEN" + ] + }, + "entities": { + "place": [ + "country/NGA", + "country/GHA" + ] + }, + "posted": { + "observationAbout": [ + "country/NGA", + "country/GHA" + ] + } + }, + { + "name": "subject-wire-role-refused", + "identity": { + "entities": { + "observationAbout": [ + "country/KEN" + ] + } + }, + "roles": { + "place": [ + "country/KEN" + ] + }, + "entities": { + "observationAbout": [ + "country/NGA" + ] + }, + "error": { + "reason": "invalid_argument", + "names": [ + "observationAbout", + "place" + ] + } + }, + { + "name": "subject-flow-key-refused", + "identity": { + "entities": { + "observationAbout": [ + "country/KEN" + ] + } + }, + "roles": { + "place": [ + "country/KEN" + ] + }, + "entities": { + "origin": [ + "country/DEU" + ] + }, + "error": { + "reason": "invalid_argument", + "names": [ + "origin", + "place" + ] + } + }, + { + "name": "flow-origin-rebound", + "identity": { + "entities": { + "origin": [ + "country/DEU" + ], + "destination": [ + "country/NGA" + ] + } + }, + "roles": { + "origin": [ + "country/DEU" + ], + "destination": [ + "country/NGA" + ] + }, + "entities": { + "origin": [ + "country/FRA" + ] + }, + "posted": { + "origin": [ + "country/FRA" + ], + "destination": [ + "country/NGA" + ] + } + }, + { + "name": "flow-open-origin-bound", + "identity": { + "entities": { + "destination": [ + "country/KEN" + ] + }, + "open_roles": [ + "origin" + ] + }, + "roles": { + "destination": [ + "country/KEN" + ], + "origin": null + }, + "entities": { + "origin": [ + "country/GBR" + ], + "destination": [ + "country/ETH" + ] + }, + "posted": { + "destination": [ + "country/ETH" + ], + "origin": [ + "country/GBR" + ] + } + }, + { + "name": "flow-total-direction-bound", + "identity": { + "entities": { + "destination": [ + "country/ETH" + ] + } + }, + "roles": { + "destination": [ + "country/ETH" + ], + "origin": null + }, + "entities": { + "origin": [ + "country/USA" + ] + }, + "posted": { + "destination": [ + "country/ETH" + ], + "origin": [ + "country/USA" + ] + } + }, + { + "name": "flow-direction-dropped", + "identity": { + "entities": { + "origin": [ + "country/DEU" + ], + "destination": [ + "country/NGA" + ] + } + }, + "roles": { + "origin": [ + "country/DEU" + ], + "destination": [ + "country/NGA" + ] + }, + "entities": { + "destination": null + }, + "posted": { + "origin": [ + "country/DEU" + ] + } + }, + { + "name": "flow-column-name-refused", + "identity": { + "entities": { + "origin": [ + "country/DEU" + ], + "destination": [ + "country/NGA" + ] + } + }, + "roles": { + "origin": [ + "country/DEU" + ], + "destination": [ + "country/NGA" + ] + }, + "entities": { + "financingRecipient": [ + "country/GHA" + ] + }, + "error": { + "reason": "invalid_argument", + "names": [ + "financingRecipient", + "origin", + "destination" + ] + } + }, + { + "name": "flow-place-refused", + "identity": { + "entities": { + "origin": [ + "country/DEU" + ], + "destination": [ + "country/NGA" + ] + } + }, + "roles": { + "origin": [ + "country/DEU" + ], + "destination": [ + "country/NGA" + ] + }, + "entities": { + "place": [ + "country/GHA" + ] + }, + "error": { + "reason": "invalid_argument", + "names": [ + "place", + "origin", + "destination" + ] + } + }, + { + "name": "flow-open-destination-roles", + "identity": { + "entities": {}, + "open_roles": [ + "destination" + ] + }, + "roles": { + "destination": null, + "origin": null + } + }, + { + "name": "no-roles-place", + "identity": { + "entities": {} + }, + "roles": {}, + "entities": { + "place": [ + "country/KEN" + ] + }, + "posted": { + "observationAbout": [ + "country/KEN" + ] + } + }, + { + "name": "no-roles-other-key-posted", + "identity": { + "entities": {} + }, + "roles": {}, + "entities": { + "financingRecipient": [ + "country/KEN" + ] + }, + "posted": { + "financingRecipient": [ + "country/KEN" + ] + } + }, + { + "name": "no-roles-wire-role-refused", + "identity": {}, + "roles": {}, + "entities": { + "observationAbout": [ + "country/KEN" + ] + }, + "error": { + "reason": "invalid_argument", + "names": [ + "observationAbout", + "place" + ] + } + } + ] +} diff --git a/python/client-contract/src/dfo_client_contract/rules/rules.schema.json b/python/client-contract/src/dfo_client_contract/rules/rules.schema.json index 78eb56e..978a167 100644 --- a/python/client-contract/src/dfo_client_contract/rules/rules.schema.json +++ b/python/client-contract/src/dfo_client_contract/rules/rules.schema.json @@ -27,7 +27,13 @@ "$ref": "#/$defs/cache_layout" }, { - "$ref": "#/$defs/column_order" + "$ref": "#/$defs/reading_order" + }, + { + "$ref": "#/$defs/client_identity" + }, + { + "$ref": "#/$defs/role_keys" } ], "$defs": { @@ -589,7 +595,203 @@ } } }, - "column_order": { + "client_identity": { + "$ref": "#/$defs/suite", + "additionalProperties": false, + "required": [ + "header", + "pattern" + ], + "properties": { + "format_version": true, + "rule": { + "const": "client-identity" + }, + "header": { + "const": "X-DFO-Client" + }, + "pattern": { + "description": "A regular expression every well-formed header value matches in full.", + "type": "string", + "minLength": 1 + }, + "vectors": { + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name" + ], + "oneOf": [ + { + "required": [ + "client", + "version", + "runtime", + "value" + ], + "not": { + "anyOf": [ + { + "required": [ + "via" + ] + }, + { + "required": [ + "headers" + ] + }, + { + "required": [ + "verdict" + ] + } + ] + } + }, + { + "required": [ + "via", + "headers", + "verdict" + ], + "not": { + "anyOf": [ + { + "required": [ + "client" + ] + }, + { + "required": [ + "version" + ] + }, + { + "required": [ + "runtime" + ] + }, + { + "required": [ + "value" + ] + } + ] + } + } + ], + "if": { + "properties": { + "verdict": { + "const": "refuse" + } + }, + "required": [ + "verdict" + ] + }, + "then": { + "required": [ + "reason" + ] + }, + "else": { + "not": { + "required": [ + "reason" + ] + } + }, + "properties": { + "name": { + "$ref": "#/$defs/name" + }, + "client": { + "enum": [ + "dfo-python", + "dfo-ts" + ] + }, + "version": { + "description": "The package version as the package states it.", + "type": "string", + "pattern": "^[0-9A-Za-z.+-]+$", + "$ref": "#/$defs/one_line" + }, + "runtime": { + "description": "The runtime's name and its full version, or `browser` with no version.", + "type": "object", + "additionalProperties": false, + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "pattern": "^[A-Za-z]+$", + "$ref": "#/$defs/one_line" + }, + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+" + } + }, + "if": { + "properties": { + "name": { + "const": "browser" + } + } + }, + "then": { + "not": { + "required": [ + "version" + ] + } + }, + "else": { + "required": [ + "version" + ] + } + }, + "value": { + "description": "The X-DFO-Client value the client sends.", + "type": "string", + "minLength": 1 + }, + "via": { + "description": "Where the caller's headers come from: the client's headers argument, or the header provider it calls before each request.", + "enum": [ + "argument", + "provider" + ] + }, + "headers": { + "description": "The caller's headers, by name as the caller writes it.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "verdict": { + "enum": [ + "accept", + "refuse" + ] + }, + "reason": { + "const": "header_invalid" + } + } + } + } + } + }, + "reading_order": { "$ref": "#/$defs/suite", "additionalProperties": false, "required": [ @@ -598,7 +800,7 @@ "properties": { "format_version": true, "rule": { - "const": "column-order" + "const": "reading-order" }, "reading_columns": { "description": "The reading slots in order. Each slot lists the column names that fill it, each long layout's label before its frame layout's code; the first name a dataset holds fills the slot.", @@ -621,12 +823,56 @@ "additionalProperties": false, "required": [ "name", - "ordered" + "members", + "columns", + "rows" ], "properties": { "name": { "$ref": "#/$defs/name" }, + "members": { + "description": "The members a dataset stacks, in member order, each its served table: its `part` value when the dataset has several members, its columns and its rows in the order the server wrote them.", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "columns", + "rows" + ], + "properties": { + "part": { + "type": "string", + "minLength": 1 + }, + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "$ref": "#/$defs/one_line" + }, + "uniqueItems": true + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": [ + "string", + "number", + "boolean", + "null" + ] + } + } + } + } + } + }, "columns": { "type": "array", "items": { @@ -635,45 +881,243 @@ "$ref": "#/$defs/one_line" }, "uniqueItems": true, - "description": "One dataset's columns in the order the server wrote them." + "description": "The columns a client presents, in its order." }, - "members": { - "description": "The columns of each member of a stacked dataset, in member order, each in the order the server wrote them.", + "rows": { "type": "array", - "minItems": 2, "items": { "type": "array", "items": { - "type": "string", - "minLength": 1, - "$ref": "#/$defs/one_line" - }, - "uniqueItems": true - } - }, - "ordered": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "$ref": "#/$defs/one_line" + "type": [ + "string", + "number", + "boolean", + "null" + ] + } }, - "uniqueItems": true, - "description": "The same columns in the order a client presents them, after `part` for a stacked dataset." + "description": "The rows a client presents, in its order, each in the presented column order." } + } + } + } + } + }, + "role_keys": { + "$ref": "#/$defs/suite", + "additionalProperties": false, + "required": [ + "keys", + "flow_keys" + ], + "properties": { + "format_version": true, + "rule": { + "const": "role-keys" + }, + "keys": { + "description": "Each key a caller types in entities=, to the wire role it posts as.", + "type": "object", + "additionalProperties": false, + "required": [ + "place", + "origin", + "destination" + ], + "properties": { + "place": { + "const": "observationAbout" + }, + "origin": { + "const": "origin" }, + "destination": { + "const": "destination" + } + } + }, + "flow_keys": { + "description": "The flow directions. A spec that names one takes both.", + "const": [ + "origin", + "destination" + ] + }, + "vectors": { + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "identity", + "roles" + ], "oneOf": [ + { + "not": { + "anyOf": [ + { + "required": [ + "entities" + ] + }, + { + "required": [ + "posted" + ] + }, + { + "required": [ + "error" + ] + } + ] + } + }, { "required": [ - "columns" - ] + "entities", + "posted" + ], + "not": { + "required": [ + "error" + ] + } }, { "required": [ - "members" - ] + "entities", + "error" + ], + "not": { + "required": [ + "posted" + ] + } + } + ], + "properties": { + "name": { + "$ref": "#/$defs/name" + }, + "identity": { + "description": "The spec's definition.identity members that name roles, by wire role.", + "type": "object", + "additionalProperties": false, + "properties": { + "entities": { + "type": "object", + "propertyNames": { + "enum": [ + "observationAbout", + "origin", + "destination" + ] + }, + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "open_roles": { + "type": "array", + "items": { + "enum": [ + "observationAbout", + "origin", + "destination" + ] + }, + "uniqueItems": true + } + } + }, + "roles": { + "description": "The caller keys the spec takes, in order, each to the dcids it binds or null: Spec.roles in Python, specRoles(spec) in TypeScript.", + "type": "object", + "propertyNames": { + "enum": [ + "place", + "origin", + "destination" + ] + }, + "additionalProperties": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + { + "type": "null" + } + ] + } + }, + "entities": { + "description": "The caller's entities=, each key to dcids or to null, which drops the role.", + "type": "object", + "minProperties": 1, + "additionalProperties": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + { + "type": "null" + } + ] + } + }, + "posted": { + "description": "The build's scope.entities, by wire role.", + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "error": { + "type": "object", + "additionalProperties": false, + "required": [ + "reason", + "names" + ], + "properties": { + "reason": { + "const": "invalid_argument" + }, + "names": { + "description": "The keys the message names: the key typed, then the spec's caller keys, or place for observationAbout on a spec that names no role.", + "type": "array", + "minItems": 2, + "items": { + "type": "string", + "minLength": 1 + } + } + } } - ] + } } } } diff --git a/python/client-contract/src/dfo_client_contract/scenarios/answer-definite-fetch.json b/python/client-contract/src/dfo_client_contract/scenarios/answer-definite-fetch.json index e471bed..7d66613 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/answer-definite-fetch.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/answer-definite-fetch.json @@ -4580,12 +4580,12 @@ "unit_label", "crsTypeOfFlow_label", "provenance_label", + "crsCooperationModality", + "crsCooperationModality_label", "financingProvider", "financingProvider_kind", "financingRecipient", "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", "crsPurpose", "crsTypeOfFinance", "crsTypeOfFinance_label", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/answer-folded-parts-unnamed.json b/python/client-contract/src/dfo_client_contract/scenarios/answer-folded-parts-unnamed.json index 371544b..9a7302c 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/answer-folded-parts-unnamed.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/answer-folded-parts-unnamed.json @@ -15516,14 +15516,14 @@ "crsTypeOfFinance_label", "crsTypeOfFlow", "unit", + "provenance", + "stat_var_label", + "observationAbout", "currency", "price_basis", "base_year", "measurementMethod", "measurementMethod_label", - "provenance", - "stat_var_label", - "observationAbout", "stat_var" ], "part_rows": [ diff --git a/python/client-contract/src/dfo_client_contract/scenarios/answer-operations-not-a-list.json b/python/client-contract/src/dfo_client_contract/scenarios/answer-operations-not-a-list.json index 788a784..8ca97a3 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/answer-operations-not-a-list.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/answer-operations-not-a-list.json @@ -4583,12 +4583,12 @@ "unit_label", "crsTypeOfFlow_label", "provenance_label", + "crsCooperationModality", + "crsCooperationModality_label", "financingProvider", "financingProvider_kind", "financingRecipient", "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", "crsPurpose", "crsTypeOfFinance", "crsTypeOfFinance_label", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/answer-several-readings-folded.json b/python/client-contract/src/dfo_client_contract/scenarios/answer-several-readings-folded.json index e2c9390..66e1e7f 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/answer-several-readings-folded.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/answer-several-readings-folded.json @@ -15518,14 +15518,14 @@ "crsTypeOfFinance_label", "crsTypeOfFlow", "unit", + "provenance", + "stat_var_label", + "observationAbout", "currency", "price_basis", "base_year", "measurementMethod", "measurementMethod_label", - "provenance", - "stat_var_label", - "observationAbout", "stat_var" ], "part_rows": [ diff --git a/python/client-contract/src/dfo_client_contract/scenarios/choose-candidate-two-parts.json b/python/client-contract/src/dfo_client_contract/scenarios/choose-candidate-two-parts.json index 4f9ca97..2b3c0e5 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/choose-candidate-two-parts.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/choose-candidate-two-parts.json @@ -16378,9 +16378,9 @@ "unit", "currency", "price_basis", - "base_year", "measurementMethod", "measurementMethod_label", + "base_year", "provenance", "stat_var" ], diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-null-member.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-null-member.json index ef3ebcf..d8795d5 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-null-member.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-null-member.json @@ -4580,12 +4580,12 @@ "unit_label", "crsTypeOfFlow_label", "provenance_label", + "crsCooperationModality", + "crsCooperationModality_label", "financingProvider", "financingProvider_kind", "financingRecipient", "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", "crsPurpose", "crsTypeOfFinance", "crsTypeOfFinance_label", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-artifact-unlisted.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-artifact-unlisted.json index 8b08a06..2906e4f 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-artifact-unlisted.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-artifact-unlisted.json @@ -989,12 +989,12 @@ "unit_label", "crsTypeOfFlow_label", "provenance_label", + "crsCooperationModality", + "crsCooperationModality_label", "financingProvider", "financingProvider_kind", "financingRecipient", "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", "crsPurpose", "crsTypeOfFinance", "crsTypeOfFinance_label", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-expired-file-selector.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-expired-file-selector.json index 27976b3..85ed981 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-expired-file-selector.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-expired-file-selector.json @@ -1040,12 +1040,12 @@ "unit_label", "crsTypeOfFlow_label", "provenance_label", + "crsCooperationModality", + "crsCooperationModality_label", "financingProvider", "financingProvider_kind", "financingRecipient", "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", "crsPurpose", "crsTypeOfFinance", "crsTypeOfFinance_label", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-expired-rebuild-same-version.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-expired-rebuild-same-version.json index 40f7b18..680ad6d 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-expired-rebuild-same-version.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-expired-rebuild-same-version.json @@ -991,12 +991,12 @@ "unit_label", "crsTypeOfFlow_label", "provenance_label", + "crsCooperationModality", + "crsCooperationModality_label", "financingProvider", "financingProvider_kind", "financingRecipient", "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", "crsPurpose", "crsTypeOfFinance", "crsTypeOfFinance_label", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-signed-url-refused.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-signed-url-refused.json index e0de697..a379515 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-signed-url-refused.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref-signed-url-refused.json @@ -1044,12 +1044,12 @@ "unit_label", "crsTypeOfFlow_label", "provenance_label", + "crsCooperationModality", + "crsCooperationModality_label", "financingProvider", "financingProvider_kind", "financingRecipient", "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", "crsPurpose", "crsTypeOfFinance", "crsTypeOfFinance_label", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref.json index e716ee8..5c43080 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-ref.json @@ -1019,12 +1019,12 @@ "unit_label", "crsTypeOfFlow_label", "provenance_label", + "crsCooperationModality", + "crsCooperationModality_label", "financingProvider", "financingProvider_kind", "financingRecipient", "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", "crsPurpose", "crsTypeOfFinance", "crsTypeOfFinance_label", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-retry-after-504.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-retry-after-504.json index bfe6e68..cd3d71e 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-retry-after-504.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-retry-after-504.json @@ -4680,12 +4680,12 @@ "unit_label", "crsTypeOfFlow_label", "provenance_label", + "crsCooperationModality", + "crsCooperationModality_label", "financingProvider", "financingProvider_kind", "financingRecipient", "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", "crsPurpose", "crsTypeOfFinance", "crsTypeOfFinance_label", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-accepted-running.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-accepted-running.json index 3956bb1..ab81f2c 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-accepted-running.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-accepted-running.json @@ -1512,8 +1512,8 @@ "dac2aAidType_label", "provenance_label", "financingProvider", - "financingProvider_kind", "financingRecipient", + "financingProvider_kind", "financingRecipient_kind", "dac2aAidType", "unit", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-async-build.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-async-build.json index e8a44cc..bcc6706 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-async-build.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-async-build.json @@ -1512,8 +1512,8 @@ "dac2aAidType_label", "provenance_label", "financingProvider", - "financingProvider_kind", "financingRecipient", + "financingProvider_kind", "financingRecipient_kind", "dac2aAidType", "unit", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-entity-dropped.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-entity-dropped.json index b247574..166b104 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-entity-dropped.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-entity-dropped.json @@ -4894,14 +4894,14 @@ "crsTypeOfFlow_label", "provenance_label", "financingProvider", - "financingProvider_kind", - "financingRecipient", - "financingRecipient_kind", "crsCooperationModality", "crsCooperationModality_label", - "crsPurpose", "crsTypeOfFinance", "crsTypeOfFinance_label", + "financingProvider_kind", + "financingRecipient", + "financingRecipient_kind", + "crsPurpose", "crsTypeOfFlow", "unit", "currency", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-entity-rebound.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-entity-rebound.json index d1761e5..d246838 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-entity-rebound.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-entity-rebound.json @@ -4581,12 +4581,12 @@ "unit_label", "crsTypeOfFlow_label", "provenance_label", + "crsCooperationModality", + "crsCooperationModality_label", "financingProvider", "financingProvider_kind", "financingRecipient", "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", "crsPurpose", "crsTypeOfFinance", "crsTypeOfFinance_label", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-frame-layout.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-frame-layout.json index 4ca9780..bd68531 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-frame-layout.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-frame-layout.json @@ -952,9 +952,9 @@ "unit", "stat_var_crsTypeOfFlow", "provenance", + "stat_var_crsCooperationModality", "measurementMethod", "status", - "stat_var_crsCooperationModality", "stat_var_crsTypeOfFinance", "stat_var" ], diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-group-narrowed.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-group-narrowed.json index 79b670b..136dde5 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-group-narrowed.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-group-narrowed.json @@ -5334,9 +5334,9 @@ "unit_label", "dac2aAidType_label", "provenance_label", + "financingRecipient", "financingProvider", "financingProvider_kind", - "financingRecipient", "financingRecipient_kind", "dac2aAidType", "unit", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-group-origin-rebound.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-group-origin-rebound.json index 603a911..f5318d4 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-group-origin-rebound.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-group-origin-rebound.json @@ -5333,9 +5333,9 @@ "unit_label", "dac2aAidType_label", "provenance_label", + "financingRecipient", "financingProvider", "financingProvider_kind", - "financingRecipient", "financingRecipient_kind", "dac2aAidType", "unit", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-operation-failed-resubmitted.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-operation-failed-resubmitted.json index 7468358..919f817 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-operation-failed-resubmitted.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-operation-failed-resubmitted.json @@ -1797,8 +1797,8 @@ "dac2aAidType_label", "provenance_label", "financingProvider", - "financingProvider_kind", "financingRecipient", + "financingProvider_kind", "financingRecipient_kind", "dac2aAidType", "unit", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-pacing-changes.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-pacing-changes.json index c3f0abc..636e329 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-pacing-changes.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-pacing-changes.json @@ -1434,8 +1434,8 @@ "dac2aAidType_label", "provenance_label", "financingProvider", - "financingProvider_kind", "financingRecipient", + "financingProvider_kind", "financingRecipient_kind", "dac2aAidType", "unit", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-rate-limited.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-rate-limited.json index 2bf0145..ea02336 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-rate-limited.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-rate-limited.json @@ -1524,8 +1524,8 @@ "dac2aAidType_label", "provenance_label", "financingProvider", - "financingProvider_kind", "financingRecipient", + "financingProvider_kind", "financingRecipient_kind", "dac2aAidType", "unit", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-refused-backoff.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-refused-backoff.json index 2535033..a6ede46 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-refused-backoff.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-refused-backoff.json @@ -1520,8 +1520,8 @@ "dac2aAidType_label", "provenance_label", "financingProvider", - "financingProvider_kind", "financingRecipient", + "financingProvider_kind", "financingRecipient_kind", "dac2aAidType", "unit", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-refused-retry-after.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-refused-retry-after.json index 42b9254..322e6f8 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-refused-retry-after.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-refused-retry-after.json @@ -1535,8 +1535,8 @@ "dac2aAidType_label", "provenance_label", "financingProvider", - "financingProvider_kind", "financingRecipient", + "financingProvider_kind", "financingRecipient_kind", "dac2aAidType", "unit", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-refused.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-refused.json index 5222a2e..a629db0 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-refused.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-spec-poll-refused.json @@ -1578,8 +1578,8 @@ "dac2aAidType_label", "provenance_label", "financingProvider", - "financingProvider_kind", "financingRecipient", + "financingProvider_kind", "financingRecipient_kind", "dac2aAidType", "unit", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/fetch-strict-info-notice.json b/python/client-contract/src/dfo_client_contract/scenarios/fetch-strict-info-notice.json index fee7446..5652a17 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/fetch-strict-info-notice.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/fetch-strict-info-notice.json @@ -4586,12 +4586,12 @@ "unit_label", "crsTypeOfFlow_label", "provenance_label", + "crsCooperationModality", + "crsCooperationModality_label", "financingProvider", "financingProvider_kind", "financingRecipient", "financingRecipient_kind", - "crsCooperationModality", - "crsCooperationModality_label", "crsPurpose", "crsTypeOfFinance", "crsTypeOfFinance_label", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/mint-async-build.json b/python/client-contract/src/dfo_client_contract/scenarios/mint-async-build.json index a65adda..9c27c02 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/mint-async-build.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/mint-async-build.json @@ -1521,8 +1521,8 @@ "dac2aAidType_label", "provenance_label", "financingProvider", - "financingProvider_kind", "financingRecipient", + "financingProvider_kind", "financingRecipient_kind", "dac2aAidType", "unit", diff --git a/python/client-contract/src/dfo_client_contract/scenarios/operation-resume.json b/python/client-contract/src/dfo_client_contract/scenarios/operation-resume.json index 6141b6f..3c31a70 100644 --- a/python/client-contract/src/dfo_client_contract/scenarios/operation-resume.json +++ b/python/client-contract/src/dfo_client_contract/scenarios/operation-resume.json @@ -8307,8 +8307,8 @@ "dac2aAidType_label", "provenance_label", "financingProvider", - "financingProvider_kind", "financingRecipient", + "financingProvider_kind", "financingRecipient_kind", "dac2aAidType", "unit", diff --git a/python/client-contract/tests/test_rules.py b/python/client-contract/tests/test_rules.py index d51cbf0..fe7224f 100644 --- a/python/client-contract/tests/test_rules.py +++ b/python/client-contract/tests/test_rules.py @@ -7,6 +7,7 @@ from __future__ import annotations +import ipaddress import ntpath import posixpath import re @@ -24,13 +25,17 @@ _DEFAULT_BASE_URL = "https://developmentfinance.org/api/v1" _TABLE = re.compile(r"\[([A-Za-z0-9_-]+)\]") _ENTRY = re.compile(r'([A-Za-z0-9_-]+)[ \t]*=[ \t]*"([^"\\\x00-\x1f\x7f]*)"') +_MAJOR_MINOR = re.compile(r"([0-9]+)\.([0-9]+)") class RuleError(Exception): - def __init__(self, reason: str, line: int | None = None) -> None: + def __init__( + self, reason: str, line: int | None = None, names: tuple[str, ...] = () + ) -> None: super().__init__(reason) self.reason = reason self.line = line + self.names = names def _vectors(name: str) -> list[dict[str, Any]]: @@ -125,13 +130,24 @@ def look_up_key(vector: dict[str, Any]) -> tuple[str, str | None]: return "credentials_file", profile["key"] +_NON_DECIMAL_IPV4 = re.compile(r"(?:[^:]*\.)?(?:0x[0-9a-f]*|[0-9]+)") + + def is_valid_base_url(url: str) -> bool: + if re.search(r"[^\x21-\x7e]", url) or "?" in url or "#" in url: + return False parts = urlsplit(url) - return ( - parts.scheme in ("http", "https") - and bool(parts.hostname) - and "@" not in parts.netloc - ) + if parts.scheme not in ("http", "https") or "@" in parts.netloc: + return False + if not parts.hostname or {".", ".."} & set(parts.path.split("/")): + return False + host = parts.hostname.removesuffix(".") + if _NON_DECIMAL_IPV4.fullmatch(host): + try: + ipaddress.IPv4Address(host) + except ValueError: + return False + return True def resolve_base_url(vector: dict[str, Any]) -> tuple[str, str]: @@ -143,7 +159,7 @@ def resolve_base_url(vector: dict[str, Any]) -> tuple[str, str]: source, url = "default", _DEFAULT_BASE_URL if not is_valid_base_url(url): raise RuleError("base_url_invalid") - return source, url + return source, url.rstrip("/") def cache_root(vector: dict[str, Any]) -> str: @@ -179,14 +195,89 @@ def cache_entry(vector: dict[str, Any]) -> str: return f"manifests/{vector['spec_id']}/{vector['content_version']}/{key}.json" -def column_order(columns: list[str], reading: list[list[str]]) -> list[str]: +def reading_order( + members: list[dict[str, Any]], reading: list[list[str]] +) -> tuple[list[str], list[list[Any]]]: + """Stacks the members, a row holding null in a column its member lacks, + then orders the columns and each member's rows.""" + stacked = [ + {**dict(zip(member["columns"], row, strict=True)), "part": member.get("part")} + for member in members + for row in member["rows"] + ] + served = list(dict.fromkeys(c for member in members for c in member["columns"])) first = [ name for names in reading - if (name := next((n for n in names if n in columns), None)) is not None + if (name := next((n for n in names if n in served), None)) is not None + ] + rest = [c for c in served if c not in first and c != "stat_var"] + varies = {c for c in rest if len({repr(row.get(c)) for row in stacked}) > 1} + columns = [ + *(["part"] if len(members) > 1 else []), + *first, + *[c for c in rest if c in varies], + *[c for c in rest if c not in varies], + *(["stat_var"] if "stat_var" in served else []), ] - rest = [c for c in columns if c not in first and c != "stat_var"] - return first + rest + ["stat_var"] * ("stat_var" in columns) + keys = [c for c in first if c not in ("value", "time_period")] + keys += [c for c in columns if c in varies] + if "stat_var" in served and len({repr(row.get("stat_var")) for row in stacked}) > 1: + keys.append("stat_var") + keys += ["time_period"] if "time_period" in first else [] + rows: list[list[Any]] = [] + for part in dict.fromkeys(row["part"] for row in stacked): + block = [row for row in stacked if row["part"] == part] + block.sort(key=lambda row: [(row.get(k) is None, row.get(k)) for k in keys]) + rows.extend([row.get(c) for c in columns] for row in block) + return columns, rows + + +def client_identity(client: str, version: str, runtime: dict[str, str]) -> str: + label = runtime["name"] + if label != "browser": + match = _MAJOR_MINOR.match(runtime["version"]) + assert match, runtime + label += f" {match[1]}.{match[2]}" + return f"{client}/{version} ({label})" + + +def check_caller_headers(headers: dict[str, str]) -> str | None: + if any(name.lower() == "x-dfo-client" for name in headers): + return "header_invalid" + return None + + +def spec_roles(identity: dict[str, Any]) -> dict[str, list[str] | None]: + caller = {wire: key for key, wire in rule("role-keys")["keys"].items()} + flow = rule("role-keys")["flow_keys"] + bound = { + caller[role]: dcids for role, dcids in identity.get("entities", {}).items() + } + named = [*bound, *(caller[role] for role in identity.get("open_roles", []))] + if any(key in flow for key in named): + named.extend(flow) + return {key: bound.get(key) for key in dict.fromkeys(named)} + + +def posted_entities( + identity: dict[str, Any], entities: dict[str, list[str] | None] +) -> dict[str, list[str]]: + keys = rule("role-keys")["keys"] + roles = spec_roles(identity) + for key in entities: + if roles and key not in roles: + raise RuleError("invalid_argument", names=(key, *roles)) + if not roles and key in keys.values() and key not in keys: + caller = next(name for name, wire in keys.items() if wire == key) + raise RuleError("invalid_argument", names=(key, caller)) + posted = dict(identity.get("entities", {})) + for key, dcids in entities.items(): + if dcids is None: + posted.pop(keys.get(key, key), None) + else: + posted[keys.get(key, key)] = dcids + return posted @pytest.mark.parametrize("name", RULES) @@ -258,11 +349,37 @@ def test_cache_layout(vector: dict[str, Any]) -> None: assert cache_entry(vector) == vector["path"] -@pytest.mark.parametrize("vector", _vectors("column-order"), ids=_ids("column-order")) -def test_column_order(vector: dict[str, Any]) -> None: - reading = rule("column-order")["reading_columns"] - if "columns" in vector: - assert column_order(vector["columns"], reading) == vector["ordered"] +@pytest.mark.parametrize("vector", _vectors("reading-order"), ids=_ids("reading-order")) +def test_reading_order(vector: dict[str, Any]) -> None: + reading = rule("reading-order")["reading_columns"] + columns, rows = reading_order(vector["members"], reading) + assert (columns, rows) == (vector["columns"], vector["rows"]) + + +@pytest.mark.parametrize( + "vector", _vectors("client-identity"), ids=_ids("client-identity") +) +def test_client_identity(vector: dict[str, Any]) -> None: + if "headers" in vector: + assert check_caller_headers(vector["headers"]) == vector.get("reason") return - gathered = list(dict.fromkeys(c for member in vector["members"] for c in member)) - assert ["part", *column_order(gathered, reading)] == vector["ordered"] + value = client_identity(vector["client"], vector["version"], vector["runtime"]) + assert value == vector["value"] + assert re.fullmatch(rule("client-identity")["pattern"], value) + + +@pytest.mark.parametrize("vector", _vectors("role-keys"), ids=_ids("role-keys")) +def test_role_keys(vector: dict[str, Any]) -> None: + roles = spec_roles(vector["identity"]) + assert list(roles.items()) == list(vector["roles"].items()) + if "error" in vector: + with pytest.raises(RuleError) as excinfo: + posted_entities(vector["identity"], vector["entities"]) + error = excinfo.value + assert (error.reason, list(error.names)) == ( + vector["error"]["reason"], + vector["error"]["names"], + ) + elif "entities" in vector: + posted = posted_entities(vector["identity"], vector["entities"]) + assert posted == vector["posted"] diff --git a/tools/README.md b/tools/README.md index dfc1122..d52b523 100644 --- a/tools/README.md +++ b/tools/README.md @@ -38,7 +38,7 @@ uv run tools/validate_provenance_docs_vectors.py uv run tools/validate_descriptions_vectors.py ``` -`validate_client_contract.py` validates the client contract under `python/client-contract/src/dfo_client_contract/` against its schemas, then checks what they cannot express: every `strict-fields.json` pointer resolves in `served/openapi.json` and stays required and non-nullable, or keeps its enum values; `actions.json` holds one row, in code order, for each active refusal code in `registries/codes.json` that reaches a problem document or a no_data reason, with a gap kind exactly for the no_data reasons; `client-codes.json` lists each code a client raises itself once, in code order, none of them a code in `actions.json` or the registry, each with a class a scenario names and a retryable flag; `proxy-codes.json` and `warning-codes.json` list their codes once, in code order, apart from every other table, and the warning codes are those the client rules issue; `served/openapi.json` holds the Search, Datasets, Graph and Presentation surfaces and nothing outside them but the platform's `/codes` route; each recorded scenario's requests match a served route and its JSON bodies validate against that route's schemas, where a required member stays required when it admits null; every problem document in a scenario, synthetic ones and failed operations' errors included, carries an active code registered at its status or a code `proxy-codes.json` lists at its status, and one with a proxy code is `application/problem+json` with the common members, its own status and `retryable` true; no 202 in a scenario reads `done`; each recorded scenario names `served/SOURCE.json`'s commit; each scenario's `expect` equals the outcomes its steps reach over its exchanges, with refusals' error classes and gap kinds from `actions.json`, each error the client raises itself carrying its code and class from `client-codes.json`, a repeated refused request counted a retry only where the retry rules allow one, a record that stops retrying while a retry remains refused, and the waits before retries and the warnings response headers issue derived; each POST carries its body, each build posts the body its spec, narrowing, layout and, for a fetch, party totals give, each dataset's columns are the column-order rule's order of the columns the downloaded Parquet files' footers name, each Search request posts the whole body its steps derive, each poll names the operation the client holds, and each signed download is the URL its redirect names; each body file a record names is a bare file name in `scenarios/` and no symlink; every absolute URL a record holds is a signed URL on the storage host with the redacted query, or a `location` under the base URL; each `DatasetRef` example gets the verdict it states, and each column-order vector orders its columns as the rule does. It does not import `dfo_client_contract`, whose `tests/test_rules.py` replays the rule vectors. Also a PEP 723 script: +`validate_client_contract.py` validates the client contract under `python/client-contract/src/dfo_client_contract/` against its schemas, then checks what they cannot express: every `strict-fields.json` pointer resolves in `served/openapi.json` and stays required and non-nullable, or keeps its enum values; `actions.json` holds one row, in code order, for each active refusal code in `registries/codes.json` that reaches a problem document or a no_data reason, with a gap kind exactly for the no_data reasons; `client-codes.json` lists each code a client raises itself once, in code order, none of them a code in `actions.json` or the registry, each with a class a scenario names and a retryable flag; `proxy-codes.json` and `warning-codes.json` list their codes once, in code order, apart from every other table, and the warning codes are those the client rules issue; `served/openapi.json` holds the Search, Datasets, Graph and Presentation surfaces and nothing outside them but the platform's `/codes` route; each recorded scenario's requests match a served route and its JSON bodies validate against that route's schemas, where a required member stays required when it admits null; every problem document in a scenario, synthetic ones and failed operations' errors included, carries an active code registered at its status or a code `proxy-codes.json` lists at its status, and one with a proxy code is `application/problem+json` with the common members, its own status and `retryable` true; no 202 in a scenario reads `done`; each recorded scenario names `served/SOURCE.json`'s commit; each scenario's `expect` equals the outcomes its steps reach over its exchanges, with refusals' error classes and gap kinds from `actions.json`, each error the client raises itself carrying its code and class from `client-codes.json`, a repeated refused request counted a retry only where the retry rules allow one, a record that stops retrying while a retry remains refused, and the waits before retries and the warnings response headers issue derived; each POST carries its body, each build posts the body its spec, narrowing, layout and, for a fetch, party totals give, each dataset's columns are the reading-order rule's order of the downloaded Parquet files' columns, each Search request posts the whole body its steps derive, each poll names the operation the client holds, and each signed download is the URL its redirect names; each body file a record names is a bare file name in `scenarios/` and no symlink; every absolute URL a record holds is a signed URL on the storage host with the redacted query, or a `location` under the base URL; each `DatasetRef` example gets the verdict it states, and each reading-order vector orders its columns and rows as the rule does. It does not import `dfo_client_contract`, whose `tests/test_rules.py` replays the rule vectors. Also a PEP 723 script: ```bash uv run tools/validate_client_contract.py diff --git a/tools/capture_client_contract.py b/tools/capture_client_contract.py index e85a3cb..0b7d221 100644 --- a/tools/capture_client_contract.py +++ b/tools/capture_client_contract.py @@ -54,7 +54,6 @@ import argparse import copy -import io import json import os import re @@ -71,7 +70,6 @@ from typing import Any, Protocol from urllib.parse import quote, unquote, urljoin, urlsplit, urlunsplit -import pyarrow.parquet as pq from validate_client_contract import ( FILE_EXPIRED, FILE_GONE, @@ -89,6 +87,7 @@ Answer, ClientError, LongBuild, + ServedTable, absolute_url_problems, answer_from_body, answer_gaps, @@ -129,6 +128,7 @@ search_warns, selector_path, sentence_entry, + served_table, shared_cacheable, stored_version, surface_problems, @@ -149,7 +149,7 @@ # totals of open party roles left out, and so the rendering a ref it writes # records. RENDERING = fetch_rendering(None) -# The reading slots of the column-order rule, which orders each dataset's +# The reading slots of the reading-order rule, which orders each dataset's # columns as a client presents them, or None when the rule file cannot be read. READING_COLUMNS = load_reading_columns(PACKAGE_DIR) # The response headers a client reads, and so the ones a record keeps, beside @@ -568,6 +568,12 @@ class Member: rows: int columns: list[str] rendering: dict[str, Any] = field(default_factory=lambda: dict(RENDERING)) + # The value of each column that holds one value in every row. + same: Mapping[str, Any] = field(default_factory=dict) + + @property + def table(self) -> ServedTable: + return ServedTable(tuple(self.columns), self.rows, self.same) @property def spec_id(self) -> str: @@ -591,8 +597,8 @@ def dataset_outcome( ) -> dict[str, Any]: """A dataset outcome. Several members stack into one long table with a `part` column first, holding each member's `part_label`, then every - member's columns in member order. The columns after `part` are in the - order the column-order rule gives.""" + member's columns in member order. The columns are in the order the + reading-order rule gives.""" summaries = [ { "parts": member.parts, @@ -605,11 +611,11 @@ def dataset_outcome( dataset: dict[str, Any] = {"rows": sum(member.rows for member in members)} if READING_COLUMNS is None: raise SystemExit( - "rules/column-order.json is missing or holds no reading_columns, so " + "rules/reading-order.json is missing or holds no reading_columns, so " "no dataset's columns can be ordered" ) dataset["columns"] = presented_columns( - [member.columns for member in members], READING_COLUMNS + [member.table for member in members], READING_COLUMNS ) if len(members) > 1: dataset["part_rows"] = [ @@ -620,9 +626,8 @@ def dataset_outcome( return {"dataset": {**dataset, **summary}} -def parquet_shape(data: bytes) -> tuple[int, list[str]]: - parquet = pq.ParquetFile(io.BytesIO(data)) - return parquet.metadata.num_rows, list(parquet.schema_arrow.names) +def parquet_shape(data: bytes) -> ServedTable: + return served_table(data) def _reached[T](results: list[T], step: str) -> T: @@ -678,7 +683,7 @@ class Session: recorder: Recorder actions: Mapping[str, Mapping[str, Any]] client_codes: Mapping[str, Mapping[str, Any]] - read_shape: Callable[[bytes], tuple[int, list[str]]] = parquet_shape + read_shape: Callable[[bytes], ServedTable] = parquet_shape sleep: Callable[[float], None] = time.sleep call: list[dict[str, Any]] = field(default_factory=list) expect: list[dict[str, Any]] = field(default_factory=list) @@ -972,16 +977,17 @@ def _member( held = None if spec is None else spec.step() sentence = self._identity_sentence(parts, definition, held) digest = artifact["artifact_digest"] - rows, columns = self.read_shape(self._download(route, digest)) + table = self.read_shape(self._download(route, digest)) return Member( parts, definition, manifest["content_version"], sentence, digest, - rows, - columns, + table.rows, + list(table.columns), dict(rendering), + table.same, ) def _identity_sentence( @@ -1081,16 +1087,17 @@ def stored_file( data, refusal = self._stored_file(member, manifest) if data is None: return None, refusal - rows, columns = self.read_shape(data) + table = self.read_shape(data) stored = Member( member["parts"], spec["definition"], version, member["sentence"], member["artifact"]["digest"], - rows, - columns, + table.rows, + list(table.columns), dict(rendering), + table.same, ) return stored, None @@ -1261,7 +1268,7 @@ class Context: transport: Transport actions: Mapping[str, Mapping[str, Any]] client_codes: Mapping[str, Mapping[str, Any]] - read_shape: Callable[[bytes], tuple[int, list[str]]] = parquet_shape + read_shape: Callable[[bytes], ServedTable] = parquet_shape _cache: dict[str, Any] = field(default_factory=dict) def session(self, name: str, **recorder: Any) -> Session: diff --git a/tools/tests/test_capture_client_contract.py b/tools/tests/test_capture_client_contract.py index ab90b8e..61dbb91 100644 --- a/tools/tests/test_capture_client_contract.py +++ b/tools/tests/test_capture_client_contract.py @@ -46,8 +46,8 @@ def _records(kind: str) -> list[str]: ) -def _shape(data: bytes) -> tuple[int, list[str]]: - return len(data), ["column"] +def _shape(data: bytes) -> capture.ServedTable: + return capture.ServedTable(("column",), len(data), {}) def _without_shapes(expect: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -404,7 +404,7 @@ def test_a_part_without_a_reading_is_a_gap_of_its_kind( def test_several_members_stack_with_a_part_column_first() -> None: - """The stacked columns follow `part` in the column-order rule's order, + """The stacked columns follow `part` in the reading-order rule's order, the reading columns first and `stat_var` last.""" member = capture.Member( [], diff --git a/tools/tests/test_validate_client_contract.py b/tools/tests/test_validate_client_contract.py index 5dee37a..9dffff3 100644 --- a/tools/tests/test_validate_client_contract.py +++ b/tools/tests/test_validate_client_contract.py @@ -165,6 +165,22 @@ def change(outcome: Any) -> None: return change +def _vector(name: str, change: Callable[[Any], None]) -> Callable[[Any], None]: + return lambda doc: change(next(v for v in doc["vectors"] if v["name"] == name)) + + +def _served_order_after_reading_columns(holder: Any) -> None: + """Moves the varying co-operation modality pair back to its served place, + just before `crsPurpose`, as the rule placed it before it put varying + columns ahead of constant ones.""" + columns = holder["dataset"]["columns"] if "dataset" in holder else holder["columns"] + varying = ["crsCooperationModality", "crsCooperationModality_label"] + for name in varying: + columns.remove(name) + at = columns.index("crsPurpose") + columns[at:at] = varying + + def _set_step(index: int, **fields: Any) -> Callable[[Any], None]: return lambda doc: doc["call"][index].update(fields) @@ -1720,20 +1736,55 @@ def change(doc: Any) -> None: "scenarios/fetch-ref.json: step 0: columns", ), ( - "rules/column-order.json", - lambda doc: doc["vectors"][0]["ordered"].reverse(), - "orders its columns otherwise than presented_columns", + "scenarios/answer-definite-fetch.json", + _set_outcome(1, _served_order_after_reading_columns), + "scenarios/answer-definite-fetch.json: step 1: columns", + ), + ( + "rules/reading-order.json", + _vector( + "varying-column-follows-reading-columns", + _served_order_after_reading_columns, + ), + ( + "vector 'varying-column-follows-reading-columns' orders its " + "columns or rows otherwise" + ), + ), + ( + "rules/reading-order.json", + _vector("stack-crs-and-dac2a", lambda v: v["columns"].remove("part")), + "vector 'stack-crs-and-dac2a' orders its columns or rows otherwise", + ), + ( + "rules/reading-order.json", + _vector( + "ties-keep-served-order", + lambda v: v["rows"].insert(1, v["rows"].pop(2)), + ), + "vector 'ties-keep-served-order' orders its columns or rows otherwise", + ), + ( + "rules/reading-order.json", + _vector( + "varying-column-follows-reading-columns", + lambda v: v["rows"].insert(1, v["rows"].pop(2)), + ), + ( + "vector 'varying-column-follows-reading-columns' orders its " + "columns or rows otherwise" + ), ), ( - "rules/column-order.json", - lambda doc: doc["vectors"][-1]["ordered"].remove("part"), - "vector 'stack-crs-and-dac2a' orders its columns otherwise", + "rules/reading-order.json", + _vector("null-sorts-last", lambda v: v["rows"].insert(0, v["rows"].pop())), + "vector 'null-sorts-last' orders its columns or rows otherwise", ), ( - "rules/column-order.json", + "rules/reading-order.json", lambda doc: doc.pop("reading_columns"), ( - "rules/column-order.json: missing or unreadable, or holds no " + "rules/reading-order.json: missing or unreadable, or holds no " "reading_columns" ), ), @@ -1984,9 +2035,13 @@ def change(doc: Any) -> None: "stacked-columns-out-of-reading-order", "served-order-of-other-columns-swapped", "served-column-dropped", - "column-order-vector-misordered", - "column-order-stacked-vector-misordered", - "column-order-without-reading-columns", + "constant-columns-before-varying-ones", + "reading-order-vector-in-served-order", + "reading-order-stacked-vector-without-part", + "reading-order-ties-reordered", + "reading-order-series-interleaved-by-year", + "reading-order-null-sorted-first", + "reading-order-without-reading-columns", ], ) def test_a_broken_contract_is_reported( @@ -2000,10 +2055,10 @@ def test_a_broken_contract_is_reported( assert any(expected in problem for problem in problems), problems -def test_a_missing_column_order_rule_is_reported(package_dir: Path) -> None: - (package_dir / "rules" / "column-order.json").unlink() +def test_a_missing_reading_order_rule_is_reported(package_dir: Path) -> None: + (package_dir / "rules" / "reading-order.json").unlink() assert any( - problem.startswith("rules/column-order.json: missing") + problem.startswith("rules/reading-order.json: missing") for problem in validator.validate() ) diff --git a/tools/validate_client_contract.py b/tools/validate_client_contract.py index ff7db6e..c84959f 100644 --- a/tools/validate_client_contract.py +++ b/tools/validate_client_contract.py @@ -61,9 +61,10 @@ `client-codes.json`, as `expected_outcomes` derives them, the waits before each poll included, under the record's base URL and again under REPLAY_ORIGIN with the recorded origin a response names moved there. A - dataset's columns are the downloaded files' columns, read from their - Parquet footers by `parquet_columns` and ordered by `presented_columns`, - its rows and per-part row counts are left to the Parquet bodies, and each + dataset's columns are the downloaded files' columns, read with pyarrow + by `served_table` and ordered by `presented_columns`, which puts the + columns whose values vary before those that hold one value, its rows and + per-part row counts are left to the Parquet bodies, and each download hashes to its artifact's digest or the outcome says it does not. Each Search request a record holds posts the input the steps derive: the question, a choice's data_query or dataset_resubmit entries, or a member's identity with the @@ -93,8 +94,8 @@ scenario's build of its spec posted. - Each rule file under `rules/` validates against `rules.schema.json`, names the rule its file name names, and names each vector once, and each - column-order vector's `ordered` is what `presented_columns` gives. A - missing or unreadable `rules/column-order.json` is a problem. + reading-order vector's `columns` and `rows` are what `presented_rows` + gives. A missing or unreadable `rules/reading-order.json` is a problem. It does not import `dfo_client_contract`, whose tests replay the rule vectors. `tools/capture_client_contract.py` imports the outcome @@ -107,9 +108,9 @@ import json import re import sys -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Collection, Iterator, Mapping, Sequence from dataclasses import dataclass, field, replace -from functools import partial +from functools import lru_cache, partial from pathlib import Path from typing import Any from urllib.parse import quote, urljoin, urlsplit, urlunsplit @@ -1777,8 +1778,12 @@ class _UnfitError(Exception): # The rendering member a fetch posts as true unless the caller keeps party # totals, which leaves out the group totals of the identity's open party roles. WITHOUT_PARTY_TOTALS = "without_party_totals" -# The column a client presents last. +# The column a client presents last, the column a stacked dataset puts +# first, the measure no row sorts by, and the sort key that comes last. STAT_VAR = "stat_var" +PART = "part" +VALUE = "value" +TIME_PERIOD = "time_period" # The snapshot refusal after which a stored version's lookup builds it again. # The version exists, but its file has expired. Any other refusal ends it. FILE_EXPIRED = "artifact_expired" @@ -1811,35 +1816,168 @@ def fetch_rendering( return rendering -def column_order(columns: Sequence[str], reading: Sequence[Sequence[str]]) -> list[str]: - """`columns` in the order a client presents them: each reading slot's - first name the columns hold, in slot order, then the other columns in - their own order, with `stat_var` last.""" - first = [ +@dataclass(frozen=True, slots=True) +class ServedTable: + """A served table as the reading-order rule reads it: its columns in the + order the server wrote them, its row count, and the value of each column + that holds one value in every row. A column `same` leaves out varies.""" + + columns: tuple[str, ...] + rows: int + same: Mapping[str, Any] + + +def table_from_rows( + columns: Sequence[str], rows: Sequence[Sequence[Any]] +) -> ServedTable: + """The `ServedTable` of a table given as rows, as a vector gives one.""" + same = { + name: rows[0][index] + for index, name in enumerate(columns) + if rows and all(row[index] == rows[0][index] for row in rows) + } + return ServedTable(tuple(columns), len(rows), same) + + +@lru_cache(maxsize=64) +def served_table(data: bytes) -> ServedTable: + """The `ServedTable` of a downloaded Parquet file. Each scenario is + replayed twice and each test validates the whole contract, so the + tables are read once per file content. + + Raises: + _UnfitError: The file is not a Parquet file. + """ + try: + table = pq.read_table(pa.BufferReader(data)) + except (pa.ArrowException, OSError) as exc: + raise _UnfitError( + f"a downloaded file is unreadable: not Parquet: {exc}" + ) from exc + same = {} + for name in table.column_names if table.num_rows else (): + values = table.column(name).unique() + if len(values) == 1: + same[name] = values[0].as_py() + return ServedTable(tuple(table.column_names), table.num_rows, same) + + +def filled_slots( + columns: Collection[str], reading: Sequence[Sequence[str]] +) -> list[str]: + """The reading slots `columns` fill, in slot order: each slot's first name + the columns hold.""" + return [ name for names in reading if (name := next((n for n in names if n in columns), None)) is not None ] - rest = [c for c in columns if c not in first and c != STAT_VAR] - return first + rest + [STAT_VAR] * (STAT_VAR in columns) + + +def _varies(name: str, tables: Sequence[ServedTable]) -> bool: + """Whether column `name` holds more than one value over the rows of + `tables` stacked, a table without the column holding null in it.""" + values = [] + for table in tables: + if not table.rows: + continue + if name in table.columns and name not in table.same: + return True + values.append(table.same.get(name)) + return any(value != values[0] for value in values[1:]) + + +@dataclass(frozen=True, slots=True) +class ColumnGroups: + """A dataset's columns in the reading-order rule's groups: `part` when + there are several members, the filled reading slots in slot order, the + other columns whose values differ between the stacked rows and those + that hold one value in every row, each in served order, and `stat_var` + with whether it varies.""" + + part: tuple[str, ...] + first: tuple[str, ...] + varying: tuple[str, ...] + same: tuple[str, ...] + last: tuple[str, ...] + last_varies: bool + + @property + def columns(self) -> list[str]: + return [*self.part, *self.first, *self.varying, *self.same, *self.last] + + @property + def sort_keys(self) -> list[str]: + """The columns a client sorts each member's rows by: the filled + reading slots in slot order without `value`, a measure, and without + the time period, then the varying columns in presented order, + `stat_var` among them when it varies, then the time period, so each + series' years read together.""" + keys = [c for c in self.first if c not in (VALUE, TIME_PERIOD)] + keys += [*self.varying, *(self.last if self.last_varies else ())] + return keys + [TIME_PERIOD] * (TIME_PERIOD in self.first) + + +def column_groups( + tables: Sequence[ServedTable], reading: Sequence[Sequence[str]] +) -> ColumnGroups: + """The reading-order rule's column groups for a dataset whose members + serve `tables`. The served order gathers the members' columns in member + order, each once.""" + gathered = list(dict.fromkeys(c for table in tables for c in table.columns)) + first = filled_slots(gathered, reading) + rest = [c for c in gathered if c not in first and c != STAT_VAR] + varying = [c for c in rest if _varies(c, tables)] + return ColumnGroups( + part=(PART,) if len(tables) > 1 else (), + first=tuple(first), + varying=tuple(varying), + same=tuple(c for c in rest if c not in varying), + last=(STAT_VAR,) if STAT_VAR in gathered else (), + last_varies=STAT_VAR in gathered and _varies(STAT_VAR, tables), + ) def presented_columns( - members: Sequence[Sequence[str]], reading: Sequence[Sequence[str]] + tables: Sequence[ServedTable], reading: Sequence[Sequence[str]] ) -> list[str]: - """The columns a client presents for a dataset whose members hold - `members`: the members' columns gathered in member order, each once, - in the order `column_order` gives, after `part` when there are several.""" - gathered = list(dict.fromkeys(c for columns in members for c in columns)) - part = ["part"] if len(members) > 1 else [] - return [*part, *column_order(gathered, reading)] + """The columns a client presents for a dataset whose members serve + `tables`, as `column_groups` orders them.""" + return column_groups(tables, reading).columns + + +def presented_rows( + members: Sequence[tuple[str | None, Sequence[str], Sequence[Sequence[Any]]]], + reading: Sequence[Sequence[str]], +) -> tuple[list[str], list[list[Any]]]: + """The columns and rows a client presents for members given as their + `part` value, served columns and served rows. Each member's rows are + sorted by `ColumnGroups.sort_keys`, ascending with nulls last and ties in served + order, and the members follow in member order. A row of a stacked + dataset holds null in a column its member lacks.""" + tables = [table_from_rows(columns, rows) for _, columns, rows in members] + groups = column_groups(tables, reading) + columns, keys = groups.columns, groups.sort_keys + presented = [] + for part, served, rows in members: + at = {name: index for index, name in enumerate(served)} + cells = [ + {**{name: row[index] for name, index in at.items()}, PART: part} + for row in rows + ] + ordered = sorted( + cells, + key=lambda cell: [(cell.get(key) is None, cell.get(key)) for key in keys], + ) + presented.extend([cell.get(name) for name in columns] for cell in ordered) + return columns, presented def load_reading_columns(package_dir: Path) -> list[list[str]] | None: - """The reading slots `rules/column-order.json` lists, or None when the - file does not hold them, which the rules check reports.""" + """The reading slots `rules/reading-order.json` lists, or None when the + file does not hold them, which the validator reports.""" try: - reading = _load(package_dir / "rules" / "column-order.json")["reading_columns"] + reading = _load(package_dir / "rules" / "reading-order.json")["reading_columns"] except (OSError, ValueError, KeyError, TypeError): return None if isinstance(reading, list) and all( @@ -1850,31 +1988,6 @@ def load_reading_columns(package_dir: Path) -> list[list[str]] | None: return None -def served_columns(data: bytes) -> list[str]: - """A downloaded file's columns, as `parquet_columns` reads them. - - Raises: - _UnfitError: The file is not a Parquet file. - """ - try: - return parquet_columns(data) - except ValueError as exc: - raise _UnfitError(f"a downloaded file is unreadable: {exc}") from exc - - -def parquet_columns(data: bytes) -> list[str]: - """The top-level column names of a Parquet file, in the order its writer - wrote them, read from its footer's schema. - - Raises: - ValueError: `data` is not a Parquet file. - """ - try: - return list(pq.read_schema(pa.BufferReader(data)).names) - except (pa.ArrowException, OSError) as exc: - raise ValueError(f"not a Parquet file: {exc}") from exc - - def posted_scope(step: Mapping[str, Any]) -> dict[str, Any] | None: """The `input.scope` a fetch of a spec posts for the step's narrowing, or None when it narrows nothing. The server's @@ -2598,7 +2711,7 @@ def member( "spec_id": built, "content_version": manifest["content_version"], "sentence": sentence, - "served_columns": served_columns(data), + "served": served_table(data), } def sentence( @@ -2636,7 +2749,7 @@ def stored_file( data, refusal = self.stored_file(member, manifest) if data is None: return None, refusal - return {**stored, "served_columns": served_columns(data)}, None + return {**stored, "served": served_table(data)}, None def rebuilt(manifest: dict[str, Any]) -> dict[str, Any]: held = {**spec, "sentence": member["sentence"]} @@ -2842,16 +2955,14 @@ def dataset( search_warning: bool | None = None, ) -> dict[str, Any]: """A dataset outcome as `dataset_summary` gives it, with the columns a - client presents: the members' served columns, gathered in member - order with each once and ordered by the column-order rule, after - `part` when there are several members. Without the rule's reading - slots, which the rules check reports, the outcome gives no columns.""" + client presents: the members' served columns ordered by the + reading-order rule, as `presented_columns` gives them. Without the + rule's reading slots, which the validator reports, the outcome gives + no columns.""" summary = dataset_summary(members, gaps, search_warning=search_warning) if self.reading is None: return {"dataset": summary} - columns = presented_columns( - [m["served_columns"] for m in members], self.reading - ) + columns = presented_columns([m["served"] for m in members], self.reading) return {"dataset": {"columns": columns, **summary}} @@ -3112,7 +3223,7 @@ def _check_expect( if columns is not None and dataset["columns"] != columns: problems.append( f"scenarios/{file_name}: step {index}: columns " - f"{json.dumps(dataset['columns'])[:300]} are not the column-order " + f"{json.dumps(dataset['columns'])[:300]} are not the reading-order " f"rule's order of the served columns, {json.dumps(columns)[:300]}" ) if compared != wanted: @@ -3561,16 +3672,19 @@ def _check_rules(validator: jsonschema.Draft202012Validator) -> list[str]: f"{label}: vector {name!r} repeats" for name in sorted({n for n in names if names.count(n) > 1}) ) - if document["rule"] == "column-order": + if document["rule"] == "reading-order": problems.extend( - f"{label}: vector {vector['name']!r} orders its columns " - f"otherwise than presented_columns" + f"{label}: vector {vector['name']!r} orders its columns or rows " + f"otherwise than presented_rows" for vector in document["vectors"] - if presented_columns( - vector.get("members", [vector.get("columns")]), + if presented_rows( + [ + (member.get("part"), member["columns"], member["rows"]) + for member in vector["members"] + ], document["reading_columns"], ) - != vector["ordered"] + != (vector["columns"], vector["rows"]) ) return problems @@ -3653,7 +3767,7 @@ def validate() -> list[str]: reading = load_reading_columns(PACKAGE_DIR) if reading is None: problems.append( - "rules/column-order.json: missing or unreadable, or holds no " + "rules/reading-order.json: missing or unreadable, or holds no " "reading_columns list of column-name lists, so no dataset's columns " "are checked" )