From bb22a5dded45fae61eadda1e4e3f7e713063981c Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sat, 24 Oct 2026 20:37:16 +0200 Subject: [PATCH 1/7] fix(statvar-members): settle stub twins under a confirmed root (#114) A possible root twin whose read the reader holds complete and that carries none of the five-tuple definition arcs answers the same body to every read, so the twin pass could never settle it and the walk refused the family with member_not_resolvable. Under a confirmed root, _read_root_twins now settles such stubs before spending the twin budget and returns them as MemberGroup.stub_candidates. The walk log shows them as stubs= and the mint trace states them as trace.stub_candidates. Without a confirmed root, with an incomplete read, or with any definition arc present, the refusal stays. --- docs/conformance-datasets-api-v1.md | 2 +- .../dfo-data-tools/src/materialise/finish.py | 2 + .../src/materialise/pipeline.py | 4 + .../dfo-data-tools/src/materialise/trace.py | 7 + .../src/statvar_members/cache.py | 3 +- .../src/statvar_members/enumerate.py | 73 ++++++++--- src/datasets_api/README.md | 2 +- src/datasets_api/adapter.py | 7 +- src/datasets_api/shape.py | 3 + .../datasets_api/test_deep_root_full_read.py | 55 +++++++- tests/materialise/test_trace.py | 5 +- tests/origin_catalogue/test_graph.py | 23 ++++ .../test_absence_reconciliation.py | 122 ++++++++++++++++++ tests/statvar_members/test_root_twins.py | 9 +- 14 files changed, 291 insertions(+), 26 deletions(-) diff --git a/docs/conformance-datasets-api-v1.md b/docs/conformance-datasets-api-v1.md index 28920894..eb7032ea 100644 --- a/docs/conformance-datasets-api-v1.md +++ b/docs/conformance-datasets-api-v1.md @@ -141,7 +141,7 @@ Five routes are outside the V1 contract text. The code registry's mappings for ` | C71 | `trace` records `graph_revision`, the graph generation stamp the run read (null when unknown), which its `datasets.materialisation` row records as `graph_generation`, and `source_revisions`, each source the run's dependency fingerprint names with its refresh token | implemented | | | C155 | `trace` records `code_index_stamp` (`content_hash`, `exported_at`, `graph_endpoint`) beside `graph_revision`, naming the code index that served the request whether member resolution derived from it or walked the graph | implemented | | | C72 | `trace` records the resolved StatVars with their role signatures | implemented | | -| C72a | `trace` records `family_member_digest` and `family_member_count`: the digest (lowercase sha256 hex of the RFC 8785 canonical JSON of the sorted bare dcids, the function Search's `resolved.shape.member_digest` uses) and the number of every member the identity admits, before antichain reduction. `resolved_stat_vars` lists the members served, which leave out each admitted member a broader member dominates. A pinned identity admits its one StatVar. Both fields are null when member resolution did not read the whole family (a graph walk whose candidate page was truncated and left candidates unread, though a confirmed root proved the family complete). Neither enters `content_version`, so a write or a read answered from a stored version returns the trace recorded when that version was first stored: the two fields describe the family at that mint, and both keys are absent on a version stored before the fields existed. A client reads an absent key and a null alike as unknown. `facets.provenance` names the identity's dataset when it binds one distinct ref, as a `value` or as a `set` of one, which hash to one `spec_id`. A mint reads a posted sidecar in its canonical form, the document the registry stores, so both spellings resolve the same members and serve the same `content_version`. A set of two or more names none, and the family is then the default election's | implemented | | +| C72a | `trace` records `family_member_digest` and `family_member_count`: the digest (lowercase sha256 hex of the RFC 8785 canonical JSON of the sorted bare dcids, the function Search's `resolved.shape.member_digest` uses) and the number of every member the identity admits, before antichain reduction. `resolved_stat_vars` lists the members served, which leave out each admitted member a broader member dominates. A pinned identity admits its one StatVar. Both fields are null when member resolution did not read the whole family (a graph walk whose candidate page was truncated and left candidates unread, though a confirmed root proved the family complete). Neither enters `content_version`, so a write or a read answered from a stored version returns the trace recorded when that version was first stored: the two fields describe the family at that mint, and both keys are absent on a version stored before the fields existed. A client reads an absent key and a null alike as unknown. `trace.stub_candidates` lists, as `dcid:` refs, the candidates a graph walk under a confirmed root left out of the family because its reader holds their arcs complete and they carry none of the arcs that define a StatVar (`typeOf`, `populationType`, `measuredProperty`, `statType`, `measurementQualifier`, `measurementDenominator`, `constraintProperties`, `observationProperties`), though the reverse read placed them in the family (`MemberGroup.stub_candidates`); the key is omitted when there are none. `facets.provenance` names the identity's dataset when it binds one distinct ref, as a `value` or as a `set` of one, which hash to one `spec_id`. A mint reads a posted sidecar in its canonical form, the document the registry stores, so both spellings resolve the same members and serve the same `content_version`. A set of two or more names none, and the family is then the default election's | implemented | | | C73 | `trace` records the facet actually selected per axis with `basis` (`bound`, `declared_default`, `deployment_fallback`) | implemented | | | C74 | `trace` records the default policy node and version with the selection-reason enum (`sole_candidate`, `most_observations`, `most_observations_over_disagreement`, `dfo_precedence`, `absent_on_selected_facet`) where a default applied | implemented | | | C75 | `trace` records the transient dispatch facet id, normalisations, one conversion entry per recipe and table-version pair where a recipe ran, listing each member it converted with that member's native unit (omitted otherwise), the frame grammar version, the materialiser build, and the capability version | implemented | | diff --git a/packages/dfo-data-tools/src/materialise/finish.py b/packages/dfo-data-tools/src/materialise/finish.py index 219283fd..0dc1d993 100644 --- a/packages/dfo-data-tools/src/materialise/finish.py +++ b/packages/dfo-data-tools/src/materialise/finish.py @@ -242,6 +242,7 @@ def _finish_member( materialiser_build=materialiser_build, capability_version=capability_version, family=request.family, + stub_candidates=request.stub_candidates, coverage_gaps=tuple(gap.as_trace_entry() for gap in outcome.coverage_gaps), ) @@ -351,6 +352,7 @@ def _finish_group( capability_version=capability_version, # Every request of a group is a member of one definition and carries its family. family=requests[0].family, + stub_candidates=requests[0].stub_candidates, ) return pipeline.MaterialisationResult( diff --git a/packages/dfo-data-tools/src/materialise/pipeline.py b/packages/dfo-data-tools/src/materialise/pipeline.py index 69ab8840..b3220d1f 100644 --- a/packages/dfo-data-tools/src/materialise/pipeline.py +++ b/packages/dfo-data-tools/src/materialise/pipeline.py @@ -93,6 +93,10 @@ class MaterialisationRequest: # The whole population the definition's identity admits, shared by every request of the # definition and stated in the trace. `None` when the resolver could not state it. family: MemberFamily | None = None + # The `dcid:` refs member resolution left out of that family as stubs, nodes whose complete + # read carries no definition arc, shared by every request of the definition and stated in + # the trace. + stub_candidates: tuple[str, ...] = () # A caller's facet filter over each probed candidate's provenance, method and unit. The # member elects only among the candidates it admits (`facets.admitted_candidates`), so # a filtered request serves its best admitted facet. None admits every candidate. diff --git a/packages/dfo-data-tools/src/materialise/trace.py b/packages/dfo-data-tools/src/materialise/trace.py index 85c14f1b..e6890b50 100644 --- a/packages/dfo-data-tools/src/materialise/trace.py +++ b/packages/dfo-data-tools/src/materialise/trace.py @@ -82,6 +82,11 @@ class ManifestTrace: family: MemberFamily | None = None # The population the definition's identity admits, from the request. `None` when the # resolver could not state the whole of it, and on a trace that describes no one family. + stub_candidates: tuple[str, ...] = () + # The `dcid:` refs the member walk left out of the family under a confirmed root because + # its reader holds their arcs complete and they carry no definition arc + # (`MemberGroup.stub_candidates`). + # Omitted from `to_dict()` when empty. coverage_gaps: tuple[dict[str, object], ...] = () # The places a national-currency unit binding serves no row at, one entry per role and # reason (`national_currency.CoverageGap.as_trace_entry`): `role`, `reason` and the @@ -126,6 +131,8 @@ def to_dict(self) -> dict[str, object]: result["resolved_stat_vars"] = list(self.resolved_stat_vars) result["family_member_digest"] = self.family.digest if self.family is not None else None result["family_member_count"] = self.family.count if self.family is not None else None + if self.stub_candidates: + result["stub_candidates"] = list(self.stub_candidates) result["facets"] = list(self.facets) result["normalisations"] = list(self.normalisations) if self.conversion is not None: diff --git a/packages/dfo-data-tools/src/statvar_members/cache.py b/packages/dfo-data-tools/src/statvar_members/cache.py index 5d2d2d5c..8ad5bd93 100644 --- a/packages/dfo-data-tools/src/statvar_members/cache.py +++ b/packages/dfo-data-tools/src/statvar_members/cache.py @@ -295,7 +295,7 @@ def _walk( logger.info( "statvar_members.walk spec_id=%s five_tuple=%s truncated=%s proven_complete=%s " "candidates_considered=%d full_read=%s inconsistent=%d undismissed=%d " - "outcome=%s elapsed_s=%.2f", + "stubs=%s outcome=%s elapsed_s=%.2f", spec_id, _five_tuple_label(five_tuple), group.truncated, @@ -304,6 +304,7 @@ def _walk( group.full_read, len(group.inconsistent_candidates), len(group.undismissed_candidates), + ",".join(group.stub_candidates) or "-", _outcome(group), time.monotonic() - started, ) diff --git a/packages/dfo-data-tools/src/statvar_members/enumerate.py b/packages/dfo-data-tools/src/statvar_members/enumerate.py index 8346e95b..153398d2 100644 --- a/packages/dfo-data-tools/src/statvar_members/enumerate.py +++ b/packages/dfo-data-tools/src/statvar_members/enumerate.py @@ -139,12 +139,15 @@ def five_tuple_from_identity(shape_member: Mapping[str, object]) -> FiveTuple: a twin naming a denominator target `_ProvisionalDenominatorReader` has not seen adds one batched `typeOf` read, and each target that reads empty in a batch of several adds one single-target re-read. The pass therefore costs at most 8 arc reads, 8 `typeOf` reads, and -one re-read per target that reads empty in a batch. Live families share one target -(`PerCapita`), which the root's read has seen, so they add none. Twins past the budget stay -in `undismissed_candidates`. Root-shaped suspects take the reconciliation -budgets first, so few reach this pass: dev walks of GDP per capita in Guatemala, +one re-read per target that reads empty in a batch. A twin whose complete read carries no +definition arc is settled without a read and spends none of the budget +(`MemberGroup.stub_candidates`). +Live families share one target (`PerCapita`), which the root's read has seen, so they add +none. Twins past the budget stay in `undismissed_candidates`. Root-shaped suspects take the +reconciliation budgets first, so few reach this pass: dev walks of GDP per capita in Guatemala, internet users in India, CO2 emissions in China, households in Kenya, and the -4,000-candidate `Count_Person` family left none.""" +4,000-candidate `Count_Person` family left none beyond the stubs +`MemberGroup.stub_candidates` describes.""" _ROLE_PROBE_BUDGET = 16 """`_settle_role_absence` reads at most 16 members one dcid at a time: those whose place in @@ -279,14 +282,28 @@ class MemberGroup: the graph's own read path, distinct from a candidate that simply does not match `slots`. Under `root_confirmed`, a possible root twin that `_read_root_twins` settles leaves this list.""" + stub_candidates: tuple[str, ...] = () + """The possible root twins `_read_root_twins` settled as non-members because the reader + holds their arcs complete (`CompleteArcReads.arcs_complete`) and their read carries none + of `_FIVE_TUPLE_PROPERTIES`, though the reverse query named them under the five-tuple. + No read of such a node confirms it, since every read answers the same body. The dev + graph's `Count_Person` marital-status splits from `dc/base/Mongolia_Demographics` are + such nodes: their forward body is a single `memberOf` arc while their reverse arcs place + them under Person/count/measuredValue. Settled only under `root_confirmed`: such a twin is a + non-member, a member the root dominates, or a same-keyset duplicate of the root, so + leaving it out narrows `members` and keeps them correct, the limit `proven_complete` + states for a duplicate past the window. Without a confirmed root a stub could stand for + the root itself, so it stays in `undismissed_candidates`. The mint trace names these + (`trace.stub_candidates`).""" undismissed_candidates: tuple[str, ...] = () """The `inconsistent_candidates` that could change `members` or their role agreement. Without `root_confirmed`, that is all of them, and role agreement is checked only when there are none. With it, `_possible_root_twins` keeps those whose reads name no constraint outside the `value`-bound slot keys, since each may be a same-keyset duplicate of the root whose roles disagree, and `_read_root_twins` drops each one its - single-dcid read settles. Role agreement among `members` is then checked regardless, - so a disagreement among confirmed members raises before any caller reads this.""" + single-dcid read settles and each one `stub_candidates` names. Role agreement among + `members` is then checked regardless, so a disagreement among confirmed members raises + before any caller reads this.""" undismissed_rereadable: bool = False """`True` when a later walk can dismiss one of `undismissed_candidates`: one whose reads the walk's reader does not hold complete (`CompleteArcReads`), so a dropped arc may have @@ -1019,6 +1036,8 @@ def _confirm_group( Missing `typeOf` is inconclusive, so winners receive a trusted re-read. A candidate rejected for an absent arc is re-read (`_reconcile_absence_rejected`) unless `arcs_complete` says its read holds every arc, which makes it a non-member. + Under a confirmed root, a possible root twin whose complete read carries no definition + arc is settled out (`MemberGroup.stub_candidates`). Extras are deduplicated and receive the same membership and reduction checks, and skip filter reconciliation, which applies only to candidates the reverse @@ -1123,13 +1142,16 @@ def _confirm_group( members, slots=slots, readings=readings, role_keys=election.role_keys ) and _root_covers_bound_keys(confirmed, slots=slots, role_keys=role_keys) undismissed = unresolved + stubs: tuple[str, ...] = () if root_confirmed: twins = _possible_root_twins( unresolved, arcs_by_dcid=arcs_by_dcid, consistent=consistent, slots=slots ) - read_twins, undismissed = _read_root_twins( + read_twins, undismissed, stubs = _read_root_twins( twins, walked=frozenset(candidate_dcids), + arcs_by_dcid=arcs_by_dcid, + arcs_complete=arcs_complete, five_tuple=five_tuple, slots=slots, graph=graph, @@ -1164,6 +1186,7 @@ def _confirm_group( ), root_confirmed=root_confirmed, inconsistent_candidates=unresolved, + stub_candidates=stubs, undismissed_candidates=undismissed, undismissed_rereadable=not all(map(arcs_complete, undismissed)), elected_open_role_keys=election.role_keys - role_keys, @@ -1303,24 +1326,37 @@ def _read_root_twins( twins: Sequence[str], *, walked: frozenset[str], + arcs_by_dcid: Mapping[str, Mapping[str, tuple[str, ...]] | None], + arcs_complete: Callable[[str], bool], five_tuple: FiveTuple, slots: Mapping[str, SlotBinding], graph: MemberArcReader, properties: Sequence[str], -) -> tuple[tuple[Member, ...], tuple[str, ...]]: - """Read the first `_TWIN_PROBE_BUDGET` possible root twins one dcid at a time. - +) -> tuple[tuple[Member, ...], tuple[str, ...], tuple[str, ...]]: + """Settle the twins whose complete read carries no definition arc, then read the first + `_TWIN_PROBE_BUDGET` of the rest one dcid at a time. + + A twin whose read in `arcs_by_dcid` carries none of `_FIVE_TUPLE_PROPERTIES` and whose + reader holds its arcs complete (`arcs_complete`) is settled as a non-member without a + read, since every read answers the same: the limit `MemberGroup.stub_candidates` states. + A batch read of a live reader can drop every arc, so a bare read it does not hold + complete settles nothing. A single-dcid read is trusted, so a read settles its twin: a confirmed member joins the group (a root twin survives beside the root, any other the root dominates), and a non-match is dismissed. A twin in `walked` is the exception when its read lacks a filter-covered arc: the reverse query proved that arc, so the read contradicts the index and settles nothing, as in `_reconcile_filter_covered`. Returns the confirmed - members and the twins still unsettled: those whose read came back empty or - contradicted the index, and those past the budget.""" + members, the twins still unsettled (those whose read came back empty or contradicted + the index, and those past the budget), and the twins settled as stubs.""" + stubs = tuple( + dcid for dcid in twins if _is_stub(arcs_by_dcid.get(dcid)) and arcs_complete(dcid) + ) + settled = frozenset(stubs) + to_read = [dcid for dcid in twins if dcid not in settled] scalar_expected, slot_expected = _filter_covered_expectations(five_tuple, slots) confirmed: list[Member] = [] unsettled: list[str] = [] - for dcid in twins[:_TWIN_PROBE_BUDGET]: + for dcid in to_read[:_TWIN_PROBE_BUDGET]: arcs = graph.node_arcs_many([dcid], properties=properties).get(dcid) if arcs is None: unsettled.append(dcid) @@ -1335,7 +1371,14 @@ def _read_root_twins( ).member if member is not None: confirmed.append(member) - return tuple(confirmed), (*unsettled, *twins[_TWIN_PROBE_BUDGET:]) + return tuple(confirmed), (*unsettled, *to_read[_TWIN_PROBE_BUDGET:]), stubs + + +def _is_stub(arcs: Mapping[str, tuple[str, ...]] | None) -> bool: + """Whether a read carries none of `_FIVE_TUPLE_PROPERTIES`: no `typeOf`, no + `populationType`, `measuredProperty` or `statType`, and no other definition arc. An + absent node's `None` read carries none either.""" + return not any((arcs or {}).get(name) for name in _FIVE_TUPLE_PROPERTIES) def _root_shaped_first( diff --git a/src/datasets_api/README.md b/src/datasets_api/README.md index 9b288604..42d6ae09 100644 --- a/src/datasets_api/README.md +++ b/src/datasets_api/README.md @@ -195,7 +195,7 @@ Each request, and each worker operation, binds one origin catalogue build (`ORIG - `citation.sources[]` entries carry `source` (`{ref, label}`, the Data Commons `Source` node behind the provenance, or for a UN System Data Commons series the agency provenance itself) and `data_commons` (`{id, label}`, with `id` one of `dfo`, `base`, `un_system`). `label`, `url`, `source` and `data_commons` come from the catalogue's provenance table and `license`, `license_type`, `license_url` and `documentation` from the graph (`materialise.provenance_doc.read_source_citations`). `documentation` is the source's `read_provenance_doc` content as read at first mint, `null` when the graph documents nothing for the source or that read failed, and it feeds the bundle's `README.md` and `SOURCES.md`. A bundle render of a version already on record fills every stored `null` source field except `ref`, `label` and `processing_level` (`license`, `license_type`, `license_url`, `url`, `source`, `data_commons` and `documentation`) from a live read, built as a first mint builds it (`datasets_api.manifest.cited_source`), for that render only and only when no servable bundle is linked under the request's rendering key. Every set field stands, and a source with every field set reads nothing. A mint leaves the stored body as it is. A bundle whose documentation or code label read failed is linked to the version as incomplete: its `rendering` carries `"metadata": "incomplete"`, its download link resolves and its digest can be pinned. No request's rendering key matches it, so the next request for the bundle renders and reads again and links a complete bundle beside it. A snapshot lists an incomplete artifact only when it is pinned, after the current artifacts, and lists bytes also linked under the complete rendering once, without the marker. A provenance the table does not list, such as a DFO provenance newer than the loaded build, is named from the graph and carries `source` and `data_commons` as `null`. - `trace.origin_catalogue_build_id` names the build that admitted the data. A version minted before the catalogue existed keeps serving as minted, with neither the trace member nor the citation members, so its absence marks a pre-catalogue version. -- `trace.family_member_digest` and `trace.family_member_count` state every member the identity admits, before antichain reduction, by the digest Search's `resolved.shape.member_digest` uses. `trace.resolved_stat_vars` lists the members served. Both fields are null when member resolution did not read the whole family, which is a graph walk whose candidate page was truncated. They stay out of `content_version`, so an answer from a stored version carries the trace recorded when that version was first stored: the fields describe the family at that mint, and both keys are absent on a version stored before they existed. Read an absent key and a null alike as unknown. +- `trace.family_member_digest` and `trace.family_member_count` state every member the identity admits, before antichain reduction, by the digest Search's `resolved.shape.member_digest` uses. `trace.resolved_stat_vars` lists the members served. Both fields are null when member resolution did not read the whole family, which is a graph walk whose candidate page was truncated. `trace.stub_candidates` lists, as `dcid:` refs, the candidates a graph walk under a confirmed root left out of the family because its reader holds their arcs complete and they carry none of the arcs that define a StatVar (`typeOf`, `populationType`, `measuredProperty`, `statType`, `measurementQualifier`, `measurementDenominator`, `constraintProperties`, `observationProperties`), though the reverse read placed them in the family (`MemberGroup.stub_candidates`); the key is omitted when there are none. They stay out of `content_version`, so an answer from a stored version carries the trace recorded when that version was first stored: the fields describe the family at that mint, and both keys are absent on a version stored before they existed. Read an absent key and a null alike as unknown. - `warnings` carries `origin_catalogue_stale` when the build is more than 14 days old; `un_system_unavailable` naming the places the UN System Data Commons refused or failed to read; and `withheld_by_registry` when a subject entity (a bound one, or for a request that binds no entity a country under Earth) came back with no rows, has no admitted data at all, and a provenance the catalogue withholds holds data there, naming those entities and the withheld provenances. A mint that lands on a version already on record adds this call's warnings to the stored body and replaces the stored `origin_catalogue_stale` entry with this call's. When no member's facet probe returns a candidate, the mint serves the empty result: a 200 manifest with `coverage.rows` at 0, `empty_result`, no facet selection and no cited source (`MaterialisationResult.probe_empty`). `prewrite_definition` first runs `origin.refuse_unserved_empty_probe`, which refuses before anything is written in two cases. When every facet for a subject-only member's variable at its places is withheld, the request refuses `excluded_by_registry` (422) naming the withheld provenances. The refusal names the variables of the read that found withheld data. A UN member with no candidate at places the UN refused or failed to read refuses `upstream_error` (502, retryable) naming those places, since its empty probe says nothing about data there. A UN member that binds no entity covers every country, including the few the UN front door refuses on every read, so it refuses `upstream_error` only when a UN read failed. With no data anywhere else it is the empty result, and its `un_system_unavailable` warning names the refused places. The refusal names the first five places and counts the rest. The facet probe runs before the mint doors open the origin record, so the probe carries what its UN reads could not serve and the materialisation adds it to that record: the refusal and the `un_system_unavailable` warning are the same at the mint doors and the worker. Both `excluded_by_registry` and `withheld_by_registry` cost a facet read with no domain filter, made only when a requested series left an entity without rows, and under the request's deadline and a 16 MiB response cap (`datasets_api.origin`). The warning takes one read, and the refusal one read per distinct list of entities among the subject-only members. On an empty probe a re-read that fails or passes the cap refuses `upstream_error` (502, retryable), and one the deadline ends refuses `timeout` (504), so no empty version or head is written unless the check completed. A member whose candidates leave a facet axis without a default (`default_facet_undefined`) gets the same two checks for that member alone (`origin.refuse_unserved_selection`) and refuses `upstream_error` or `excluded_by_registry` in its place. For a `default_facet_undefined` refusal and for the warning, a failed re-read leaves the answer without the explanation. A subject-only request that binds no entity is re-read at the countries under Earth, and for the refusal at Earth too, the places its facet probe read. The check on an empty probe reads those countries again, and a failed read refuses `upstream_error` (502, retryable). A UN member that binds none refuses on a failed UN read before that read, since its probe read only Earth and its countries. For such a request nearly every mint leaves a country without rows, so nearly every mint makes the re-read. The warning takes the countries from the materialisation result, so a mint reads containment once, in its facet probe. Role-bearing requests and `ContainedIn` scopes are not re-read, because role-bearing series are DFO's and always admitted, and a scope's members are unknown before the read. diff --git a/src/datasets_api/adapter.py b/src/datasets_api/adapter.py index 80c26716..afffa5b8 100644 --- a/src/datasets_api/adapter.py +++ b/src/datasets_api/adapter.py @@ -438,6 +438,7 @@ def to_materialisation_request( index_sourced=_identity_is_index_sourced(identity, code_index=code_index), dataset=shape.dataset, family=admitted_family(identity, shape), + stub_candidates=shape.stub_candidates, ) @@ -716,7 +717,9 @@ def resolve_definition_members( them (`MemberGroup.undismissed_candidates` is empty). A candidate that may duplicate the root still refuses, since its roles are unverified. The refusal is retryable when a later walk can settle a candidate it names - (`MemberGroup.undismissed_rereadable`). + (`MemberGroup.undismissed_rereadable`). A possible duplicate whose complete read carries + no definition arc is settled out under the root (`MemberGroup.stub_candidates`), and + each shape names it (`StatVarShape.stub_candidates`) for the trace. Both paths elect one role class (`statvar_members.elect_role_class`): the members whose declared roles cover the role keys `entities` is bound under, least-constrained first, then those declaring the fewest roles, so a member @@ -832,6 +835,7 @@ def resolve_definition_members( walked = frozenset(group.family) stated = _member_family(group.stated_family) if group.stated_family is not None else None + stubs = tuple(f"{_STAT_VAR_PREFIX}{candidate}" for candidate in group.stub_candidates) shapes = tuple( StatVarShape( stat_var=f"{_STAT_VAR_PREFIX}{member.stat_var}", @@ -840,6 +844,7 @@ def resolve_definition_members( measured_property=five_tuple.measured_property, population_type=five_tuple.population_type, family=stated, + stub_candidates=stubs, ) for member in group.members ) diff --git a/src/datasets_api/shape.py b/src/datasets_api/shape.py index 5c450a46..77b2fac1 100644 --- a/src/datasets_api/shape.py +++ b/src/datasets_api/shape.py @@ -45,6 +45,8 @@ class StatVarShape: `dataset` is the dataset elected for a family the code index resolved, and `None` for a shape read from the graph. `family` is the whole population the identity the shape was resolved for admits, and `None` when member resolution could not state it. + `stub_candidates` holds the `dcid:` refs the graph walk left out of that family because + their complete read carries no definition arc (`MemberGroup.stub_candidates`). """ stat_var: str @@ -54,6 +56,7 @@ class StatVarShape: population_type: str | None dataset: ElectedDataset | None = None family: MemberFamily | None = None + stub_candidates: tuple[str, ...] = () def resolve_shape(stat_var: str, *, graph: GraphReader) -> StatVarShape: diff --git a/tests/datasets_api/test_deep_root_full_read.py b/tests/datasets_api/test_deep_root_full_read.py index 705a8785..c54e41f6 100644 --- a/tests/datasets_api/test_deep_root_full_read.py +++ b/tests/datasets_api/test_deep_root_full_read.py @@ -65,16 +65,23 @@ def _member_arcs(gender: str | None) -> dict[str, tuple[str, ...]]: return arcs -def _graph() -> FirstPageWithholdingGraphReader: - return FirstPageWithholdingGraphReader( +def _graph( + reader: type[FirstPageWithholdingGraphReader] = FirstPageWithholdingGraphReader, + *, + extra_arcs: dict[str, dict[str, tuple[str, ...]]] | None = None, + **kwargs, +) -> FirstPageWithholdingGraphReader: + return reader( withheld=frozenset({_ROOT}), arcs={ _ROOT: _member_arcs(None), _MALE: _member_arcs("Male"), _FEMALE: _member_arcs("Female"), + **(extra_arcs or {}), }, facets={(_ROOT, _RECIPIENT): [_FACET]}, labels={"OECD_DAC2A": "OECD DAC2a", _RECIPIENT: "Kenya", _ROOT: "Population"}, + **kwargs, ) @@ -109,9 +116,14 @@ def _mint_body(document: dict) -> dict: } -def _client(*, settings: DatasetsBuildSettings | None = None, **kwargs): +def _client( + *, + graph: FirstPageWithholdingGraphReader | None = None, + settings: DatasetsBuildSettings | None = None, + **kwargs, +): return _base_client( - graph=_graph(), + graph=graph or _graph(), head_writer=_FakeHeadStore(), sdmx_transport=_sdmx_transport(), coverage=_coverage_service(stat_vars=(_ROOT,), recipient=_RECIPIENT), @@ -132,6 +144,41 @@ def test_a_mint_settles_a_root_past_the_first_page_and_states_its_family(): assert (trace["family_member_count"], trace["family_member_digest"]) == (3, _FAMILY_DIGEST) +class _StubGraphReader(FirstPageWithholdingGraphReader): + """Holds every node's arcs complete and answers forward reads of `stubs` with one + `memberOf` arc while the reverse read still places them in the family, as dev reads the + `dc/base/Mongolia_Demographics` marital-status splits of `Count_Person`.""" + + def __init__(self, *, stubs: frozenset[str], **kwargs) -> None: + super().__init__(**kwargs) + self._stubs = stubs + + def arcs_complete(self, dcid: str) -> bool: + return True + + def node_arcs_many(self, dcids, *, properties, individual_probe_budget=None, deadline=None): + read = super().node_arcs_many(dcids, properties=properties) + return { + dcid: dict.fromkeys(properties, ()) if dcid in self._stubs else arcs + for dcid, arcs in read.items() + } + + +def test_a_mint_leaves_out_a_stub_under_the_root_and_names_it(): + stub = "Count_Person_15OrMoreYears_Female_Unmarried" + graph = _graph( + _StubGraphReader, extra_arcs={stub: _member_arcs("Female")}, stubs=frozenset({stub}) + ) + + resp = _client(graph=graph).post("/api/datasets", json=_mint_body(_identity())) + + assert resp.status_code == 200, resp.text + trace = resp.json()["trace"] + assert [entry["ref"] for entry in trace["resolved_stat_vars"]] == [f"dcid:{_ROOT}"] + assert (trace["family_member_count"], trace["family_member_digest"]) == (3, _FAMILY_DIGEST) + assert trace["stub_candidates"] == [f"dcid:{stub}"] + + def test_a_get_with_no_head_settles_the_registered_definition(): document = _identity() spec_id = recompute_identity(document).spec_id diff --git a/tests/materialise/test_trace.py b/tests/materialise/test_trace.py index 3c05e210..35a9934c 100644 --- a/tests/materialise/test_trace.py +++ b/tests/materialise/test_trace.py @@ -167,13 +167,16 @@ def test_to_dict_carries_the_origin_catalogue_build_only_when_set(): def test_to_dict_states_the_family_beside_the_members_served(): trace = replace( - _build_trace(applied_window=None), family=MemberFamily(digest="ab" * 32, count=3) + _build_trace(applied_window=None), + family=MemberFamily(digest="ab" * 32, count=3), + stub_candidates=("dcid:Count_Person_Stub",), ) result = trace.to_dict() assert result["family_member_digest"] == "ab" * 32 assert result["family_member_count"] == 3 + assert result["stub_candidates"] == ["dcid:Count_Person_Stub"] assert len(result["resolved_stat_vars"]) == 1 diff --git a/tests/origin_catalogue/test_graph.py b/tests/origin_catalogue/test_graph.py index d1b80e50..942a8be1 100644 --- a/tests/origin_catalogue/test_graph.py +++ b/tests/origin_catalogue/test_graph.py @@ -44,6 +44,7 @@ is_un_dcid, ) from origin_catalogue.topic_walk import walk_topics +from statvar_members.enumerate import _candidate_properties from tests.graph_release.support import origin_release from tests.origin_catalogue.fake_catalogue import restrictive_catalogue, un_mirror, un_variable from tests.origin_catalogue.support import build_arc_tables @@ -838,6 +839,28 @@ def test_a_node_an_arc_names_takes_its_label_from_the_labels_and_keeps_its_arc_n assert graph.requests == [] +def test_a_node_held_with_only_member_of_reads_as_a_whole_body_with_no_definition_arc( + graph: _Graph, client: OriginGraphClient, bind_catalogue +) -> None: + """Dev's base table holds `Count_Person_15OrMoreYears_Female_Unmarried` as one `memberOf` + arc. The member walk reads only the arcs that define a StatVar and its slots + (`_candidate_properties`), so its read is a held body with every requested arc empty, not + an absence, and the tables answer it whole (`arcs_complete`), with no request.""" + stub = "Count_Person_15OrMoreYears_Female_Unmarried" + group = "dc/g/Person_Gender-Female_MaritalStatus-Unmarried" + arcs = build_arc_tables( + dfo={}, base={stub: {"memberOf": {"nodes": [{"dcid": group}]}}}, schema={} + ) + bind_catalogue(_catalogue(arcs=arcs)) + properties = _candidate_properties({"gender": ("unbound",), "maritalStatus": ("unbound",)}) + + assert client.node_arcs_many([stub], properties=properties) == { + stub: dict.fromkeys(properties, ()) + } + assert client.arcs_complete(stub) + assert graph.requests == [] + + def test_a_node_the_arc_tables_hold_empty_reads_as_absent_with_no_request( graph: _Graph, client: OriginGraphClient, bind_catalogue ) -> None: diff --git a/tests/statvar_members/test_absence_reconciliation.py b/tests/statvar_members/test_absence_reconciliation.py index a60e0026..19a73f66 100644 --- a/tests/statvar_members/test_absence_reconciliation.py +++ b/tests/statvar_members/test_absence_reconciliation.py @@ -284,3 +284,125 @@ def test_only_complete_reads_skip_the_absence_re_read( first, *rereads = reader.reads assert set(first) == set(arcs) assert not {dcid for read in rereads for dcid in read} & frozenset(complete_ages) + + +def _gender_arcs(gender: str) -> dict[str, tuple[str, ...]]: + return _arcs(constraintProperties=("gender",), gender=(gender,)) + + +class _StubReader(_PartlyCompleteReader): + """Answers forward reads of the dcids in `bodies` from those bodies, while the reverse + query still finds them by `arcs`, as our mixer's in-arcs and out-arcs disagree for the + `dc/base/Mongolia_Demographics` StatVars. A `None` body reads absent.""" + + def __init__(self, *, bodies: dict[str, dict[str, tuple[str, ...]] | None], **kwargs) -> None: + super().__init__(**kwargs) + self._bodies = bodies + + def node_arcs_many(self, dcids, *, properties, individual_probe_budget=None, deadline=None): + read = super().node_arcs_many(dcids, properties=properties) + for dcid in set(read) & set(self._bodies): + body = self._bodies[dcid] + read[dcid] = None if body is None else {prop: body.get(prop, ()) for prop in properties} + return read + + +_MEMBER_OF_ONLY = {"memberOf": ("dc/g/Amount_Stub",)} + + +@pytest.mark.parametrize( + ("with_root", "slots", "body", "stub_complete", "members", "stubs", "undismissed"), + [ + pytest.param( + True, + {"gender": _UNBOUND}, + _MEMBER_OF_ONLY, + True, + ["Amount"], + ("Amount_Stub",), + (), + id="member_of_only_settled_under_a_confirmed_root", + ), + pytest.param( + True, + {"gender": _UNBOUND}, + None, + True, + ["Amount"], + ("Amount_Stub",), + (), + id="absent_settled_under_a_confirmed_root", + ), + pytest.param( + True, + {"gender": _UNBOUND}, + _MEMBER_OF_ONLY, + False, + ["Amount"], + (), + ("Amount_Stub",), + id="refused_when_the_read_is_not_complete", + ), + pytest.param( + True, + {"gender": _UNBOUND}, + {**_MEMBER_OF_ONLY, "typeOf": ("StatisticalVariable",)}, + True, + ["Amount"], + (), + ("Amount_Stub",), + id="refused_when_the_body_carries_a_definition_arc", + ), + pytest.param( + False, + {"gender": _UNBOUND}, + _MEMBER_OF_ONLY, + True, + ["Amount_Female", "Amount_Male"], + (), + ("Amount_Stub",), + id="refused_without_a_root", + ), + pytest.param( + True, + {"gender": ("set", ("Male", "Female"))}, + _MEMBER_OF_ONLY, + True, + ["Amount_Female", "Amount_Male"], + (), + ("Amount_Stub",), + id="refused_when_a_set_slot_rules_the_root_out", + ), + ], +) +def test_a_complete_stub_settles_only_under_a_confirmed_root( + with_root: bool, + slots: dict[str, SlotBinding], + body: dict[str, tuple[str, ...]] | None, + stub_complete: bool, + members: list[str], + stubs: tuple[str, ...], + undismissed: tuple[str, ...], +): + """The reverse query names `Amount_Stub` under the five-tuple, but its forward body is + one `memberOf` arc, as dev's `Count_Person_15OrMoreYears_Female_Unmarried` reads. Under a + confirmed root a complete read carrying no definition arc settles it out and the group + names it. Without a confirmed root it could stand for the root, a read the reader does not + hold complete could have dropped its arcs, and a body with a definition arc contradicts + the reverse read, so each of those still refuses.""" + arcs = { + "Amount_Male": _gender_arcs("Male"), + "Amount_Female": _gender_arcs("Female"), + "Amount_Stub": _arcs(), + **({"Amount": _arcs()} if with_root else {}), + } + complete = frozenset(arcs) - (set() if stub_complete else {"Amount_Stub"}) + reader = _StubReader(bodies={"Amount_Stub": body}, complete=complete, arcs=arcs) + + group = enumerate_members(five_tuple=_NO_DENOMINATOR_ASK, slots=slots, graph=reader) + + assert [member.stat_var for member in group.members] == members + assert group.stub_candidates == stubs + assert group.undismissed_candidates == undismissed + assert group.undismissed_rereadable is not stub_complete + assert "Amount_Stub" not in group.family diff --git a/tests/statvar_members/test_root_twins.py b/tests/statvar_members/test_root_twins.py index 04f52a4c..b5756e76 100644 --- a/tests/statvar_members/test_root_twins.py +++ b/tests/statvar_members/test_root_twins.py @@ -8,6 +8,7 @@ _REPROBE_BUDGET, _TWIN_PROBE_BUDGET, MemberRoleDisagreement, + _arcs_unproven, _ProvisionalDenominatorReader, _read_root_twins, enumerate_members, @@ -114,9 +115,11 @@ def test_a_twin_read_missing_a_filtered_arc_settles_only_a_caller_supplied_twin( arcs=_CONTRADICTED_TWIN, drop_always={"Count_Person_Z": "statType"} ) - confirmed, still_unsettled = _read_root_twins( + confirmed, still_unsettled, _ = _read_root_twins( ["Count_Person_Z"], walked=walked, + arcs_by_dcid={}, + arcs_complete=_arcs_unproven, five_tuple=_COUNT, slots={}, graph=reader, @@ -177,9 +180,11 @@ def test_the_twin_pass_adds_a_type_read_for_each_unseen_denominator_target(): arcs[f"Denominator{i:02d}"] = {"typeOf": ("ProvisionalNode",)} reader = FakeMemberArcReader(arcs=arcs) - confirmed, unsettled = _read_root_twins( + confirmed, unsettled, _ = _read_root_twins( twins, walked=frozenset(twins), + arcs_by_dcid={}, + arcs_complete=_arcs_unproven, five_tuple=ask, slots={}, graph=_ProvisionalDenominatorReader(reader), From dd259260f6eec32ec6f3ad9f2da3d1a3b307388a Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sat, 24 Oct 2026 20:37:16 +0200 Subject: [PATCH 2/7] fix(materialise): tell a group's off-cadence members from its empty ones in coverage_below_scope (#120) A group mint named every member the facet election did not count as "No observations were served for dcid:...", whether the member reported the requested years at another frequency, which the figures leave out, or held nothing in the window. `_plan_group_members` now splits the two, and the warning carries them as `off_cadence_members` and `out_of_window_members` (`{count, refs}`) beside `member_count`. The out-of-window case is dropped when the served provenance declares absent members as zero. The stored message states counts only; the presentation store labels the refs in its existing label read and the card names them within its length limit. --- docs/conformance-datasets-api-v1.md | 2 +- docs/datasets-facet-policy.md | 9 +- .../src/assembler/mint/__init__.py | 3 +- .../src/assembler/mint/provenance.py | 68 +++++++++++++-- .../dfo-data-tools/src/materialise/codes.py | 18 +++- .../dfo-data-tools/src/materialise/finish.py | 85 +++++++++++++------ .../src/materialise/generation.py | 41 ++++++--- .../src/materialise/pipeline.py | 13 ++- src/datasets_api/manifest.py | 10 +-- src/datasets_api/serve.py | 3 + src/presentation_store/frame_store.py | 14 ++- tests/assembler/test_mint.py | 67 ++++++++++++++- tests/datasets_api/test_rate_cache_wiring.py | 33 ++++++- tests/datasets_api/test_warning_model.py | 17 ++++ tests/materialise/test_dispatch_cadence.py | 72 ++++++++++++---- tests/presentation_store/test_frame_store.py | 35 ++++++++ 16 files changed, 417 insertions(+), 73 deletions(-) diff --git a/docs/conformance-datasets-api-v1.md b/docs/conformance-datasets-api-v1.md index eb7032ea..6ac3543b 100644 --- a/docs/conformance-datasets-api-v1.md +++ b/docs/conformance-datasets-api-v1.md @@ -148,7 +148,7 @@ Five routes are outside the V1 contract text. The code registry's mappings for ` | C150 | On definition retrieval and its options sibling, `trace` records `served_from`, `head` when the stored record was served, either inside the re-check interval or after a matching probe, and `materialise` when this request rebuilt the frame, on a probe mismatch, past the maximum age, or when a window the definition left open now ends at another year than the stored record read, inside the re-check interval too. The options route reports `served_from` under the same rule; a `materialise` answer records the version and advances the head, as definition retrieval does (C158 for a posted definition) | implemented | | | C151 | `trace` records `confirmed_at`, the UTC timestamp of when the stored head was last confirmed. Definition retrieval records a probe match, so its value moves. The options route confirms no probe match, so a head it serves reports the stored instant. A `done` operation poll reports `served_from: materialise` and the `confirmed_at` its run recorded when it completed. Snapshot retrieval carries neither field, since it reads one stored version with no currency judgement | implemented | | | C76 | `coverage_clamp` marks a window narrowed to what the source holds, paired with `coverage_below_scope` | implemented | | -| C77 | `warnings`: `{code, severity, message}` from the platform code registry, plus `contributing_member_count`, `stat_var_ref`, and `stat_var_label` on `stat_vars_summed` only (C118); a warning scoped to one rendering (`enrichment_unavailable`) rides its artifact entry instead of the top-level list | partial | `enrichment_unavailable` is emitted on the top-level `warnings` list; artifact entries carry no `warnings` member. Issue: none | +| C77 | `warnings`: `{code, severity, message}` from the platform code registry, plus `contributing_member_count`, `stat_var_ref`, and `stat_var_label` on `stat_vars_summed` only (C118), and on a group's `coverage_below_scope` `member_count` with `off_cadence_members` (members reporting the requested years at another frequency, left out of the figures) and `out_of_window_members` (members with no observation in the requested years, absent when the served provenance reads an absent member as zero, the rule the rendered sum reads), each `{count, refs}` with the `dcid:` refs of the first five, while the message counts them and names none. `WarningModel` forbids unknown fields, so a revision without these fields cannot read a version recorded with them; a warning scoped to one rendering (`enrichment_unavailable`) rides its artifact entry instead of the top-level list | partial | `enrichment_unavailable` is emitted on the top-level `warnings` list; artifact entries carry no `warnings` member. Issue: none | | C78 | `artifacts` names exact bytes, the rendering that produced them, size, media type, creation time, tier, and `url`, the artifact's stable download link `/api/datasets/{id}/{content_version}/files/{artifact_digest}`, which never expires | implemented | | | C79 | A mint or definition retrieval returns the one artifact for its rendering (C165), plus its `series-statistics/1` artifact whenever `series_count` is present. A snapshot manifest lists every currently servable listable rendering, working and pinned alike, except an unpinned superseded bundle (C164), plus that same statistics artifact regardless of narrowing. An unnarrowed snapshot manifest also lists the version's canonical frame, rendering `{"format": "frame"}` and media type `application/vnd.apache.parquet`, after the requested renderings and before the statistics artifact. A frame1 document is never listed | implemented | | diff --git a/docs/datasets-facet-policy.md b/docs/datasets-facet-policy.md index 252a730e..533c9da4 100644 --- a/docs/datasets-facet-policy.md +++ b/docs/datasets-facet-policy.md @@ -362,7 +362,14 @@ by facet id and each facet's series count across the members (`materialise.group_facets.group_breadth`), and then the one the earliest member holds. Every member dispatches only its facets at that period. A member holding observations at another period alone dispatches nothing, -and the result carries a `coverage_below_scope` warning naming it. A group +and the result carries a `coverage_below_scope` warning naming it by ref +in a structured field. The warning counts such members apart from those whose +observations all fall outside a range window. The first report the +requested years at another frequency, so the frame leaves their figures +out. The second report nothing in those years, and raise no warning when +the served dataset reads an absent member as zero +(`code_index.members.absent_member_reads_zero`), since they add nothing to +a sum over the requested years. A group chunk fetches the union of its members' dispatch sets, and each member serves only rows from its own (`materialise.columnar.dispatched_rows`). diff --git a/packages/dfo-data-tools/src/assembler/mint/__init__.py b/packages/dfo-data-tools/src/assembler/mint/__init__.py index 2aa14da5..2a8c3323 100644 --- a/packages/dfo-data-tools/src/assembler/mint/__init__.py +++ b/packages/dfo-data-tools/src/assembler/mint/__init__.py @@ -10,7 +10,7 @@ from assembler.mint.builders import complete_window, default_window from assembler.mint.card_description import DescribedRevision, with_card_description from assembler.mint.dispatch import MintedComponent, mint_component, stamp_envelope -from assembler.mint.provenance import DESCRIPTION_PROVENANCE_MEMBERS +from assembler.mint.provenance import DESCRIPTION_PROVENANCE_MEMBERS, warned_member_refs from assembler.mint.selection import selection_facts __all__ = [ @@ -22,5 +22,6 @@ "mint_component", "selection_facts", "stamp_envelope", + "warned_member_refs", "with_card_description", ] diff --git a/packages/dfo-data-tools/src/assembler/mint/provenance.py b/packages/dfo-data-tools/src/assembler/mint/provenance.py index 35e03201..529601c3 100644 --- a/packages/dfo-data-tools/src/assembler/mint/provenance.py +++ b/packages/dfo-data-tools/src/assembler/mint/provenance.py @@ -14,6 +14,8 @@ from dataclasses import dataclass from typing import cast +from dfo_frame_grammar.grammar import STAT_VAR_REF_PREFIX + from assembler.bind import BoundSeries from assembler.limits import LABEL_MAX from assembler.ports import NamedSeries, ResolvedFrame @@ -178,10 +180,40 @@ def _build_citation( return citation_line(retrieved=retrieved, sources=names, processed=processed)[:_CITATION_MAX] -def _translate_warnings(warnings: Sequence[Mapping[str, object]]) -> list[dict[str, object]]: +# Each member field of a group's `coverage_below_scope` warning +# (`datasets_api.manifest_models.WarningModel`), and the words that lead its names on a card. +_MEMBER_FIELDS = ( + ("off_cadence_members", "At a different frequency"), + ("out_of_window_members", "No observations in the requested years"), +) + + +def warned_member_refs(*, manifest: Mapping[str, object]) -> list[str]: + """The `dcid:` refs the member fields of `manifest`'s warnings name, in warning and field + order: the group members a card names by label (`_translate_warnings`).""" + warnings = cast("Sequence[Mapping[str, object]]", manifest.get("warnings") or ()) + return [ + ref + for warning in warnings + for field, _lead in _MEMBER_FIELDS + for ref in cast("Sequence[str]", _members(warning, field).get("refs", ())) + ] + + +def _members(warning: Mapping[str, object], field: str) -> Mapping[str, object]: + return cast("Mapping[str, object]", warning.get(field) or {}) + + +def _translate_warnings( + warnings: Sequence[Mapping[str, object]], *, labels: Mapping[str, str] +) -> list[dict[str, object]]: """Maps manifest warnings to the envelope's closed `{kind, detail}` vocabulary. - Drops `severity` and raises `AssertionError` for an unmapped code.""" + A warning's `detail` is its message, then, for each member field it carries, the field's + lead words and its refs' `labels`, the bare dcid where `labels` holds none + (`_named_members`). The fields share the room the message leaves within + `_WARNING_DETAIL_MAX` equally, and a field's unused share passes to the next. Drops + `severity` and raises `AssertionError` for an unmapped code.""" translated: list[dict[str, object]] = [] for warning in warnings: code = cast(str, warning["code"]) @@ -190,11 +222,37 @@ def _translate_warnings(warnings: Sequence[Mapping[str, object]]) -> list[dict[s f"manifest warning code {code!r} has no entry in mint's own " "envelope-kind translation table (_WARNING_KIND_BY_CODE)" ) - detail = str(warning["message"])[:_WARNING_DETAIL_MAX] - translated.append({"kind": _WARNING_KIND_BY_CODE[code], "detail": detail}) + detail = str(warning["message"]) + present = [ + (lead, members) + for field, lead in _MEMBER_FIELDS + if (members := _members(warning, field)) + ] + for index, (lead, members) in enumerate(present): + names = [ + labels.get(ref, ref.removeprefix(STAT_VAR_REF_PREFIX))[:LABEL_MAX] + for ref in cast("Sequence[str]", members["refs"]) + ] + room = (_WARNING_DETAIL_MAX - len(detail)) // (len(present) - index) + detail += _named_members(lead, names, count=cast(int, members["count"]), budget=room) + translated.append( + {"kind": _WARNING_KIND_BY_CODE[code], "detail": detail[:_WARNING_DETAIL_MAX]} + ) return translated +def _named_members(lead: str, names: Sequence[str], *, count: int, budget: int) -> str: + """`" {lead}: "`, then as many whole `names` as fit within `budget` characters, with + `" and N more"` for the `count` members not named, closed by a full stop. When no name + fits, the count stands for them all, whatever the budget, so the field keeps its count.""" + for shown in range(len(names), 0, -1): + hidden = count - shown + text = f" {lead}: {', '.join(names[:shown])}{f' and {hidden} more' if hidden > 0 else ''}." + if len(text) <= budget: + return text + return f" {lead}: {count} series." + + @dataclass(frozen=True) class LeftOut: """The series a rolling component leaves out, one per display label, and the period they @@ -289,7 +347,7 @@ def _build_provenance( "next_expected": None, "partial_period": False, }, - "warnings": _translate_warnings(warnings), + "warnings": _translate_warnings(warnings, labels=resolved_frame.labels), "generated_at": clock().astimezone(datetime.UTC).isoformat().replace("+00:00", "Z"), "citation": _build_citation( sources=sources, diff --git a/packages/dfo-data-tools/src/materialise/codes.py b/packages/dfo-data-tools/src/materialise/codes.py index b160f583..392eecc9 100644 --- a/packages/dfo-data-tools/src/materialise/codes.py +++ b/packages/dfo-data-tools/src/materialise/codes.py @@ -2,12 +2,20 @@ from __future__ import annotations -from typing import TypedDict +from typing import NotRequired, TypedDict from dfo_platform.codes import CodeCatalogue from dfo_platform.diagnostics import WarningSeverity +class UnservedMembers(TypedDict): + """The group members of one kind that served nothing: how many, and the `dcid:` refs of + the first five in request order.""" + + count: int + refs: list[str] + + class MaterialisationWarning(TypedDict): """One `{code, severity, message}` warning entry, as carried on `MaterialisationResult.warnings` (pipeline.py). `severity` is always one @@ -15,6 +23,11 @@ class MaterialisationWarning(TypedDict): (`dfo_platform.diagnostics.WarningSeverity`), always `"warn"` for the warnings this library raises. + A group's `coverage_below_scope` also carries `member_count`, the series the group + requested, with `off_cadence_members`, those reporting the requested years at another + frequency than the group serves, and `out_of_window_members`, those holding no observation + in the requested years, each present only when it names a member. + This is a `TypedDict`. No shared platform pydantic warning model exists yet to alias. """ @@ -22,6 +35,9 @@ class MaterialisationWarning(TypedDict): code: str severity: WarningSeverity message: str + member_count: NotRequired[int] + off_cadence_members: NotRequired[UnservedMembers] + out_of_window_members: NotRequired[UnservedMembers] MATERIALISE_CODE_CATALOGUE: CodeCatalogue = { diff --git a/packages/dfo-data-tools/src/materialise/finish.py b/packages/dfo-data-tools/src/materialise/finish.py index 0dc1d993..aa58e201 100644 --- a/packages/dfo-data-tools/src/materialise/finish.py +++ b/packages/dfo-data-tools/src/materialise/finish.py @@ -15,7 +15,7 @@ # `pipeline` imports this module during initialisation. `MaterialisationResult` resolves when # `_finish_member` runs, after `pipeline` has loaded. from materialise import pipeline -from materialise.codes import MATERIALISE_CODE_CATALOGUE, MaterialisationWarning +from materialise.codes import MATERIALISE_CODE_CATALOGUE, MaterialisationWarning, UnservedMembers from materialise.content_version import mint_version_from_digest from materialise.convert import ConversionRecord from materialise.dependencies import Dependencies @@ -306,6 +306,7 @@ def _finish_group( enrichment: EnrichmentLookup, dependencies: Dependencies, graph_revision: str | None, + absent_reads_zero: bool = False, ) -> pipeline.MaterialisationResult: """Runs after generation returns. Mints from `outcome.frame_sha256`, computed during the group's single fetch, so hashing rereads no rows. The group's single fetch builds @@ -361,7 +362,7 @@ def _finish_group( content_version=content_version, trace=trace, coverage=outcome.coverage, - warnings=_build_warnings(requests[0], outcome), + warnings=_build_warnings(requests[0], outcome, absent_reads_zero=absent_reads_zero), series_statistics=series_statistics, output_profile=output_profile, conversion_record=None, @@ -377,16 +378,22 @@ def _finish_group( def _build_warnings( - request: MaterialisationRequest, outcome: _MemberOutcome | _GroupOutcome | _PreviewOutcome + request: MaterialisationRequest, + outcome: _MemberOutcome | _GroupOutcome | _PreviewOutcome, + *, + absent_reads_zero: bool = False, ) -> tuple[MaterialisationWarning, ...]: """Adds `empty_result` for zero rows, which suppresses `coverage_below_scope`. Its message says the request's places hold no observations when the facet probe returned no candidate (`_empty_result_warning`). `coverage_below_scope` covers window clamps and unserved bound entities, naming up to five entities, a request binding no place that was served the - countries with data in place of every place, and a group member that served nothing - because it holds no facet at the group's one cadence, or under a range window none with a - point in it (`_GroupOutcome.off_cadence_members`). One `coverage_below_scope` states every - cause that applies. + countries with data in place of every place, and, when the group served rows, its members + that served nothing, which it counts in its message and names by ref in its structured + fields: those at another observation period in the requested years + (`_GroupOutcome.off_cadence_members`), and those holding observations only outside them + (`_GroupOutcome.out_of_window_members`). The second are left out with `absent_reads_zero`, + when the served dataset reads an absent member as zero, since such a member adds nothing + to the requested years. One `coverage_below_scope` states every cause that applies. `currency_unmapped_omitted` applies when any conversion record omitted an entity's rows; an omitted entity the request bound counts as reported there, so `coverage_below_scope` does not name it. `deflator_missing_omitted` applies when any record left out rows for a @@ -432,12 +439,17 @@ def _build_warnings( if entity not in omitted ] countries_fallback = row_count > 0 and not outcome.default_scope.every_place - off_cadence = ( - outcome.off_cadence_members if isinstance(outcome, _GroupOutcome) and row_count else () - ) + off_cadence: Sequence[str] = () + out_of_window: Sequence[str] = () + member_count = 0 + if isinstance(outcome, _GroupOutcome) and row_count: + off_cadence = outcome.off_cadence_members + out_of_window = () if absent_reads_zero else outcome.out_of_window_members + member_count = len(outcome.member_observation_counts) if ( countries_fallback or off_cadence + or out_of_window or coverage_below_scope( outcome.coverage, applied_window=outcome.applied_window, unserved=unserved ) @@ -447,7 +459,8 @@ def _build_warnings( unserved, countries_fallback=countries_fallback, off_cadence=off_cadence, - range_window=request.window is not None and request.window.kind == "range", + out_of_window=out_of_window, + member_count=member_count, ) ) @@ -497,32 +510,56 @@ def _coverage_below_scope_warning( unserved: Sequence[str], *, countries_fallback: bool, - off_cadence: Sequence[str] = (), - range_window: bool = False, + off_cadence: Sequence[str], + out_of_window: Sequence[str], + member_count: int, ) -> MaterialisationWarning: - """Adds up to five unserved dcids to the catalogue detail, then up to five group members - in `off_cadence` that served nothing because they hold no facet at the group's - observation period, or with `range_window` none with a point in the requested years, - then, with `countries_fallback`, that a request binding no place was - served the countries with data because the full list of places with data was too large - or unavailable (`place_scope.DefaultPlaceScope`). Without any the detail stands for a pure - window clamp. Names appear as raw dcids.""" + """Adds up to five unserved entity dcids to the catalogue detail. Then, of the group's + `member_count` series, it counts those in `off_cadence`, which report the requested years + at another frequency than the group serves and are left out, and those in + `out_of_window`, which hold no observation in the requested years. The message names no + member, so a stored version holds no name that can go stale. The warning's + `off_cadence_members` and `out_of_window_members` carry each count and the first five + refs (`UnservedMembers`), beside `member_count`, for a reader to name them. Then, with + `countries_fallback`, it says a request binding no place was served the countries with + data because the full list of places with data was too large or unavailable + (`place_scope.DefaultPlaceScope`). Without any the detail stands for a pure window + clamp.""" _title, detail = MATERIALISE_CODE_CATALOGUE["coverage_below_scope"] message = detail if unserved: message = f"{message} No observations were served for {_capped_names(unserved)}." if off_cadence: message = ( - f"{message} No observations were served for {_capped_names(off_cadence)}, " - "which hold none at the observation period the other members serve" - f"{' in the requested years' if range_window else ''}." + f"{message} {len(off_cadence)} of {member_count} series report at a different " + "frequency and are left out of the figures." + ) + if out_of_window: + message = ( + f"{message} {len(out_of_window)} of {member_count} series hold no observations " + "in the requested years." ) if countries_fallback: message = ( f"{message} No place was named, so the answer covers the countries with data, " "because the full list of places with data was too large or unavailable." ) - return {"code": "coverage_below_scope", "severity": "warn", "message": message} + warning: MaterialisationWarning = { + "code": "coverage_below_scope", + "severity": "warn", + "message": message, + } + if off_cadence or out_of_window: + warning["member_count"] = member_count + if off_cadence: + warning["off_cadence_members"] = _unserved(off_cadence) + if out_of_window: + warning["out_of_window_members"] = _unserved(out_of_window) + return warning + + +def _unserved(refs: Sequence[str]) -> UnservedMembers: + return {"count": len(refs), "refs": list(refs[:_MAX_UNSERVED_NAMED])} def _capped_names(names: Sequence[str]) -> str: diff --git a/packages/dfo-data-tools/src/materialise/generation.py b/packages/dfo-data-tools/src/materialise/generation.py index e8cb6795..f52359ae 100644 --- a/packages/dfo-data-tools/src/materialise/generation.py +++ b/packages/dfo-data-tools/src/materialise/generation.py @@ -72,6 +72,7 @@ fetch_role_bindings, joint_dispatch_facet_ids, joint_matches, + overlaps_window, probe_collection_candidates, probe_facets, probe_from_candidates, @@ -422,8 +423,8 @@ class _GroupOutcome: `None` when no arithmetic ran; recipe IDs are `None` for ref/unbound or native-satisfied units. `default_scope` is the group's default place scope, for members that bind no entity. `series_conversions` holds every - member's converted series' records (`_convert_by_series`). `off_cadence_members` is - `_GroupPlan.off_cadence_members`. `probe_empty` is true when no member's probe returned a + member's converted series' records (`_convert_by_series`). `off_cadence_members` and + `out_of_window_members` are `_GroupPlan`'s. `probe_empty` is true when no member's probe returned a candidate, and the outcome then holds no row and no selection. `provenance_refresh` is `_MemberOutcome.provenance_refresh` for the group's shared selection. `method_periods` maps each `measurementMethod` the rows name to the earliest and latest period its rows @@ -446,6 +447,7 @@ class _GroupOutcome: default_scope: DefaultPlaceScope = NO_DEFAULT_SCOPE series_conversions: dict[SeriesKey, SeriesConversion] = field(default_factory=dict) off_cadence_members: tuple[str, ...] = () + out_of_window_members: tuple[str, ...] = () probe_empty: bool = False provenance_refresh: Mapping[str, ProvenanceRefresh] = field(default_factory=dict) method_periods: Mapping[str, tuple[str, str]] = field(default_factory=dict) @@ -2151,14 +2153,18 @@ def _probe_group_members( @dataclass(frozen=True) class _GroupPlan: """Shared group selections and per-member fetch plans in request order. - `off_cadence_members` names, in request order, each member whose probe jointly matches - the selections and whom the election did not count (`FacetElection.members`): it holds no - facet at the group's cadence, or under a range window none with a point in it, so it - serves nothing.""" + Each member whose probe jointly matches the selections and whom the election did not + count (`FacetElection.members`) serves nothing, and is named, in request order, in one of + two tuples. `off_cadence_members` holds those with a jointly matching facet holding a point + in the requested window (`facets.overlaps_window`, every facet under a window that is not + a range): their observations in those years are at another observation period than the + group's, so the frame leaves them out. `out_of_window_members` holds those with no such + facet: they hold observations only outside the requested years.""" selections: dict[str, FacetSelection] plans: list[_MemberFetchPlan] off_cadence_members: tuple[str, ...] = () + out_of_window_members: tuple[str, ...] = () def _plan_group_members( @@ -2179,8 +2185,8 @@ def _plan_group_members( Each member dispatches only its facets at the group's one cadence (`FacetElection.cadence`, the one serving the most members). A member the election did - not count serves nothing and is named in `off_cadence_members`, for - `finish._build_warnings` to report. + not count serves nothing and is named in `off_cadence_members` or + `out_of_window_members` (`_GroupPlan`), for `finish._build_warnings` to report. """ graph = _planning_graph(requests, graph=graph) election = select_group_facets( @@ -2212,12 +2218,20 @@ def _plan_group_members( ) for index in range(len(requests)) ] - off_cadence_members = tuple( - request.stat_var + window = requests[0].window + # Each uncounted member's dcid, and whether a facet it matches holds a point in the window. + uncounted = [ + (request.stat_var, any(overlaps_window(facet, window=window) for facet in matches)) for index, (request, probe) in enumerate(zip(requests, probes, strict=True)) - if index not in election.members and joint_matches(probe.candidates, selections) + if index not in election.members + and (matches := joint_matches(probe.candidates, selections)) + ] + return _GroupPlan( + selections=selections, + plans=plans, + off_cadence_members=tuple(ref for ref, in_window in uncounted if in_window), + out_of_window_members=tuple(ref for ref, in_window in uncounted if not in_window), ) - return _GroupPlan(selections=selections, plans=plans, off_cadence_members=off_cadence_members) def _empty_group_plan(probes: Sequence[MemberProbe]) -> _GroupPlan: @@ -2582,6 +2596,7 @@ def _assemble_group_outcome( ceiling: int, default_scope: DefaultPlaceScope = NO_DEFAULT_SCOPE, off_cadence_members: tuple[str, ...] = (), + out_of_window_members: tuple[str, ...] = (), probe_empty: bool = False, ) -> _GroupOutcome: """Drives `streamer`'s streaming fetch to completion, then assembles @@ -2651,6 +2666,7 @@ def _assemble_group_outcome( default_scope=default_scope, series_conversions=streamer.series_conversions, off_cadence_members=off_cadence_members, + out_of_window_members=out_of_window_members, probe_empty=probe_empty, method_periods=streamer.method_periods, latest_years=streamer.latest_years, @@ -2758,6 +2774,7 @@ def _generate_group( ceiling=ceiling, default_scope=default_scope_of(probed), off_cadence_members=group_plan.off_cadence_members, + out_of_window_members=group_plan.out_of_window_members, probe_empty=probe_empty, ) return replace(outcome, provenance_refresh=provenance_refresh) diff --git a/packages/dfo-data-tools/src/materialise/pipeline.py b/packages/dfo-data-tools/src/materialise/pipeline.py index b3220d1f..0d2d572c 100644 --- a/packages/dfo-data-tools/src/materialise/pipeline.py +++ b/packages/dfo-data-tools/src/materialise/pipeline.py @@ -255,6 +255,7 @@ def materialise_group( coverage: CoverageIndex | None = None, deadline: float | None = None, progress: RowProgress | None = None, + absent_reads_zero: Callable[[str], bool] | None = None, ) -> MaterialisationResult: """Generate and finish a group. Returns one frame, trace and content version covering every member. @@ -301,6 +302,12 @@ def materialise_group( rows whose deflator the table lacks; when no row of the group survives and some were omitted, the call refuses `currency_unmapped` naming the unnamed entities, else `deflator_missing` naming each entity and year (`refuse_if_every_row_omitted`). + + `absent_reads_zero` says whether a provenance reads an absent member as zero + (`code_index.members.absent_member_reads_zero`), the rule a sum over the frame reads. A + group served from one provenance it holds true for raises no warning for its members + holding no observation in the requested years (`finish._build_warnings`). Without it + every provenance reads unknown. """ if not requests: raise MaterialisationUsageError("materialise_group requires at least one member") @@ -360,6 +367,7 @@ def materialise_group( refuse_if_every_row_omitted( served_rows=outcome.coverage.rows, records=outcome.member_conversion_records.values() ) + served = dispatched_provenances(outcome.selections) return _finish_group( requests, outcome, @@ -367,9 +375,12 @@ def materialise_group( materialiser_build=materialiser_build, capability_version=capability_version, enrichment=enrichment, + absent_reads_zero=( + absent_reads_zero is not None and len(served) == 1 and absent_reads_zero(served[0]) + ), dependencies=build_dependencies( requests, - source_refs=dispatched_provenances(outcome.selections), + source_refs=served, provenance_refresh=outcome.provenance_refresh, conversion_records=outcome.member_conversion_records.values(), latest_years=outcome.latest_years, diff --git a/src/datasets_api/manifest.py b/src/datasets_api/manifest.py index d915b0b7..c03c07d6 100644 --- a/src/datasets_api/manifest.py +++ b/src/datasets_api/manifest.py @@ -126,12 +126,10 @@ def warning_model(warning: Mapping[str, object]) -> WarningModel: - """Converts a complete `{code, severity, message}` warning mapping without reformatting it.""" - return WarningModel( - code=cast(str, warning["code"]), - severity=cast(str, warning["severity"]), - message=cast(str, warning["message"]), - ) + """Converts a complete `{code, severity, message}` warning mapping, with any member fields + a group's `coverage_below_scope` carries (`materialise.codes.MaterialisationWarning`), + without reformatting it.""" + return WarningModel.model_validate(warning) def reshape_warning_models(warnings: Sequence[Mapping[str, object]]) -> tuple[WarningModel, ...]: diff --git a/src/datasets_api/serve.py b/src/datasets_api/serve.py index a55ba0d9..f9e9ba0a 100644 --- a/src/datasets_api/serve.py +++ b/src/datasets_api/serve.py @@ -58,10 +58,12 @@ from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass, field, replace from datetime import datetime +from functools import partial from typing import cast from artifact_store import ArtifactStore, ArtifactStoreError from artifact_store.inventory import InventoryBatch, InventoryEntry +from code_index.members import absent_member_reads_zero from code_index.query import CodeIndex from dataset_store.clock import utc_now from dataset_store.snapshot import ( @@ -610,6 +612,7 @@ def prewrite_definition( precomputed_probe=precomputed_probe, deadline=deadline, progress=progress, + absent_reads_zero=partial(absent_member_reads_zero, code_index), ) except DefaultFacetUndefinedError as exc: if exc.request is not None: diff --git a/src/presentation_store/frame_store.py b/src/presentation_store/frame_store.py index 2ca8a6fa..538ac8a2 100644 --- a/src/presentation_store/frame_store.py +++ b/src/presentation_store/frame_store.py @@ -59,6 +59,10 @@ record not final, so a later load reads the inputs again. A failed input read raises as a failed record read does. +A version's `coverage_below_scope` warning names the group members that served nothing by +ref (`assembler.mint.warned_member_refs`). They hold no row, so their refs join the frame's +label read ahead of its own refs, and a card names them by label. + A frame's continuation (`ResolvedFrame.continuation`, `assembler.frame_view.frame_continuation`) comes from its stored version alone: a dataset version's `trace.continuations`, or the `continuations` a computed version stored when it was published, inherited from the versions it @@ -154,6 +158,7 @@ frame_continuation, ) from assembler.limits import LABEL_MAX, joined_label +from assembler.mint import warned_member_refs from assembler.ports import ( FrameRef, FrameUnavailable, @@ -772,6 +777,7 @@ def _resolve( record=record, composed=_composed(record, stored=stored), inputs=inputs, + warned=warned_member_refs(manifest=document), deadline=deadline, ) continuation = frame_continuation(manifest=document) @@ -836,6 +842,7 @@ def _resolve( record=record, composed=_composed(record, stored=stored), inputs=inputs, + warned=warned_member_refs(manifest=document), deadline=deadline, ) if record_changed and not labels_resolved: @@ -1370,9 +1377,12 @@ def _resolve_labels( record: DescriptionRecord | None, composed: Sequence[ComposedVariable], inputs: Sequence[DescriptionRecord], + warned: Sequence[str], deadline: float, ) -> tuple[dict[str, str], dict[str, str], bool]: - """Resolve the frame's ref labels (`_resolve_ref_labels`), with the refs each of + """Resolve the frame's ref labels (`_resolve_ref_labels`), with the group members its + version's warnings name (`warned`, `assembler.mint.warned_member_refs`) under their + `dcid:` refs, since a member that served nothing has no row, and the refs each of `inputs` names in its title (`datasets_api.description_record.title_named`) and the entities `record`'s steps record on their sides (`bound_entities`) under their bare dcids, then add the name of each variable `record` and `composed` name @@ -1387,7 +1397,7 @@ def _resolve_labels( ) labels, resolved = self._resolve_ref_labels( doc, - leading_stat_vars=_composed_sides(composed), + leading_stat_vars=[*_composed_sides(composed), *warned], worded=sorted(worded), named=frozenset(_named_refs(record, composed=composed)), deadline=deadline, diff --git a/tests/assembler/test_mint.py b/tests/assembler/test_mint.py index 101f0654..79f6a519 100644 --- a/tests/assembler/test_mint.py +++ b/tests/assembler/test_mint.py @@ -10,7 +10,7 @@ from assembler.bind import BoundSeries, bind_all from assembler.errors import BindingInvalid, SchemaInvalid from assembler.limits import LABEL_MAX -from assembler.mint import default_window, mint_component, stamp_envelope +from assembler.mint import default_window, mint_component, stamp_envelope, warned_member_refs from assembler.mint.provenance import _WARNING_KIND_BY_CODE, DESCRIPTION_PROVENANCE_MEMBERS from assembler.ports import FrameRef from assembler.resolve import ResolvedSeries, resolve_selectors @@ -1723,12 +1723,12 @@ def test_freshness_data_through_is_null_when_the_manifest_records_no_window(rele # -- provenance warnings ------------------------------------------------- -def _resolved_frame_with_warnings(warnings): +def _resolved_frame_with_warnings(warnings, *, labels=()): return build_resolved_frame( frame_ref=malaria_ethiopia_frame_ref(), role_keys=("recipient",), rows=malaria_ethiopia_rows(), - labels={"country/ETH": "Ethiopia"}, + labels={"country/ETH": "Ethiopia", **dict(labels)}, manifest={"schema_version": 1, "warnings": list(warnings)}, ) @@ -1802,6 +1802,67 @@ def test_provenance_warnings_translate_independently_and_preserve_order(release) ] +def _long_label(index: int) -> str: + return f"{index}".ljust(LABEL_MAX, "x") + + +_LONG = {f"dcid:Long{i}": _long_label(i) for i in range(10)} + + +@pytest.mark.parametrize( + ("members", "expected"), + [ + pytest.param( + {"off_cadence_members": {"count": 2, "refs": ["dcid:Health", "dcid:Water"]}}, + "Short of scope. At a different frequency: Basic health, Water.", + id="other-frequency", + ), + # A member the labels hold no name for reads as its bare dcid, and the members the + # field counts past its refs as a count. + pytest.param( + {"out_of_window_members": {"count": 7, "refs": ["dcid:Health", "dcid:Roads"]}}, + "Short of scope. No observations in the requested years: " + "Basic health, Roads and 5 more.", + id="out-of-window", + ), + # Long names share the room the message leaves: each field names the whole labels + # that fit and counts the rest, and a field fitting none states its count alone. + pytest.param( + { + "off_cadence_members": {"count": 6, "refs": [f"dcid:Long{i}" for i in range(5)]}, + "out_of_window_members": { + "count": 6, + "refs": [f"dcid:Long{i}" for i in range(5, 10)], + }, + }, + f"Short of scope. At a different frequency: {_long_label(0)} and 5 more. " + "No observations in the requested years: 6 series.", + id="both-long", + ), + ], +) +def test_a_coverage_warning_names_its_members_by_label(release, members, expected): + resolved_frame = _resolved_frame_with_warnings( + [ + { + "code": "coverage_below_scope", + "severity": "warn", + "message": "Short of scope.", + "member_count": 9, + **members, + } + ], + labels={"dcid:Health": "Basic health", "dcid:Water": "Water", **_LONG}, + ) + + provenance = _mint_line_chart_provenance(resolved_frame=resolved_frame, release=release) + + assert provenance["warnings"] == [{"kind": "coverage_gap", "detail": expected}] + assert warned_member_refs(manifest=resolved_frame.manifest) == [ + ref for field in members.values() for ref in field["refs"] + ] + + def test_provenance_warnings_refuses_a_code_absent_from_the_translation_table(release): """Unknown manifest codes raise `AssertionError` before minting an invalid envelope.""" resolved_frame = _resolved_frame_with_warnings( diff --git a/tests/datasets_api/test_rate_cache_wiring.py b/tests/datasets_api/test_rate_cache_wiring.py index c02b1292..3fd15183 100644 --- a/tests/datasets_api/test_rate_cache_wiring.py +++ b/tests/datasets_api/test_rate_cache_wiring.py @@ -1,11 +1,14 @@ """Each door reuses one `RateTableCache` across materialisations and passes it to the -materialise entry point. A spy checks cache identity across definition calls. +materialise entry point. A spy checks cache identity across definition calls, and that the +door passes its code index's absent-observation rule. """ from __future__ import annotations import datasets_api.serve as serve_module from materialise.rate_tables import RateTableCache +from tests.code_index.aid_identity import CRS_PROVENANCE +from tests.code_index.support import INDEX_FIXTURE, load_index from .test_mint_spec_id import _STATVAR_SET_BODY, _spec_id_body from .test_mint_spec_id import _client as _spec_client @@ -36,3 +39,31 @@ def spy(*args, **kwargs): assert recorded[0] is not None assert isinstance(recorded[0], RateTableCache) assert recorded[0] is recorded[1] + + +def test_the_definition_door_reads_absence_as_its_code_index_declares(monkeypatch): + """The group's out-of-window members follow the absent-observation rule the rendered sum + reads (`absent_member_reads_zero`), over the door's own code index, whichever path + resolved the members.""" + recorded: list[object] = [] + real_materialise_group = serve_module.materialise_group + + def spy(*args, **kwargs): + recorded.append(kwargs.get("absent_reads_zero")) + return real_materialise_group(*args, **kwargs) + + monkeypatch.setattr(serve_module, "materialise_group", spy) + + from ._artifact_doubles import FakeArtifactStore + + client = _spec_client(artifact_store=FakeArtifactStore(), code_index=load_index(INDEX_FIXTURE)) + + resp = client.post("/api/datasets", json=_STATVAR_SET_BODY) + assert resp.status_code == 200, resp.text + + [absent_reads_zero] = recorded + assert callable(absent_reads_zero) + assert (absent_reads_zero(CRS_PROVENANCE), absent_reads_zero("provenance/Undeclared")) == ( + True, + False, + ) diff --git a/tests/datasets_api/test_warning_model.py b/tests/datasets_api/test_warning_model.py index 0b52ea81..e4b4a9f6 100644 --- a/tests/datasets_api/test_warning_model.py +++ b/tests/datasets_api/test_warning_model.py @@ -2,6 +2,7 @@ from __future__ import annotations +from datasets_api.manifest import warning_model from datasets_api.manifest_models import WarningModel @@ -53,3 +54,19 @@ def test_a_stored_coverage_warning_with_member_fields_reads_back_unchanged(): } assert WarningModel.model_validate(stored).model_dump(mode="json") == stored + + +def test_a_coverage_warning_keeps_the_members_it_names_through_the_manifest(): + """A group's `coverage_below_scope` reaches the stored version with its member fields, and + a field naming no member stays out of the dump.""" + warning = { + "code": "coverage_below_scope", + "severity": "warn", + "message": "m", + "member_count": 9, + "off_cadence_members": {"count": 6, "refs": [f"dcid:Member{i}" for i in range(5)]}, + } + + dumped = warning_model(warning).model_dump(mode="json") + + assert dumped == warning diff --git a/tests/materialise/test_dispatch_cadence.py b/tests/materialise/test_dispatch_cadence.py index f423377c..8b32ed9a 100644 --- a/tests/materialise/test_dispatch_cadence.py +++ b/tests/materialise/test_dispatch_cadence.py @@ -10,7 +10,7 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Callable, Sequence from dataclasses import replace import httpx @@ -20,6 +20,7 @@ from assembler.frame_view import compute_structural_view from dc_sdmx import SdmxClient from materialise import EnrichmentLookup, MaterialisationRequest, generation, materialise_group +from materialise.codes import MaterialisationWarning from materialise.errors import MixedGrainError from materialise.facets import ( Cadence, @@ -125,11 +126,12 @@ def _materialise( requests: Sequence[MaterialisationRequest], graph: FakeGraphReader, sdmx: SdmxClient, - warnings: dict[str, str] | None = None, + warnings: dict[str, MaterialisationWarning] | None = None, witnesses: set[str] | None = None, + absent_reads_zero: Callable[[str], bool] | None = None, ) -> dict[str, list[str]]: """Each StatVar's served periods, after a default-window bind check over the frame. The - result's warnings go to `warnings`, message by code, and the trace's per-axis + result's warnings go to `warnings`, by code, and the trace's per-axis `dispatch_facet_id`s to `witnesses`, when given.""" result = materialise_group( list(requests), @@ -139,9 +141,10 @@ def _materialise( materialiser_build="build-123", capability_version="cap-v1", member_concurrency=1, + absent_reads_zero=absent_reads_zero, ) if warnings is not None: - warnings.update((warning["code"], warning["message"]) for warning in result.warnings) + warnings.update((warning["code"], warning) for warning in result.warnings) if witnesses is not None: witnesses.update(str(entry["dispatch_facet_id"]) for entry in result.trace.facets) frame = only_chunk(result) @@ -298,7 +301,7 @@ def test_a_fully_bound_group_serves_the_groups_one_cadence_for_every_member(): fetched, ) - warnings: dict[str, str] = {} + warnings: dict[str, MaterialisationWarning] = {} periods = _materialise( [_request(_RATE, **_ALL_BOUND), _request(_YOUTH, **_ALL_BOUND)], graph, sdmx, warnings @@ -333,7 +336,7 @@ def test_a_group_elects_the_cadence_serving_the_most_members(): }, fetched, ) - warnings: dict[str, str] = {} + warnings: dict[str, MaterialisationWarning] = {} periods = _materialise(requests, graph, sdmx, warnings) @@ -382,13 +385,16 @@ def unstated(values: dict[str, float]) -> Facet: ) fetched: list[set[str]] = [] sdmx = _sdmx({(_RATE, _UNSTATED_ID): rate, (_YOUTH, _UNSTATED_ID): youth}, fetched) - warnings: dict[str, str] = {} + warnings: dict[str, MaterialisationWarning] = {} periods = _materialise([_request(_RATE), _request(_YOUTH)], graph, sdmx, warnings) assert fetched == [{_UNSTATED_ID}] assert periods == {f"dcid:{_RATE}": sorted(rate)} - assert f"dcid:{_YOUTH}" in warnings["coverage_below_scope"] + assert warnings["coverage_below_scope"].get("off_cadence_members") == { + "count": 1, + "refs": [f"dcid:{_YOUTH}"], + } def test_a_chunk_serves_nothing_for_a_member_whose_populated_probe_dispatches_no_facet(): @@ -712,19 +718,40 @@ def test_a_groups_cadence_ranks_over_its_members_matching_facets(declares, candi _EARLY_WINDOW = RequestedWindow(kind="range", start_year=2012, end_year=2015) -def test_a_member_holding_the_groups_period_only_before_the_window_is_reported(): +_MEMBER_FIELDS = ("member_count", "off_cadence_members", "out_of_window_members") + + +@pytest.mark.parametrize( + ("youth_in_window", "reads_zero", "fields"), + [ + # Its quarters fall in the window, so the figures leave out what it holds there. + pytest.param(True, False, {"off_cadence_members"}, id="other-frequency"), + pytest.param(True, True, {"off_cadence_members"}, id="other-frequency-reads-zero"), + # It holds nothing in the window, a gap the dataset may read as zero. + pytest.param(False, False, {"out_of_window_members"}, id="out-of-window"), + pytest.param(False, True, None, id="out-of-window-reads-zero"), + ], +) +def test_a_member_holding_the_groups_period_only_before_the_window_is_reported( + youth_in_window, reads_zero, fields +): # Two members hold the annual facet in the window. The youth rate holds it only before - # the window and holds quarters in it, so the group serves years and the youth rate, - # though it dispatches the annual facet, serves nothing and is named. + # the window, so the group serves years and the youth rate, though it dispatches the + # annual facet, serves nothing. Holding quarters in the window, it reports there at + # another frequency; holding none, it reports nothing there, and a dataset reading an + # absent member as zero raises no warning for it. in_window = {str(year): 1.0 for year in range(2012, 2016)} early = {str(year): 1.0 for year in range(2000, 2006)} quarters = {f"{year}-{month:02d}": 1.0 for year in range(2012, 2016) for month in (1, 4, 7, 10)} + youth = [_facet(_ANNUAL_ID, early)] + if youth_in_window: + youth.insert(0, _facet(_QUARTERLY_ID, quarters)) adult = f"{_RATE}_Adult" graph = FakeGraphReader( facets={ (_RATE, _KENYA): [_facet(_ANNUAL_ID, in_window)], (adult, _KENYA): [_facet(_ANNUAL_ID, in_window)], - (_YOUTH, _KENYA): [_facet(_QUARTERLY_ID, quarters), _facet(_ANNUAL_ID, early)], + (_YOUTH, _KENYA): youth, } ) sdmx = _sdmx( @@ -736,15 +763,30 @@ def test_a_member_holding_the_groups_period_only_before_the_window_is_reported() }, [], ) - warnings: dict[str, str] = {} requests = [ replace(_request(stat_var), window=_EARLY_WINDOW) for stat_var in (_RATE, _YOUTH, adult) ] + warnings: dict[str, MaterialisationWarning] = {} + read: list[str] = [] - periods = _materialise(requests, graph, sdmx, warnings) + def absent_reads_zero(provenance: str) -> bool: + read.append(provenance) + return reads_zero + + periods = _materialise(requests, graph, sdmx, warnings, absent_reads_zero=absent_reads_zero) assert periods == {f"dcid:{_RATE}": sorted(in_window), f"dcid:{adult}": sorted(in_window)} - assert f"dcid:{_YOUTH}" in warnings["coverage_below_scope"] + # The rule is the served provenance's. + assert read == ["undata/p/ILO"] + if fields is None: + assert "coverage_below_scope" not in warnings + else: + warning = warnings["coverage_below_scope"] + unserved = {"count": 1, "refs": [f"dcid:{_YOUTH}"]} + assert {field: warning[field] for field in _MEMBER_FIELDS if field in warning} == { + "member_count": 3, + **dict.fromkeys(fields, unserved), + } @pytest.mark.parametrize("month_first", [True, False]) diff --git a/tests/presentation_store/test_frame_store.py b/tests/presentation_store/test_frame_store.py index edc296f5..f93dae74 100644 --- a/tests/presentation_store/test_frame_store.py +++ b/tests/presentation_store/test_frame_store.py @@ -789,6 +789,41 @@ def test_load_resolves_a_frame_with_view_manifest_and_labels(bind_catalogue): ) +def test_load_labels_the_group_members_a_coverage_warning_names(): + """A member that served nothing holds no row, so the refs a `coverage_below_scope` + warning names join the frame's one label batch.""" + snapshots, artifacts, frame_ref, _digest = _happy_path_fixture() + version = snapshots.versions[(frame_ref.spec_id, frame_ref.content_version)] + warning = { + "code": "coverage_below_scope", + "severity": "warn", + "message": "Short of scope.", + "member_count": 3, + "off_cadence_members": {"count": 1, "refs": ["dcid:YouthRate"]}, + "out_of_window_members": {"count": 1, "refs": ["dcid:AdultRate"]}, + } + snapshots.versions[(frame_ref.spec_id, frame_ref.content_version)] = replace( + version, facts=replace(version.facts, warnings=[warning]) + ) + graph = _FakeGraphReader( + labels={ + "MalariaGrantDisbursements": "Malaria Grant Disbursements", + "YouthRate": "Youth rate", + "AdultRate": "Adult rate", + } + ) + store = _store(snapshots=snapshots, artifacts=artifacts, graph=graph) + + resolved = store.load(frame_ref=frame_ref) + + assert (resolved.labels["dcid:YouthRate"], resolved.labels["dcid:AdultRate"]) == ( + "Youth rate", + "Adult rate", + ) + assert graph.calls == 1 + assert {"YouthRate", "AdultRate"} <= set(graph.node_arcs_many_calls[0]) + + _WDI_GDP = "worldBank/NY_GDP_MKTP_CD" From 229a6aef162baa13a26307353e76fe6f60acc1ec Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sat, 24 Oct 2026 21:16:10 +0200 Subject: [PATCH 3/7] fix(graph): give every node one display name on every surface (#124) A node holding several `name` values was labelled by position, and two surfaces picked different positions: the node door took the first name that differs from the dcid ("United States"), the frame store the last one ("United States of America"). The graph returns names in the order they were stored, so either pick depended on how the graph was imported. `dc_graph.display_name` takes the greatest name by code point, which matches the label the dev mixer records on every recorded multi-name node, and `label_or_self_name` also accepts a dcid when that is a node's only name. Every site that turns names into a label calls one of them, Search's open-axis value labels included, and `alternateName` never becomes a label. --- .../dfo-data-tools/src/code_index/export.py | 22 ++- .../dfo-data-tools/src/dc_graph/__init__.py | 4 + .../dfo-data-tools/src/dc_graph/client.py | 54 +++--- .../src/entity_resolution/resolver.py | 10 +- .../src/materialise/enrichment.py | 23 ++- .../src/origin_catalogue/card_facts.py | 13 +- .../src/origin_catalogue/families.py | 9 +- .../src/origin_catalogue/graph.py | 10 +- .../src/origin_catalogue/job.py | 4 +- .../src/origin_catalogue/node_reads.py | 15 +- scripts/export_graph_entities.py | 14 +- src/graph_api/card_definition.py | 5 +- src/graph_api/card_reads.py | 6 +- src/graph_api/doors/members.py | 11 +- src/graph_api/doors/node.py | 5 +- src/graph_api/refs.py | 8 +- src/presentation_store/frame_store.py | 9 +- src/search/catalogue/job.py | 2 +- src/search/engine/quantity.py | 2 +- src/search/engine/region/alternate_guard.py | 20 +- .../engine/region/candidate_grounding.py | 1 + src/search/engine/region/phase_answer.py | 4 +- src/search/engine/region/phase_rank_bind.py | 18 +- src/search/engine/region/promote_ground.py | 1 + src/search/engine/region/promote_shape.py | 14 +- src/search/engine/region/state.py | 3 +- src/search/engine/related.py | 4 +- src/search/engine/retrieval/bound_values.py | 35 ++-- src/search/engine/retrieval/derive.py | 13 +- src/search/engine/retrieval/facts.py | 27 +-- src/search/engine/retrieval/member_facts.py | 15 +- src/search/engine/retrieval/standard.py | 2 +- .../engine/unit/conversion_catalogue.py | 11 +- tests/dc_graph/test_client_labels.py | 22 +++ tests/engine/test_admissibility_table.py | 13 +- tests/engine/test_arc_tables_serving.py | 16 +- tests/engine/test_open_axis_facts.py | 61 +++--- tests/engine/test_unit_conversion_offer.py | 8 +- .../requests/test_dataset_resubmit.json | 180 +++++++++--------- .../test_resubmit_place_scope_default.json | 2 +- tests/graph_api/test_members_door.py | 28 ++- tests/graph_api/test_node_blocks.py | 18 ++ tests/origin_catalogue/test_arc_reads.py | 8 +- tests/origin_catalogue/test_graph.py | 2 +- tests/presentation_store/test_frame_store.py | 19 ++ 45 files changed, 463 insertions(+), 308 deletions(-) diff --git a/packages/dfo-data-tools/src/code_index/export.py b/packages/dfo-data-tools/src/code_index/export.py index e4aa03c4..63e3d674 100644 --- a/packages/dfo-data-tools/src/code_index/export.py +++ b/packages/dfo-data-tools/src/code_index/export.py @@ -5,7 +5,7 @@ document: provenances and parties from the coverage it joins, and the stamp. `graph_release.dfo` builds a release's code index part with it. `index_bytes` is the one serialisation of an index document, so the same document gives the same bytes. Nothing here -imports `dc_graph`: a reader supplies the graph reads, and the two names the export needs from +imports `dc_graph`: a reader supplies the graph reads, and the names the export needs from `dc_graph` are imported where they are used. """ @@ -377,7 +377,7 @@ def export_parties( "dcid": dcid, "kind": kind, "party_class": _party_class(kind, private=dcid in private), - "label": _first_or_none(party_arcs.get("name", ())), + "label": _graph_label(dcid, party_arcs), "totals": None if total is None else total.group, "residuals": [] if total is None else sorted(total.residuals), "sums_members": True if total is None else total.sums_members, @@ -538,6 +538,20 @@ def node_arcs_batch(self, dcids: list[str]) -> Mapping[str, dict | None]: ... +def _graph_label(dcid: str, arcs: Mapping[str, tuple[str, ...]]) -> str | None: + """The node's label as `GraphClient.node_label` gives it (`dc_graph.label_or_self_name`).""" + from dc_graph import label_or_self_name + + return label_or_self_name(arcs.get("name", ()), dcid) + + +def _required_name(dcid: str, arcs: Mapping[str, tuple[str, ...]]) -> str: + name = _graph_label(dcid, arcs) + if name is None: + raise ExportRefused(f"{dcid} carries no name arc -- cannot export a row missing it") + return name + + def _first_or_none(values: tuple[str, ...]) -> str | None: return values[0] if values else None @@ -554,7 +568,7 @@ def _fetch_arcs_many( def _build_code_row(dcid: str, code_system: str, arcs: dict[str, tuple[str, ...]]) -> dict: code = _single_value(dcid, "classificationCode", arcs.get("classificationCode", ())) - name = _single_value(dcid, "name", arcs.get("name", ())) + name = _required_name(dcid, arcs) label = strip_system_suffix(name) synonyms = sorted( {v for v in arcs.get("alternateName", ()) if v and v != label} @@ -1746,7 +1760,7 @@ def export_measure_values( f"{dcid} is a {axis.axis!r} measure value an exported StatVar row carries " "or the export declares, but the graph holds no node for it" ) - name = _single_value(dcid, "name", arcs.get("name", ())) + name = _required_name(dcid, arcs) overrides = MEASURE_VALUE_TEXT.get(dcid, MeasureValueText()) label = overrides.label if overrides.label is not None else strip_system_suffix(name) synonyms = sorted( diff --git a/packages/dfo-data-tools/src/dc_graph/__init__.py b/packages/dfo-data-tools/src/dc_graph/__init__.py index d1e0385c..319067d1 100644 --- a/packages/dfo-data-tools/src/dc_graph/__init__.py +++ b/packages/dfo-data-tools/src/dc_graph/__init__.py @@ -18,8 +18,10 @@ SubjectPage, SubjectsCoverageFlag, declared_role_refresh_deadline, + display_name, earliest_date, is_front_door_refusal, + label_or_self_name, latest_date, parse_arcs, reject_unsafe_filter_token, @@ -62,8 +64,10 @@ "SubjectsCoverageFlag", "declared_role_refresh_deadline", "dfo_visible", + "display_name", "earliest_date", "is_front_door_refusal", + "label_or_self_name", "latest_date", "mixer_visible", "parse_arcs", diff --git a/packages/dfo-data-tools/src/dc_graph/client.py b/packages/dfo-data-tools/src/dc_graph/client.py index 83319616..7c4cb9ef 100644 --- a/packages/dfo-data-tools/src/dc_graph/client.py +++ b/packages/dfo-data-tools/src/dc_graph/client.py @@ -23,7 +23,7 @@ import re import threading import time -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, as_completed, wait from dataclasses import dataclass, replace from functools import partial @@ -738,9 +738,9 @@ def _await_retry(self, attempt: int, deadline: float | None) -> bool: return True def node_label(self, dcid: str) -> str | None: - """Return the last `->name` value or `None` for an absent or empty node. It shares the - optional cache with `node_labels_batch`. A node the graph answered without a name caches as - `None`; an unanswered dcid caches nothing. + """Return the node's `label_or_self_name`, or `None` for an absent or unnamed node. It + shares the optional cache with `node_labels_batch`. A node the graph answered without a + name caches as `None`; an unanswered dcid caches nothing. """ stamp = self._generation.stamp() return self._cached_single_arc( @@ -749,7 +749,7 @@ def node_label(self, dcid: str) -> str | None: field="value", cache=self._label_cache, counters=self._label_counters, - pick=_pick_last_truthy, + pick=label_or_self_name, stamp=stamp, ) @@ -776,7 +776,7 @@ def _cached_single_arc( field: str, cache: cachetools.TLRUCache | None, counters: _CacheCounters, - pick: Callable[[list[str]], str | None], + pick: Callable[[list[str], str], str | None], stamp: GenerationStamp, ) -> str | None: cache_key = (stamp.key, dcid) @@ -794,15 +794,16 @@ def _cached_single_arc( if arcs: arc_name = arc_property.removeprefix("->") values = [n.get(field) for n in arcs.get(arc_name, {}).get("nodes", [])] - result = pick(values) + result = pick(values, dcid) if dcid in data and cache is not None: with self._cache_lock: cache[cache_key] = result return result def node_labels_batch(self, dcids: list[str]) -> dict[str, str]: - """Fetch labels in chunks of at most `single_arc_batch_chunk_size`. Omit dcids with no - `->name` arc. Continuations trigger chunk halving, and results share `node_label`'s cache. + """Fetch each node's `label_or_self_name` in chunks of at most + `single_arc_batch_chunk_size`. Omit dcids with none. Continuations trigger chunk halving, + and results share `node_label`'s cache. """ return self._single_arc_batch( dcids, @@ -811,7 +812,7 @@ def node_labels_batch(self, dcids: list[str]) -> dict[str, str]: field="value", cache=self._label_cache, counters=self._label_counters, - pick=_pick_last_truthy, + pick=label_or_self_name, ) def node_types_batch(self, dcids: list[str]) -> dict[str, str]: @@ -837,7 +838,7 @@ def _single_arc_batch( field: str, cache: cachetools.TLRUCache | None, counters: _CacheCounters, - pick: Callable[[list[str]], str | None], + pick: Callable[[list[str], str], str | None], ) -> dict[str, str]: """Fetch uncached dcids in chunks and merge hits with results in first-occurrence input order. A dcid the graph answered without a value is omitted from the result, and caches as @@ -885,7 +886,7 @@ def _single_arc_batch( for n in node_data.get("arcs", {}).get(arc_name, {}).get("nodes", []) if n.get(field) ] - picked = pick(values) + picked = pick(values, dcid) fetched[dcid] = picked # An empty entry from a multi-dcid chunk may depend on the chunk's composition, so # only a dcid read alone is trusted empty. @@ -2308,16 +2309,7 @@ def clear(self) -> None: self._blocked_observation_entities = ("", frozenset()) -def _pick_label(values: list[str]) -> str: - return values[-1] - - -def _pick_last_truthy(values: list[str]) -> str | None: - truthy = [v for v in values if v] - return _pick_label(truthy) if truthy else None - - -def _pick_first(values: list[str]) -> str | None: +def _pick_first(values: list[str], dcid: str) -> str | None: return values[0] if values else None @@ -2382,6 +2374,24 @@ def resolved_value(entry: dict) -> str | None: } +def display_name(names: Iterable[str], dcid: str) -> str | None: + """The one display name of node `dcid` among its `name` values: the greatest by code point, + `None` where it has none. The graph keys arcs on provenance, so a node can carry several names + in an order the read does not fix; taking the greatest gives the same name whatever the order. + A name equal to the dcid, which some imports assert, an empty name, and `UNRESOLVED_NODE` are + no name.""" + return max( + (name for name in names if name and name not in (dcid, UNRESOLVED_NODE)), default=None + ) + + +def label_or_self_name(names: Collection[str], dcid: str) -> str | None: + """The `display_name` of node `dcid`, or the dcid itself for a node named only by its own + dcid. `node_label` answers this, and `exists_batch` reads the self-name as evidence the + node exists.""" + return display_name(names, dcid) or (dcid if dcid in names else None) + + def _arc_spec(properties: Sequence[str]) -> str: """`"->prop"` for one property, `"->[p1,p2,...]"` for several.""" for prop in properties: diff --git a/packages/dfo-data-tools/src/entity_resolution/resolver.py b/packages/dfo-data-tools/src/entity_resolution/resolver.py index ae6de9db..6ade6aee 100644 --- a/packages/dfo-data-tools/src/entity_resolution/resolver.py +++ b/packages/dfo-data-tools/src/entity_resolution/resolver.py @@ -43,6 +43,7 @@ from resolvekit import ResolutionContext, ResolutionStatus, Resolver from resolvekit.core.model.result import CandidateSummary +from dc_graph import label_or_self_name from dc_graph.errors import GraphDeadlineExceededError from dfo_platform.executor import run_in_executor_with_context from entity_resolution import pack @@ -185,7 +186,7 @@ def _confirmed_name( """The candidate's pack name, or its graph name for a code-only row, which the pack names by its dcid.""" name = _candidate_name(candidate) - return _fallback_name(arcs=arcs, text=name) if name == dcid else name + return _fallback_name(dcid=dcid, arcs=arcs, text=name) if name == dcid else name def _unconfirmed_kind( @@ -423,9 +424,8 @@ def _entity_from_confirmation( return Entity(dcid=dcid, name=name, kind=kind, graph_type=graph_type) -def _fallback_name(*, arcs: dict[str, tuple[str, ...]], text: str) -> str: - names = arcs.get("name", ()) - return names[0] if names else text +def _fallback_name(*, dcid: str, arcs: dict[str, tuple[str, ...]], text: str) -> str: + return label_or_self_name(arcs.get("name", ()), dcid) or text def _admits(kinds: frozenset[EntityKind] | None, entity: Entity) -> bool: @@ -892,7 +892,7 @@ def _assemble_fallback( entity = self._confirmed( dcid=dcid, arcs=arcs, - name=_fallback_name(arcs=arcs, text=text), + name=_fallback_name(dcid=dcid, arcs=arcs, text=text), code_index=code_index, ) candidates.append((entity, Matched(text=text, matched_as=matched_as))) diff --git a/packages/dfo-data-tools/src/materialise/enrichment.py b/packages/dfo-data-tools/src/materialise/enrichment.py index 6e34b917..a8f6d252 100644 --- a/packages/dfo-data-tools/src/materialise/enrichment.py +++ b/packages/dfo-data-tools/src/materialise/enrichment.py @@ -14,6 +14,7 @@ from collections.abc import Callable, Collection, Iterable, Mapping from code_index.types import PartyKind, PartyRow +from dc_graph import label_or_self_name from dfo_platform.roles import role_key_axis from graph_release.context import served_origin_key from materialise.graph import GraphReader @@ -50,6 +51,7 @@ ENRICHMENT_FIELDS: tuple[str, ...] = (*ENRICHMENT_PROPERTIES, PARTY_KIND_FIELD) """Every field a lookup can return.""" +_NAME_PROPERTY = "name" _TYPE_PROPERTY = "typeOf" _PROPERTIES = tuple(ENRICHMENT_PROPERTIES.values()) @@ -116,7 +118,7 @@ def lookup( deadline=deadline, ) resolved = { - ref: {} if (node := fetched.get(ref)) is None else _node_fields(node) + ref: {} if (node := fetched.get(ref)) is None else _node_fields(ref, node) for ref in missing } self._store(resolved) @@ -167,15 +169,22 @@ def _with_party_kind( return {**fields, PARTY_KIND_FIELD: kind} -def _node_fields(node: Mapping[str, tuple[str, ...]]) -> dict[str, str]: - """Each graph field's first value, except `type`: every `typeOf` value of the place, sorted - and comma-joined (`AdministrativeArea1,Country,EurostatNUTS2`), since a place can hold - several levels at once.""" +def _node_fields(dcid: str, node: Mapping[str, tuple[str, ...]]) -> dict[str, str]: + """Each graph field's first value, except `label` and `type`. `label` is the node's + `label_or_self_name`. `type` is every `typeOf` value of the place, sorted and comma-joined + (`AdministrativeArea1,Country,EurostatNUTS2`), since a place can hold several levels at + once.""" fields: dict[str, str] = {} for property_, field in _FIELDS_BY_PROPERTY.items(): values = node.get(property_, ()) - if values: - fields[field] = ",".join(sorted(values)) if property_ == _TYPE_PROPERTY else values[0] + if property_ == _NAME_PROPERTY: + value = label_or_self_name(values, dcid) + elif property_ == _TYPE_PROPERTY: + value = ",".join(sorted(values)) + else: + value = values[0] if values else None + if value: + fields[field] = value return fields diff --git a/packages/dfo-data-tools/src/origin_catalogue/card_facts.py b/packages/dfo-data-tools/src/origin_catalogue/card_facts.py index 8d6f6616..39271775 100644 --- a/packages/dfo-data-tools/src/origin_catalogue/card_facts.py +++ b/packages/dfo-data-tools/src/origin_catalogue/card_facts.py @@ -34,7 +34,7 @@ from collections.abc import Mapping, Sequence -from dc_graph import UNRESOLVED_NODE +from dc_graph import UNRESOLVED_NODE, display_name from origin_catalogue.catalogue import CardFacts, Licence, SourceDocumentation DATASET_TYPE = "Dataset" @@ -98,14 +98,9 @@ def without_unresolved( def node_label(arcs: Arcs | None, dcid: str) -> str | None: - """The first non-empty `name` other than the node's own dcid, which some - imports also assert as a name.""" - if arcs is None: - return None - for value in arcs.get("name", ()): - if value and value != dcid: - return value - return None + """The `display_name` of node `dcid` among its arcs `arcs`, `None` for a node + the graph does not hold.""" + return None if arcs is None else display_name(arcs.get("name", ()), dcid) def first_dataset_parent[A: Arcs]( diff --git a/packages/dfo-data-tools/src/origin_catalogue/families.py b/packages/dfo-data-tools/src/origin_catalogue/families.py index 025a7c26..70a5b605 100644 --- a/packages/dfo-data-tools/src/origin_catalogue/families.py +++ b/packages/dfo-data-tools/src/origin_catalogue/families.py @@ -30,6 +30,7 @@ from dataclasses import dataclass, field from typing import Final, NamedTuple +from dc_graph import label_or_self_name from origin_catalogue.un_mirror import UnArc, UnMirror, import_added _PROVISIONAL_TYPES: Final[tuple[str, ...]] = ("ProvisionalNode",) @@ -123,7 +124,7 @@ def base_without( `dropped` variable belongs to, summarised again from its other members in table order, or None when it has none. Families no dropped variable belongs to are left out.""" gone = {dcid for dcid in dropped if variables.get(dcid)} - keys = {_VariableFacts.read(variables[dcid]).key for dcid in gone} + keys = {_VariableFacts.read(dcid, variables[dcid]).key for dcid in gone} resummarised: dict[FamilyKey, FamilySummary | None] = {} for key in keys: kept = [dcid for dcid in summaries[key].members if dcid not in gone] @@ -202,7 +203,7 @@ def summarise( for dcid, arcs in variables: if not arcs: continue - facts = _VariableFacts.read(arcs) + facts = _VariableFacts.read(dcid, arcs) builder = builders.get(facts.key) if builder is None: builder = builders[facts.key] = _Builder() @@ -222,7 +223,7 @@ class _VariableFacts: name: str | None @staticmethod - def read(arcs: tuple[UnArc, ...]) -> _VariableFacts: + def read(dcid: str, arcs: tuple[UnArc, ...]) -> _VariableFacts: by_property: dict[str, list[UnArc]] = {} for arc in arcs: by_property.setdefault(arc.property, []).append(arc) @@ -275,7 +276,7 @@ def provisional(property: str) -> bool: key=key, constrained=bool(declared), constraints=constraints, - name=names[-1] if names else None, + name=label_or_self_name(names, dcid), ) diff --git a/packages/dfo-data-tools/src/origin_catalogue/graph.py b/packages/dfo-data-tools/src/origin_catalogue/graph.py index 56f0c4b3..c2bc8700 100644 --- a/packages/dfo-data-tools/src/origin_catalogue/graph.py +++ b/packages/dfo-data-tools/src/origin_catalogue/graph.py @@ -73,6 +73,7 @@ ReadErrorFactory, SubjectPage, SubjectsCoverageFlag, + label_or_self_name, parse_arcs, ) from graph_generation import ( @@ -258,13 +259,14 @@ def _table_names(catalogue: Catalogue, dcid: str) -> tuple[str, ...] | _Unmirror def _table_label(catalogue: Catalogue, dcid: str) -> str | _Unmirrored: - """A dcid's label as `GraphClient.node_label` picks it: a held node's last `name` - value, the label the arc tables read for a node an arc of theirs names, or the name - a UN arc naming the node carries.""" + """A dcid's label as `GraphClient.node_label` picks it: a held node's + `label_or_self_name`, the label the arc tables read for a node an arc of theirs names, or + the name a UN arc naming the node carries.""" arcs = _node_arcs(catalogue, dcid) if arcs is not None: names = [arc.value for arc in arcs if arc.property == "name" and arc.value] - return names[-1] if names else _OMITTED + label = label_or_self_name(names, dcid) + return _OMITTED if label is None else label if _absent(catalogue, dcid): return _OMITTED if is_un_dcid(dcid): diff --git a/packages/dfo-data-tools/src/origin_catalogue/job.py b/packages/dfo-data-tools/src/origin_catalogue/job.py index fa01b54f..27a8c8da 100644 --- a/packages/dfo-data-tools/src/origin_catalogue/job.py +++ b/packages/dfo-data-tools/src/origin_catalogue/job.py @@ -1042,7 +1042,9 @@ def _dfo_rows( ), ) ) - label = entry.label if entry is not None else _first(names.get(source), "name") or source + label = ( + entry.label if entry is not None else node_label(names.get(source), source) or source + ) dataset = _dataset(arcs.get(provenance), parents) row = _row( provenance, diff --git a/packages/dfo-data-tools/src/origin_catalogue/node_reads.py b/packages/dfo-data-tools/src/origin_catalogue/node_reads.py index 4fe314b5..1964310b 100644 --- a/packages/dfo-data-tools/src/origin_catalogue/node_reads.py +++ b/packages/dfo-data-tools/src/origin_catalogue/node_reads.py @@ -37,7 +37,7 @@ from dataclasses import dataclass, field from typing import Final, Literal, Protocol -from dc_graph import FrontDoorRefused, GraphReadError +from dc_graph import FrontDoorRefused, GraphReadError, label_or_self_name READ_CONCURRENCY: Final[int] = 4 """The most requests `iter_node_arcs` and `iter_node_labels` keep in flight.""" @@ -349,12 +349,13 @@ def _answered_arcs(answer: Mapping | None) -> dict[str, list[dict]]: def _labels(run: NodeRun, nodes: Sequence[str]) -> dict[str, str | None]: """Each node's label from its `->name` arcs (`read_arcs`), as - `dc_graph.GraphClient.node_label` picks it: the last non-empty `name` - value, `None` for a node a read naming it alone answers with none.""" + `dc_graph.GraphClient.node_label` picks it (`label_or_self_name`), `None` + for a node a read naming it alone answers with none.""" found: dict[str, str | None] = {} for node, arcs in read_arcs(run, nodes, "->name").items(): - values = [value for entry in arcs.get("name", ()) if (value := entry.get("value"))] - found[node] = run.string(values[-1]) if values else None + names = [value for entry in arcs.get("name", ()) if (value := entry.get("value"))] + label = label_or_self_name(names, node) + found[node] = None if label is None else run.string(label) return found @@ -398,8 +399,8 @@ def iter_node_labels( graph: NodeGraph, nodes: Sequence[str], *, deadline: float | None = None ) -> Iterator[tuple[str, str | None]]: """Each of `nodes` with the label `dc_graph.GraphClient.node_label` - gives it, the last non-empty value of its `->name` arcs, or `None` for - a node with no name. Nodes are read `_LABEL_NODES_PER_REQUEST` a + gives it (`dc_graph.label_or_self_name` over its `->name` arcs), or + `None` for a node with no name. Nodes are read `_LABEL_NODES_PER_REQUEST` a request, as `node_labels_batch` reads them, `READ_CONCURRENCY` reads in flight, by `read_arcs`' rules, so a node a read naming several leaves out or answers with no name is read again alone. Nodes come read by diff --git a/scripts/export_graph_entities.py b/scripts/export_graph_entities.py index 0e7b16f5..fa763b5c 100644 --- a/scripts/export_graph_entities.py +++ b/scripts/export_graph_entities.py @@ -29,7 +29,7 @@ from code_index.query import CodeIndex from code_index.shared import get_index -from dc_graph import GraphClient +from dc_graph import GraphClient, label_or_self_name from dfo_platform.cloud_run_auth import cloud_run_auth from graph_release.context import bound_served_release from graph_release.service import get_release_service @@ -105,10 +105,12 @@ def _graph_base_url() -> str: return base -def _single_name(values: list[str]) -> str: - if not values: +def _single_name(dcid: str, values: tuple[str, ...]) -> str: + """The node's label as `GraphClient.node_label` gives it (`label_or_self_name`).""" + name = label_or_self_name(values, dcid) + if name is None: raise SystemExit("a node's own name property is empty -- cannot export a nameless entity") - return values[0] + return name def _split_dac_codes(values: list[str]) -> tuple[str | None, list[str]]: @@ -194,7 +196,7 @@ def export_entities( arcs = arcs_by_dcid.get(dcid) if arcs is None: raise SystemExit(f"declared total {dcid} is absent from the graph") - graph_name = _single_name(list(arcs.get("name", ()))) + graph_name = _single_name(dcid, arcs.get("name", ())) if graph_name != label: raise SystemExit( f"declared total {dcid} has graph name {graph_name!r}, expected {label!r}" @@ -231,7 +233,7 @@ def export_entities( records.append( { "dcid": dcid, - "name": _single_name(list(arcs.get("name", ()))), + "name": _single_name(dcid, arcs.get("name", ())), "kind": kind_by_dcid[dcid], "aliases": list(dict.fromkeys([*aliases, *other_codes])), "dac_code": dac_code, diff --git a/src/graph_api/card_definition.py b/src/graph_api/card_definition.py index b8c3640d..93121070 100644 --- a/src/graph_api/card_definition.py +++ b/src/graph_api/card_definition.py @@ -39,10 +39,11 @@ from code_index.labels import compose_measure_label from code_index.query import CodeIndex from code_index.types import MeasureIndex, label_for_pairs +from dc_graph import display_name from description.world_bank import WORLD_BANK_NAMESPACE, world_bank_definition from dfo_platform.wire import ResponseModel from graph_api.reader import GraphNodeReader, NodeArcsByDcid -from graph_api.refs import NodeRef, first_name, first_value +from graph_api.refs import NodeRef, first_value from materialise.graph import GraphReadError from materialise.roles import DegenerateObservationProperties, declared_roles @@ -498,4 +499,4 @@ def _variable_prose(dcid: str, description: str | None) -> str | None: def _name(arcs: Arcs, dcid: str) -> str | None: - return first_name(arcs.get("name", ()), dcid) + return display_name(arcs.get("name", ()), dcid) diff --git a/src/graph_api/card_reads.py b/src/graph_api/card_reads.py index 2142a8b0..28edef47 100644 --- a/src/graph_api/card_reads.py +++ b/src/graph_api/card_reads.py @@ -54,6 +54,7 @@ from typing import TYPE_CHECKING from code_index.query import CodeIndex +from dc_graph import display_name from dfo_platform.executor import submit_with_context from dfo_platform.single_flight_cache import SingleFlightCache from entity_resolution.membership import MembershipPolicy @@ -72,7 +73,6 @@ from graph_api.deps import DcpExecutorSaturated, DcpExecutorShutDown from graph_api.kinds import Kind, classify from graph_api.reader import GraphNodeReader -from graph_api.refs import first_name from graph_generation import GenerationProvider from graph_release.context import served_origin_key from materialise.errors import UpstreamResponseError @@ -110,7 +110,7 @@ @dataclass(frozen=True, slots=True) class NodeFacts: - """A node's first name unequal to its dcid, and its types. A node the graph does not hold + """A node's `dc_graph.display_name`, and its types. A node the graph does not hold has neither.""" name: str | None @@ -268,7 +268,7 @@ def _read_facts(self, keys: list[_FactsKey], deadline: float) -> dict[_FactsKey, def _facts(dcid: str, arcs: Mapping[str, tuple[str, ...]] | None) -> NodeFacts: if arcs is None: return NodeFacts(name=None, types=()) - return NodeFacts(name=first_name(arcs.get("name", ()), dcid), types=arcs.get("typeOf", ())) + return NodeFacts(name=display_name(arcs.get("name", ()), dcid), types=arcs.get("typeOf", ())) @dataclass(frozen=True, slots=True) diff --git a/src/graph_api/doors/members.py b/src/graph_api/doors/members.py index 447d44f9..150cd705 100644 --- a/src/graph_api/doors/members.py +++ b/src/graph_api/doors/members.py @@ -80,6 +80,7 @@ from code_index.query import CodeIndex from code_index.shared import get_index from code_index.types import CodeRow +from dc_graph import display_name from dfo_platform import ProblemSpec from dfo_platform.doors import DoorContribution from dfo_platform.static_response import gapped_public_json @@ -115,7 +116,6 @@ from graph_api.kinds import code_system_of from graph_api.node_models import BuiltFromTable from graph_api.reader import GraphNodeReader -from graph_api.refs import first_name from materialise.graph import GraphDeadlineExceededError, GraphReadError from origin_catalogue import current_catalogue, load_registry @@ -549,12 +549,13 @@ def _page[R]( def _names_sync(graph: GraphNodeReader, dcids: list[str], *, deadline: float) -> dict[str, str]: - """Each dcid's first name unequal to the dcid; a dcid with none is left out.""" - names = graph.node_names_many(dcids, deadline=deadline) + """Each dcid's `display_name`, read from its `name` arc alone, as the node door labels it; + a dcid with none is left out. An `alternateName` never becomes a label.""" + arcs = graph.node_arcs_many(dcids, properties=("name",), deadline=deadline) return { dcid: label - for dcid, values in names.items() - if (label := first_name(values, dcid)) is not None + for dcid, held in arcs.items() + if held is not None and (label := display_name(held.get("name", ()), dcid)) is not None } diff --git a/src/graph_api/doors/node.py b/src/graph_api/doors/node.py index b4e894d3..488c4259 100644 --- a/src/graph_api/doors/node.py +++ b/src/graph_api/doors/node.py @@ -62,6 +62,7 @@ from code_index.query import CodeIndex from code_index.shared import get_index, get_measure_index, per_index from code_index.types import MeasureIndex +from dc_graph import display_name from dfo_platform import ProblemSpec from dfo_platform.doors import DoorContribution from dfo_platform.refusal import TIMEOUT_DEFAULT @@ -114,7 +115,7 @@ ) from graph_api.origin_models import DataCommonsRef from graph_api.reader import GraphNodeReader, NodeArcsByDcid -from graph_api.refs import DOCUMENTATION_PATH, NodeRef, first_name, first_value +from graph_api.refs import DOCUMENTATION_PATH, NodeRef, first_value from materialise.cadence import cadence_label from materialise.graph import GraphDeadlineExceededError, GraphReadError from origin_catalogue import current_catalogue @@ -193,7 +194,7 @@ def __init__(self, length: int) -> None: def _label(arcs: Arcs, dcid: str) -> str | None: - return first_name(arcs.get(_LABEL_PROPERTY, ()), dcid) + return display_name(arcs.get(_LABEL_PROPERTY, ()), dcid) @dataclass(frozen=True, slots=True) diff --git a/src/graph_api/refs.py b/src/graph_api/refs.py index bddd9bf9..689a7618 100644 --- a/src/graph_api/refs.py +++ b/src/graph_api/refs.py @@ -3,7 +3,7 @@ from __future__ import annotations -from collections.abc import Iterable, Mapping +from collections.abc import Mapping from pydantic import ConfigDict @@ -23,12 +23,6 @@ class NodeRef(ResponseModel): label: str | None -def first_name(names: Iterable[str], dcid: str) -> str | None: - """The first of a node's `name` values that differs from its `dcid`. Some imports add a - self-referential arc, which is no name.""" - return next((name for name in names if name != dcid), None) - - def first_value(arcs: Mapping[str, tuple[str, ...]], prop: str) -> str | None: """The first value of `prop` among a node's arcs, `None` where it has none.""" values = arcs.get(prop, ()) diff --git a/src/presentation_store/frame_store.py b/src/presentation_store/frame_store.py index 538ac8a2..c77038af 100644 --- a/src/presentation_store/frame_store.py +++ b/src/presentation_store/frame_store.py @@ -196,6 +196,7 @@ from datasets_api.recipe_store import RecipeReader from datasets_api.render import servable_frame_location from datasets_api.subject_ids import served_subject_ids +from dc_graph import label_or_self_name from description.citation import figures_processed from description.compose import version_title from description.facts import ComposedVariable, DescriptionRecord @@ -210,7 +211,7 @@ canonical_frame_document, read_canonical_frame_table, ) -from materialise.graph import UNRESOLVED_NODE, GraphReader, GraphReadError +from materialise.graph import GraphReader, GraphReadError from materialise.provenance_doc import SourceCitation, read_source_citations from materialise.unit_cells import composed_unit_sides from origin_catalogue import current_catalogue @@ -1559,11 +1560,9 @@ def _label_refs( for bare, arcs in arcs_by_bare.items(): if arcs is None: continue - # The last truthy resolved name is the fuller roll-up label. - values = [value for value in arcs.get("name", ()) if value and value != UNRESOLVED_NODE] - if not values: + label = label_or_self_name(arcs.get("name", ()), bare) + if label is None: continue - label = values[-1] for ref in graph_bare_to_refs[bare]: labels[ref] = label return labels, family_read_ok diff --git a/src/search/catalogue/job.py b/src/search/catalogue/job.py index 3643be44..463c14aa 100644 --- a/src/search/catalogue/job.py +++ b/src/search/catalogue/job.py @@ -708,7 +708,7 @@ async def resolve(request: ResolveRequest) -> ResolveResponse: ) def place_names(dcids: Sequence[str]) -> Mapping[str, str]: - return {dcid: names[0] for dcid, names in graph.node_names_many(dcids).items() if names} + return graph.node_labels_batch(list(dcids)) yield resolve, place_names diff --git a/src/search/engine/quantity.py b/src/search/engine/quantity.py index 250c31e1..d1fba164 100644 --- a/src/search/engine/quantity.py +++ b/src/search/engine/quantity.py @@ -143,7 +143,7 @@ def read_denominators( arcs = arcs_by_dcid.get(denominator) reads[denominator] = DenominatorRead( kind=denominator_kind_from_arcs(arcs), - name=read_name(arcs) if arcs is not None else None, + name=read_name(arcs, denominator) if arcs is not None else None, ) return reads diff --git a/src/search/engine/region/alternate_guard.py b/src/search/engine/region/alternate_guard.py index 7e5c4d96..718cbbf0 100644 --- a/src/search/engine/region/alternate_guard.py +++ b/src/search/engine/region/alternate_guard.py @@ -420,22 +420,24 @@ def _label_residence(label: str) -> frozenset[str]: return frozenset(_RESIDENCE_TOKEN_RE.findall(label.lower())) -def _read_member(dcid: str, facts: MemberFacts, label: str) -> _Member: +def _read_member( + dcid: str, facts: MemberFacts, label: str, value_labels: Mapping[str, str] +) -> _Member: """Read a member. A member with no constraint arcs, such as a World Bank code series - whose code carries the slice, takes its residence from `label`.""" + whose code carries the slice, takes its residence from `label`. Any other takes it from + the labels `value_labels` gives its residence values, or from the values' dcids.""" constraints = facts.read_constraints() unresolved = facts.read_unresolved_constraints() if not facts.read_declared_constraint_properties(): return _Member( dcid, constraints, unresolved, _label_residence(label), _RESIDENCE_LABEL_AXIS ) - names = facts.read_constraint_value_names() residence_props = [prop for prop in _RESIDENCE_PROPERTIES if prop in constraints] tokens = frozenset( token for prop in residence_props for token in _RESIDENCE_TOKEN_RE.findall( - names.get(constraints[prop], constraints[prop]).lower() + value_labels.get(constraints[prop], constraints[prop]).lower() ) ) axis = residence_props[0] if residence_props else _RESIDENCE_LABEL_AXIS @@ -467,7 +469,7 @@ def _index_members( rows_key = (shape_five_tuple(shape), datasets) if rows_key not in by_rows: by_rows[rows_key] = tuple( - _read_member(row.dcid, IndexMemberFacts(row=row), "") for row in rows + _read_member(row.dcid, IndexMemberFacts(row=row), "", {}) for row in rows ) return by_rows[rows_key] @@ -598,7 +600,7 @@ def members(self, shape: ShapeDraft) -> tuple[tuple[_Member, ...], bool]: def label(dcid: str) -> str: if count == 1: return shape.label - return read_name(arcs.get(dcid) or {}) or "" + return read_name(arcs.get(dcid) or {}, dcid) or "" if rows: # A row with no constraint reads its residence from a label, which belongs @@ -606,14 +608,16 @@ def label(dcid: str) -> str: members = tuple( member if member.constraints - else _read_member(row.dcid, IndexMemberFacts(row=row), label(row.dcid)) + else _read_member(row.dcid, IndexMemberFacts(row=row), label(row.dcid), {}) for row, member in zip( rows, _index_members(shape, datasets, rows, index), strict=True ) ) else: members = tuple( - _read_member(dcid, ArcsMemberFacts(arcs=a), label(dcid)) + _read_member( + dcid, ArcsMemberFacts(arcs=a), label(dcid), shape.value_labels or {} + ) for dcid, a in arcs.items() ) self._members[key] = ( diff --git a/src/search/engine/region/candidate_grounding.py b/src/search/engine/region/candidate_grounding.py index 8a4b2411..61fd31c8 100644 --- a/src/search/engine/region/candidate_grounding.py +++ b/src/search/engine/region/candidate_grounding.py @@ -701,6 +701,7 @@ def _finish_candidate( cand_open_axis_facts = open_constraint_axes( cand_measure_bound_values, dict(zip(cand_mat.sv_dcids, cand_served_members, strict=True)), + value_labels=cand_shape.value_labels or {}, ) # Reuse materialised role facts without extra SDMX reads. Labelled entity options are # built only for a surviving definite answer. diff --git a/src/search/engine/region/phase_answer.py b/src/search/engine/region/phase_answer.py index 519613df..aa48a588 100644 --- a/src/search/engine/region/phase_answer.py +++ b/src/search/engine/region/phase_answer.py @@ -959,7 +959,9 @@ def _build_definite_spec( served_members = [served_member_facts[dcid] for dcid in mat_result.sv_dcids] # Record served values for every open constraint axis. open_axis_facts = open_constraint_axes( - measure_bound_values, dict(zip(mat_result.sv_dcids, served_members, strict=True)) + measure_bound_values, + dict(zip(mat_result.sv_dcids, served_members, strict=True)), + value_labels=state.shape_draft.value_labels or {}, ) spec = build_spec( shape=state.shape_model, diff --git a/src/search/engine/region/phase_rank_bind.py b/src/search/engine/region/phase_rank_bind.py index 004b3f0b..be887a17 100644 --- a/src/search/engine/region/phase_rank_bind.py +++ b/src/search/engine/region/phase_rank_bind.py @@ -3153,7 +3153,7 @@ def _member_names(shape: ShapeDraft) -> dict[str, frozenset[str]]: return { dcid: _label_words(name) for dcid, arcs in shape_population_arcs(shape).items() - if (name := read_name(arcs)) is not None + if (name := read_name(arcs, dcid)) is not None } @@ -3284,7 +3284,7 @@ def served_member_names(state: ResolveState) -> dict[str, str | None]: role_keys = assignment.role_keys if isinstance(assignment, PlaceAssignment) else RoleKeys() arcs = shape_population_arcs(shape) names = { - dcid: read_name(arcs.get(dcid, {})) + dcid: read_name(arcs.get(dcid, {}), dcid) for dcid in members_served( shape, bindings, role_keys, member_has_data_where_asked(state, shape) ) @@ -3585,8 +3585,8 @@ def judge_value_requirements( for value in binding.value_dcids } selected = _selected_members(shape, bindings) - constraints = [read_constraints(arcs) for arcs in selected] - names = [_casefolded_words(read_name(arcs) or "") for arcs in selected] + constraints = [read_constraints(arcs) for arcs in selected.values()] + names = [_casefolded_words(read_name(arcs, dcid) or "") for dcid, arcs in selected.items()] stated = _series_words(shape) | _dataset_stated_words(shape, get_index(), state) read_broadly = { (reading.property_dcid, reading.value_dcid) @@ -3644,7 +3644,7 @@ def met(requirement: Requirement, rule: ValueRule) -> bool: ) -def _selected_members(shape: ShapeDraft, bindings: Sequence[SlotBindingDraft]) -> list[dict]: +def _selected_members(shape: ShapeDraft, bindings: Sequence[SlotBindingDraft]) -> dict[str, dict]: """The arcs of each member `shape` stands for (`shape_population_arcs`) that `bindings` select, as materialisation selects them (`bound_values_from_bindings`): one holding a value of each property they bind as a value or set, or a descendant of one, and @@ -3657,13 +3657,13 @@ def _selected_members(shape: ShapeDraft, bindings: Sequence[SlotBindingDraft]) - for binding in bindings if binding.property_dcid is not None and binding.kind == "absent" } - selected: list[dict] = [] - for arcs in shape_population_arcs(shape).values(): + selected: dict[str, dict] = {} + for dcid, arcs in shape_population_arcs(shape).items(): member = read_constraints(arcs) if all(member.get(prop) in values for prop, values in bound.items()) and not ( absent & member.keys() ): - selected.append(arcs) + selected[dcid] = arcs return selected @@ -4225,7 +4225,7 @@ def _bind_variable(state: ResolveState, shapes: Sequence[ShapeDraft]) -> str: (len(arcs.get("observationProperties", {}).get("nodes", [])), name) for shape in shapes for dcid, arcs in (shape.sv_arc_facts or {}).items() - if dcid in code.dcids and (name := read_name(arcs)) is not None + if dcid in code.dcids and (name := read_name(arcs, dcid)) is not None ] return min(named)[1] if named else state.variable diff --git a/src/search/engine/region/promote_ground.py b/src/search/engine/region/promote_ground.py index 762a07b6..ff9ec2cf 100644 --- a/src/search/engine/region/promote_ground.py +++ b/src/search/engine/region/promote_ground.py @@ -499,6 +499,7 @@ def _ground_and_build_spec( _open_axis_facts = open_constraint_axes( _measure_bound_values, dict(zip(state.resolved_stat_var_dcids, _served_members, strict=True)), + value_labels=state.shape_draft.value_labels or {}, ) state.open_axis_facts = _open_axis_facts # Count the menus before the Spec takes them, so the finalisation probe and the diff --git a/src/search/engine/region/promote_shape.py b/src/search/engine/region/promote_shape.py index 02a5bfcc..24ee4a01 100644 --- a/src/search/engine/region/promote_shape.py +++ b/src/search/engine/region/promote_shape.py @@ -26,7 +26,7 @@ identity_query, walked_family_label, ) -from search.engine.retrieval.derive import index_shape_label, member_group_class +from search.engine.retrieval.derive import index_shape_label, member_group_class, value_labels_of from search.engine.retrieval.facts import ( FiveTuple, group_roles, @@ -204,6 +204,17 @@ def _phase_build_shape(state: _PromoteState) -> RegionResult | None: for dcid in (state.ft.pop_type_dcid, *held.shared, *held.shared.values()) if dcid not in label_dcids ) + # The same read labels the resolved members' constraint values, which open axes and + # residence readings label from (`ShapeDraft.value_labels`). + member_values = list( + dict.fromkeys( + value + for sv in state.resolved_stat_var_dcids + for value in read_constraints(state.resolved_arcs[sv]).values() + if value not in label_dcids + ) + ) + label_dcids.extend(member_values) prop_label_map = state.graph.node_labels_batch(label_dcids) if label_dcids else {} prop_labels = {p: prop_label_map.get(p, p) for p in constraint_props} @@ -256,6 +267,7 @@ def _phase_build_shape(state: _PromoteState) -> RegionResult | None: prop_labels=prop_labels, prop_observed_values=prop_observed_values, sv_arc_facts=sv_arc_facts, + value_labels=value_labels_of(member_values, prop_label_map), # These arcs form the complete elected family. sv_arc_facts_complete=True, declared_roles=roles.declared, diff --git a/src/search/engine/region/state.py b/src/search/engine/region/state.py index fa90d66c..27fe212f 100644 --- a/src/search/engine/region/state.py +++ b/src/search/engine/region/state.py @@ -254,7 +254,8 @@ def _elected_stat_vars( # materialise runs only after bind sets `shape_draft`. arc_facts = shape_population_arcs(shape_draft) return [ - GraphRef(ref=dcid, label=read_name(arc_facts.get(dcid, {})) or dcid) for dcid in sv_dcids + GraphRef(ref=dcid, label=read_name(arc_facts.get(dcid, {}), dcid) or dcid) + for dcid in sv_dcids ] diff --git a/src/search/engine/related.py b/src/search/engine/related.py index 74c4a556..8c2560ba 100644 --- a/src/search/engine/related.py +++ b/src/search/engine/related.py @@ -157,7 +157,7 @@ def read_related( held = member_arcs.get(dcid) if held is None or (key := _identity_key(held)) is None or key == elected_key: continue - by_key.setdefault(key, {})[dcid] = read_name(held) or dcid + by_key.setdefault(key, {})[dcid] = read_name(held, dcid) or dcid elected_spans = _spans(facets.get(stat_var, {}), places) elected_span = _hull(elected_spans) if elected_spans is not None else None @@ -182,7 +182,7 @@ def read_related( if not ranked: return NO_RELATED ranked.sort(key=lambda item: (-item[0], item[1].dcids)) - group_label = read_name(member_arcs.get(group) or {}) or group + group_label = read_name(member_arcs.get(group) or {}, group) or group return Related( group=GraphRef(ref=group, label=group_label), members=tuple(member for _, member in ranked[:RELATED_MEASURES_MAX]), diff --git a/src/search/engine/retrieval/bound_values.py b/src/search/engine/retrieval/bound_values.py index 097757ce..8da1a97c 100644 --- a/src/search/engine/retrieval/bound_values.py +++ b/src/search/engine/retrieval/bound_values.py @@ -1045,8 +1045,8 @@ class OpenAxisFact: `member_values` maps a member dcid to the value dcid it declares, or `None` when it declares none. Keying by member makes a union over overlapping member sets count - each member once. `value_names` maps a value dcid to the graph `name` a served - member's arc carries, for values the code index does not label. + each member once. `value_names` maps a value dcid to its label from the shape's value + labels (`ShapeDraft.value_labels`), for values the code index does not label. """ property: str @@ -1068,20 +1068,20 @@ def value_counts(self) -> Counter[str]: def open_constraint_axes( - bound_values: Mapping[str, list[str]], members: Mapping[str, MemberFacts] + bound_values: Mapping[str, list[str]], + members: Mapping[str, MemberFacts], + *, + value_labels: Mapping[str, str], ) -> tuple[OpenAxisFact, ...]: """Return unbound constraint axes in first-observed member order. `members` maps each served member's dcid to its facts. Include unresolved properties. A member with no resolved value contributes `None`. Each fact carries - the members' graph names for its values. Return `()` when no open axis exists. + the labels `value_labels` holds for its values: the shape's value labels + (`ShapeDraft.value_labels`), which one batched label read gave every constraint value + of the shape's members and family. Return `()` when no open axis exists. """ member_constraints = {dcid: m.read_constraints() for dcid, m in members.items()} - value_names = { - value: name - for m in members.values() - for value, name in m.read_constraint_value_names().items() - } order: list[str] = [] seen: set[str] = set() for dcid, m in members.items(): @@ -1100,7 +1100,11 @@ def open_constraint_axes( OpenAxisFact( property=prop, member_values=member_values, - value_names={v: value_names[v] for v in member_values.values() if v in value_names}, + value_names={ + v: label + for v in member_values.values() + if v is not None and (label := value_labels.get(v)) is not None + }, ) ) return tuple(facts) @@ -3366,7 +3370,10 @@ def walked_family_label( prop_labels=prop_labels, value_labels=value_labels, ) - return next((name for arcs in members.values() if (name := read_name(arcs)) is not None), None) + return next( + (name for dcid, arcs in members.items() if (name := read_name(arcs, dcid)) is not None), + None, + ) def _shape_held_family(shape: ShapeDraft) -> FamilySummary | None: @@ -3433,7 +3440,11 @@ def _recalled_role_class_shape(shape: ShapeDraft, query: FamilyQuery) -> ShapeDr shape.label if shape.held_family is not None else next( - (name for sv in least_constrained if (name := read_name(population[sv])) is not None), + ( + name + for sv in least_constrained + if (name := read_name(population[sv], sv)) is not None + ), shape.label, ) ) diff --git a/src/search/engine/retrieval/derive.py b/src/search/engine/retrieval/derive.py index 230cd337..68eaf184 100644 --- a/src/search/engine/retrieval/derive.py +++ b/src/search/engine/retrieval/derive.py @@ -541,16 +541,19 @@ def _fetch_constraint_labels( labels_raw = graph.node_labels_batch( [*all_constraint_props, *all_constraint_values, *population_types] ) - prop_labels_raw = labels_raw + return labels_raw, value_labels_of(all_constraint_values, labels_raw) + +def value_labels_of(values: Iterable[str], labels: Mapping[str, str]) -> dict[str, str]: + """A shape's value labels (`ShapeDraft.value_labels`): the graph `labels` of one batched + read, with each of `values` the code index holds labelled by its code row instead.""" code_index = get_index() - value_labels = dict(labels_raw) - for value_dcid in all_constraint_values: + value_labels = dict(labels) + for value_dcid in values: row = code_index.code_by_dcid(value_dcid) if row is not None: value_labels[value_dcid] = row.label - - return prop_labels_raw, value_labels + return value_labels def _shape_draft_for_group( diff --git a/src/search/engine/retrieval/facts.py b/src/search/engine/retrieval/facts.py index 24baf692..d202acf7 100644 --- a/src/search/engine/retrieval/facts.py +++ b/src/search/engine/retrieval/facts.py @@ -5,6 +5,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, NamedTuple +from dc_graph import label_or_self_name from materialise.roles import declared_roles, roles_by_axis from search.engine.errors import RoleSignatureError @@ -34,14 +35,11 @@ def _dcids_for(arcs: dict, prop: str) -> list[str]: return [n["dcid"] for n in nodes if "dcid" in n] -def read_name(arcs: dict) -> str | None: - """Return the last truthy `->name` value, matching the graph label policy. - - Reads already-fetched arcs without another graph call. - """ - name_nodes = arcs.get("name", {}).get("nodes", []) - values = [n["value"] for n in name_nodes if n.get("value")] - return values[-1] if values else None +def read_name(arcs: dict, dcid: str) -> str | None: + """Return the label of node `dcid` from its already-fetched `->name` arcs as the graph + client's `node_label` reads it (`label_or_self_name`), without another graph call.""" + names = [value for n in arcs.get("name", {}).get("nodes", []) if (value := n.get("value"))] + return label_or_self_name(names, dcid) def _value_for(arcs: dict, prop: str) -> str | None: @@ -154,19 +152,6 @@ def read_unresolved_constraints(sv_arcs: dict) -> frozenset[str]: return frozenset(p for p in constraint_prop_dcids if _is_unresolved(sv_arcs, p)) -def read_constraint_value_names(sv_arcs: dict) -> dict[str, str]: - """Map each constraint value dcid to the graph `name` its arc node carries. - - Values whose node carries no name are omitted. - """ - return { - node["dcid"]: node["name"] - for prop_dcid in _dcids_for(sv_arcs, "constraintProperties") - for node in sv_arcs.get(prop_dcid, {}).get("nodes", []) - if node.get("dcid") and node.get("name") - } - - class GroupRoles(NamedTuple): """The roles a five-tuple group's members declare.""" diff --git a/src/search/engine/retrieval/member_facts.py b/src/search/engine/retrieval/member_facts.py index a5844606..a30d6b04 100644 --- a/src/search/engine/retrieval/member_facts.py +++ b/src/search/engine/retrieval/member_facts.py @@ -12,7 +12,6 @@ from materialise.facets import ElectedDataset from search.engine.retrieval.facts import ( FiveTuple, - read_constraint_value_names, read_constraints, read_declared_constraint_properties, read_declared_roles, @@ -30,7 +29,6 @@ def constraints_known(self) -> bool: ... def read_constraints(self) -> dict[str, str]: ... def read_declared_constraint_properties(self) -> tuple[str, ...]: ... def read_unresolved_constraints(self) -> frozenset[str]: ... - def read_constraint_value_names(self) -> dict[str, str]: ... def read_declared_roles(self) -> tuple[str, ...]: ... def read_dataset(self) -> ElectedDataset | None: ... @@ -46,10 +44,10 @@ def asserted_constraint_properties(facts: MemberFacts) -> frozenset[str] | None: return frozenset(facts.read_constraints()) | facts.read_unresolved_constraints() -def member_name(facts: MemberFacts) -> str | None: - """The graph name of a member whose fetched arcs `facts` holds (`ArcsMemberFacts`). +def member_name(dcid: str, facts: MemberFacts) -> str | None: + """The graph name of member `dcid` whose fetched arcs `facts` holds (`ArcsMemberFacts`). None for a member with no name arc and for a code index row, which carries none.""" - return read_name(facts.arcs) if isinstance(facts, ArcsMemberFacts) else None + return read_name(facts.arcs, dcid) if isinstance(facts, ArcsMemberFacts) else None @dataclass(frozen=True, slots=True) @@ -80,9 +78,6 @@ def read_declared_constraint_properties(self) -> tuple[str, ...]: def read_unresolved_constraints(self) -> frozenset[str]: return read_unresolved_constraints(self.arcs) - def read_constraint_value_names(self) -> dict[str, str]: - return read_constraint_value_names(self.arcs) - def read_declared_roles(self) -> tuple[str, ...]: return read_declared_roles(self.arcs) @@ -126,10 +121,6 @@ def read_declared_constraint_properties(self) -> tuple[str, ...]: def read_unresolved_constraints(self) -> frozenset[str]: return frozenset() - def read_constraint_value_names(self) -> dict[str, str]: - # The code index labels every value a row carries. - return {} - def read_declared_roles(self) -> tuple[str, ...]: return self.row.observation_properties or ("observationAbout",) diff --git a/src/search/engine/retrieval/standard.py b/src/search/engine/retrieval/standard.py index 5f6d2db1..d8994444 100644 --- a/src/search/engine/retrieval/standard.py +++ b/src/search/engine/retrieval/standard.py @@ -692,7 +692,7 @@ def bind_named_member_values( ) if not partitions: break - names = {sv: member_name(member) for sv, member in served.items()} + names = {sv: member_name(sv, member) for sv, member in served.items()} if any(name is None for name in names.values()): break named = [sv for sv, name in names.items() if name and normalised_key(name) == asked] diff --git a/src/search/engine/unit/conversion_catalogue.py b/src/search/engine/unit/conversion_catalogue.py index 072efe4a..8237f644 100644 --- a/src/search/engine/unit/conversion_catalogue.py +++ b/src/search/engine/unit/conversion_catalogue.py @@ -35,6 +35,7 @@ from dataclasses import dataclass, field from types import MappingProxyType +from dc_graph import label_or_self_name from dc_sdmx import SdmxClient from dfo_platform.executor import submit_with_context from materialise.convert import converted_facets, deflator_key @@ -208,7 +209,9 @@ def table(key: _TableKey) -> RateTable: } | national.keys() ) - names = reader.node_names_many(named, deadline=deadline) if named else {} + names = ( + reader.node_arcs_many(named, properties=("name",), deadline=deadline) if named else {} + ) return ConversionCatalogue( recipes=MappingProxyType(recipes.result()), tables=MappingProxyType( @@ -222,7 +225,11 @@ def table(key: _TableKey) -> RateTable: national_currencies=MappingProxyType(national), default_base_year=base_year.result(), entity_names=MappingProxyType( - {dcid: found[0] for dcid, found in names.items() if found} + { + dcid: name + for dcid, arcs in names.items() + if arcs is not None and (name := label_or_self_name(arcs.get("name", ()), dcid)) + } ), ) diff --git a/tests/dc_graph/test_client_labels.py b/tests/dc_graph/test_client_labels.py index 734c14d3..5dba0aa2 100644 --- a/tests/dc_graph/test_client_labels.py +++ b/tests/dc_graph/test_client_labels.py @@ -217,6 +217,28 @@ def fake_post(self, path, payload, *, deadline=None): assert result == {"country/BBB": "B"} +@pytest.mark.parametrize( + "names", + [ + ["United States", "United States of America", "country/USA"], + ["country/USA", "United States of America", "United States"], + ], +) +def test_a_node_of_several_names_takes_one_label_whatever_their_order(monkeypatch, names) -> None: + """The graph keys names on provenance and fixes no order among them, so the label is the + `display_name`, and a name equal to the dcid is none.""" + + def fake_post(self, path, payload, *, deadline=None): + nodes = [{"value": name} for name in names] + return {"data": {"country/USA": {"arcs": {"name": {"nodes": nodes}}}}} + + monkeypatch.setattr(GraphClient, "post", fake_post) + client = GraphClient(base_url="http://fake/core/api") + + assert client.node_label("country/USA") == "United States of America" + assert client.node_labels_batch(["country/USA"]) == {"country/USA": "United States of America"} + + def test_node_labels_batch_error_before_cache_write(monkeypatch) -> None: """A GraphReadError from post propagates without any cache write.""" diff --git a/tests/engine/test_admissibility_table.py b/tests/engine/test_admissibility_table.py index af24c916..5bb38f4b 100644 --- a/tests/engine/test_admissibility_table.py +++ b/tests/engine/test_admissibility_table.py @@ -107,8 +107,18 @@ def _shape( units: frozenset[str | None] | None = None, place_available: bool | None = None, ) -> ShapeDraft: - """A shape. `members` maps member dcids to arcs, or lists dcids with no constraint arcs.""" + """A shape. `members` maps member dcids to arcs, or lists dcids with no constraint arcs. + Its value labels are the names the member arcs give their values, as the shape's one + batched label read would.""" facts = {dcid: {} for dcid in members} if isinstance(members, tuple) else members + value_labels = { + node["dcid"]: node["name"] + for arcs in facts.values() + for prop, entry in arcs.items() + if prop != "constraintProperties" + for node in entry["nodes"] + if "name" in node + } return ShapeDraft( shape_id=label or f"{pop}/{prop}/{qual}/{denom}", label=label, @@ -120,6 +130,7 @@ def _shape( slot_keys=(), meas_denom_unresolved=unresolved, sv_arc_facts=facts or None, + value_labels=value_labels or None, declared_roles=roles, place_units=units, place_available=place_available, diff --git a/tests/engine/test_arc_tables_serving.py b/tests/engine/test_arc_tables_serving.py index 7c5d9f2d..77534a6c 100644 --- a/tests/engine/test_arc_tables_serving.py +++ b/tests/engine/test_arc_tables_serving.py @@ -462,7 +462,11 @@ def test_the_family_tables_summarise_each_family_as_derive_reads_its_members() - for prop, value in read_constraints(member["arcs"]).items(): values.setdefault(prop, {})[value] = None total = next( - (m for m in members if not read_constraints(m["arcs"]) and read_name(m["arcs"])), + ( + m + for m in members + if not read_constraints(m["arcs"]) and read_name(m["arcs"], m["dcid"]) + ), None, ) expected[key] = FamilySummary( @@ -474,7 +478,7 @@ def test_the_family_tables_summarise_each_family_as_derive_reads_its_members() - if len(found) == 1 and all(prop in read_constraints(m["arcs"]) for m in members) }, total=None if total is None else total["dcid"], - total_name=None if total is None else read_name(total["arcs"]), + total_name=None if total is None else read_name(total["arcs"], total["dcid"]), un_only=False, ) @@ -682,7 +686,9 @@ def _flagged_member_constraint_properties(shape: ShapeDraft) -> list[list[str] | materialise elected, which recall did not find, under a request's memo. The state holds what the refusal reads; the rest are placeholders.""" graph = FakeGraph( - nodes={dcid: {"label": read_name(arcs), "arcs": arcs} for dcid, arcs in _DEBT.items()}, + nodes={ + dcid: {"label": read_name(arcs, dcid), "arcs": arcs} for dcid, arcs in _DEBT.items() + }, obs={}, detect={}, resolve={}, @@ -787,7 +793,9 @@ def _value_state(shape: ShapeDraft) -> ResolveState: id="pre-bind-members", ), pytest.param( - lambda shape: [read_name(arcs) for arcs in _selected_members(shape, [_imf()])], + lambda shape: [ + read_name(arcs, dcid) for dcid, arcs in _selected_members(shape, [_imf()]).items() + ], ["Outstanding external debt to the IMF, general government"], id="requirement-judging", ), diff --git a/tests/engine/test_open_axis_facts.py b/tests/engine/test_open_axis_facts.py index ffeeb49e..bdea753b 100644 --- a/tests/engine/test_open_axis_facts.py +++ b/tests/engine/test_open_axis_facts.py @@ -56,7 +56,7 @@ def _fact(axes: tuple[OpenAxisFact, ...], prop: str) -> OpenAxisFact: def test_members_sharing_one_value_publish_it_as_not_varying(): members = _index_members(_SHARED_PURPOSE_A, _SHARED_PURPOSE_B) - axes = open_constraint_axes({}, members) + axes = open_constraint_axes({}, members, value_labels={}) purpose = _fact(axes, "crsPurpose") assert purpose.values == frozenset({"CRSPurposeCode11330"}) assert purpose.undeclared_count == 0 @@ -64,14 +64,14 @@ def test_members_sharing_one_value_publish_it_as_not_varying(): def test_members_that_differ_publish_more_than_one_value(): members = _index_members(_SHARED_PURPOSE_A, _SHARED_PURPOSE_B) - axes = open_constraint_axes({}, members) + axes = open_constraint_axes({}, members, value_labels={}) modality = _fact(axes, "crsCooperationModality") assert modality.values == frozenset({"CRSCooperationModalityA01", "CRSCooperationModalityA02"}) def test_undeclared_count_equals_members_missing_the_key(): members = _index_members(_SHARED_PURPOSE_A, _SHARED_PURPOSE_B, _NO_MODALITY) - axes = open_constraint_axes({}, members) + axes = open_constraint_axes({}, members, value_labels={}) modality = _fact(axes, "crsCooperationModality") assert modality.member_values[_NO_MODALITY] is None assert modality.undeclared_count == 1 @@ -80,7 +80,7 @@ def test_undeclared_count_equals_members_missing_the_key(): def test_a_bound_property_never_reaches_the_output(): members = _index_members(_SHARED_PURPOSE_A, _SHARED_PURPOSE_B) - axes = open_constraint_axes({"crsPurpose": ["CRSPurposeCode11330"]}, members) + axes = open_constraint_axes({"crsPurpose": ["CRSPurposeCode11330"]}, members, value_labels={}) assert all(a.property != "crsPurpose" for a in axes) @@ -92,21 +92,23 @@ def test_index_sourced_and_recall_sourced_members_agree(): } from_index = { - a.property: (a.values, a.undeclared_count) for a in open_constraint_axes({}, index_members) + a.property: (a.values, a.undeclared_count) + for a in open_constraint_axes({}, index_members, value_labels={}) } from_arcs = { - a.property: (a.values, a.undeclared_count) for a in open_constraint_axes({}, arcs_members) + a.property: (a.values, a.undeclared_count) + for a in open_constraint_axes({}, arcs_members, value_labels={}) } assert from_index == from_arcs def test_no_members_publishes_nothing_to_report(): - assert open_constraint_axes({}, {}) == () + assert open_constraint_axes({}, {}, value_labels={}) == () def test_a_member_with_no_constraint_properties_at_all_publishes_nothing(): members = {"sv": ArcsMemberFacts(arcs={})} - assert open_constraint_axes({}, members) == () + assert open_constraint_axes({}, members, value_labels={}) == () # --------------------------------------------------------------------------- @@ -141,7 +143,7 @@ def test_an_axis_every_member_declares_nothing_for_still_publishes_its_own_entry def test_varying_axis_publishes_varies_true(): members = _index_members(_SHARED_PURPOSE_A, _SHARED_PURPOSE_B) - facts = open_constraint_axes({}, members) + facts = open_constraint_axes({}, members, value_labels={}) published = _open_axis_facts(facts) @@ -195,7 +197,7 @@ def test_distinct_count_counts_every_distinct_declared_value_on_a_varying_axis() def test_each_declared_value_publishes_its_member_count(): members = _index_members(_SHARED_PURPOSE_A, _SHARED_PURPOSE_B, _NO_MODALITY) - published = _open_axis_facts(open_constraint_axes({}, members)) + published = _open_axis_facts(open_constraint_axes({}, members, value_labels={})) assert published is not None by_property = {f.property: f for f in published} @@ -229,26 +231,35 @@ def test_values_tied_on_member_count_order_by_label(): assert [v.value.ref for v in published[0].values] == ["undata/SEX-B", "undata/SEX-A"] -def _un_member_arcs(sex_name: str | None) -> dict: - """A UN System member asserting Sex=Male, with or without the value's graph name.""" - node = {"dcid": "undata/SEX-M"} | ({"name": sex_name} if sex_name is not None else {}) +def _un_member_arcs(arc_name: str | None) -> dict: + """A UN System member asserting Sex=Male, with or without the name its `->*` arc carries.""" + node = {"dcid": "undata/SEX-M"} | ({"name": arc_name} if arc_name is not None else {}) return { "constraintProperties": {"nodes": [{"dcid": "undataSex"}]}, "undataSex": {"nodes": [node]}, } -def test_a_value_the_code_index_lacks_publishes_its_graph_name(): - facts = open_constraint_axes({}, {"sv": ArcsMemberFacts(arcs=_un_member_arcs("Male"))}) +@pytest.mark.parametrize( + "names", + [("Male", "Male population"), ("Male population", "Male")], +) +def test_a_value_the_code_index_lacks_publishes_its_label_and_not_its_arc_name(names): + """The graph returns a node's names in no fixed order, and the `->*` arc carries one of + them. The published label is the shape's value label, the node's `display_name` that + one batched `node_labels_batch` read gives, whichever name the arc carries.""" + members = {"sv": ArcsMemberFacts(arcs=_un_member_arcs(names[0]))} + facts = open_constraint_axes({}, members, value_labels={"undata/SEX-M": "Male population"}) published = _open_axis_facts(facts) assert published is not None - assert published[0].value == GraphRef(ref="undata/SEX-M", label="Male") + assert published[0].value == GraphRef(ref="undata/SEX-M", label="Male population") -def test_a_value_with_no_graph_name_publishes_its_dcid(): - facts = open_constraint_axes({}, {"sv": ArcsMemberFacts(arcs=_un_member_arcs(None))}) +def test_a_value_with_no_label_publishes_its_dcid(): + members = {"sv": ArcsMemberFacts(arcs=_un_member_arcs("Male"))} + facts = open_constraint_axes({}, members, value_labels={}) published = _open_axis_facts(facts) @@ -257,14 +268,14 @@ def test_a_value_with_no_graph_name_publishes_its_dcid(): @pytest.mark.parametrize( - ("prop", "value", "arc_name", "expected"), + ("prop", "value", "label", "expected"), [ pytest.param( "undataAge", "undata/AGE-M24T59", "24 to 59 months old", "ODA Grants for 24 to 59 months old.", - id="un-value-renders-its-graph-name", + id="un-value-renders-its-label", ), pytest.param( "activitySource", @@ -275,16 +286,18 @@ def test_a_value_with_no_graph_name_publishes_its_dcid(): ), ], ) -def test_a_constant_value_the_code_index_lacks_renders_from_its_arc_name( - prop: str, value: str, arc_name: str, expected: str +def test_a_constant_value_the_code_index_lacks_renders_from_its_label( + prop: str, value: str, label: str, expected: str ): """The live graph names `undata/AGE-M24T59` "24 to 59 months old" and names `GrossDomesticProduction` by its own dcid, which the renderer humanises.""" arcs = { "constraintProperties": {"nodes": [{"dcid": prop}]}, - prop: {"nodes": [{"dcid": value, "name": arc_name}]}, + prop: {"nodes": [{"dcid": value, "name": label}]}, } - published = _open_axis_facts(open_constraint_axes({}, {"sv": ArcsMemberFacts(arcs=arcs)})) + published = _open_axis_facts( + open_constraint_axes({}, {"sv": ArcsMemberFacts(arcs=arcs)}, value_labels={value: label}) + ) spec = _spec(slots=[_what_slot("ODA Grants")], open_axis_facts=published) diff --git a/tests/engine/test_unit_conversion_offer.py b/tests/engine/test_unit_conversion_offer.py index 0eab91c3..d2b031fd 100644 --- a/tests/engine/test_unit_conversion_offer.py +++ b/tests/engine/test_unit_conversion_offer.py @@ -577,6 +577,8 @@ def __init__(self, extra: Mapping[str, Mapping[str, tuple[str, ...]]] | None = N unit: {"typeOf": ("CurrencyUnit",), "nationalCurrencyOf": places} for unit, places in self._CURRENCIES.items() }, + "DACCountriesTotal": {"name": ("DAC countries, total",)}, + "country/GTM": {"name": ("Guatemala",)}, **(extra or {}), } @@ -589,12 +591,6 @@ def node_arcs_many( arcs[dcid] = {prop: node[prop] for prop in properties if prop in node} or None return arcs - def node_names_many( - self, dcids: Sequence[str], *, deadline: float | None = None - ) -> dict[str, tuple[str, ...]]: - names = {"DACCountriesTotal": ("DAC countries, total",), "country/GTM": ("Guatemala",)} - return {dcid: names[dcid] for dcid in dcids if dcid in names} - def node_subjects( self, dcid: str, diff --git a/tests/fixtures/requests/test_dataset_resubmit.json b/tests/fixtures/requests/test_dataset_resubmit.json index e4c0faab..71eda506 100644 --- a/tests/fixtures/requests/test_dataset_resubmit.json +++ b/tests/fixtures/requests/test_dataset_resubmit.json @@ -1,7 +1,7 @@ { "test_a_resubmit_drops_a_label_only_group_and_states_a_spec_sentence_without_departures": [ ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], - ["FakeGraph.node_labels_batch",{"dcids":["activitySource"]}], + ["FakeGraph.node_labels_batch",{"dcids":["GrossDomesticProduction","activitySource"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/IND"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"measurementMethod"}], @@ -647,7 +647,7 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.node_labels_batch",{"dcids":["dc/base/WorldDevelopmentIndicators"]}], ["FakeGraph.node_arcs_batch",{"dcids":["FertilityRate_Person_Female"]}], - ["FakeGraph.node_labels_batch",{"dcids":["gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Female","gender"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"measurementMethod"}], @@ -656,7 +656,7 @@ ["FakeGraph.node_subjects_many",{"dcids":["dc/g/Health"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["LifeExpectancy_Person","dc/g/Health"],"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"]}], - ["FakeGraph.node_labels_batch",{"dcids":["medicalStatus"]}], + ["FakeGraph.node_labels_batch",{"dcids":["LiveBirth","medicalStatus"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"measurementMethod"}], @@ -802,7 +802,7 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.node_labels_batch",{"dcids":["dc/base/WorldDevelopmentIndicators"]}], ["FakeGraph.node_arcs_batch",{"dcids":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"]}], - ["FakeGraph.node_labels_batch",{"dcids":["medicalStatus"]}], + ["FakeGraph.node_labels_batch",{"dcids":["LiveBirth","medicalStatus"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"measurementMethod"}], @@ -810,15 +810,15 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/BirthEvent_MedicalStatus-LiveBirth"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth","dc/g/BirthEvent_MedicalStatus-LiveBirth"],"deadline":"set"}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["FertilityRate_Person_Female"]}], - ["FakeGraph.node_labels_batch",{"dcids":["gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Female","gender"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"measurementMethod"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"unit"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/Health"],"property":"memberOf","max_results":40,"deadline":"set"}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], ["FakeGraph.node_labels_batch",{"dcids":["BirthEvent","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Count_Person","LiveBirth","count","country/ETH","measuredValue","medicalStatus"]}], ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], @@ -831,7 +831,7 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.node_labels_batch",{"dcids":["dc/base/WorldDevelopmentIndicators"]}], ["FakeGraph.node_arcs_batch",{"dcids":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"]}], - ["FakeGraph.node_labels_batch",{"dcids":["medicalStatus"]}], + ["FakeGraph.node_labels_batch",{"dcids":["LiveBirth","medicalStatus"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"measurementMethod"}], @@ -840,7 +840,7 @@ ["FakeGraph.node_subjects_many",{"dcids":["dc/g/BirthEvent_MedicalStatus-LiveBirth"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth","dc/g/BirthEvent_MedicalStatus-LiveBirth"],"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["FertilityRate_Person_Female"]}], - ["FakeGraph.node_labels_batch",{"dcids":["gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Female","gender"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"measurementMethod"}], @@ -848,11 +848,11 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/Health"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], + ["FakeGraph.node_arcs_batch",{"dcids":["LifeExpectancy_Person","dc/g/Health"],"deadline":"set"}], ["FakeGraph.node_labels_batch",{"dcids":["BirthEvent","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Count_Person","LiveBirth","count","country/ETH","measuredValue","medicalStatus"]}], ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"],"entities":["country/ETH"],"needs_dates":true}], - ["FakeGraph.node_arcs_batch",{"dcids":["LifeExpectancy_Person","dc/g/Health"],"deadline":"set"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female","LifeExpectancy_Person"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], ["FakeGraph.node_labels_batch",{"dcids":["Female","FertilityRate_Person_Female","Person","country/ETH","dc/base/WorldDevelopmentIndicators","fertilityRate","gender","measuredValue"]}], ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], @@ -986,7 +986,7 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.node_labels_batch",{"dcids":["dc/base/WorldDevelopmentIndicators"]}], ["FakeGraph.node_arcs_batch",{"dcids":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"]}], - ["FakeGraph.node_labels_batch",{"dcids":["medicalStatus"]}], + ["FakeGraph.node_labels_batch",{"dcids":["LiveBirth","medicalStatus"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"measurementMethod"}], @@ -994,19 +994,19 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/BirthEvent_MedicalStatus-LiveBirth"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth","dc/g/BirthEvent_MedicalStatus-LiveBirth"],"deadline":"set"}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], + ["FakeGraph.node_labels_batch",{"dcids":["BirthEvent","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Count_Person","LiveBirth","count","country/ETH","measuredValue","medicalStatus"]}], + ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.node_arcs_batch",{"dcids":["FertilityRate_Person_Female"]}], - ["FakeGraph.node_labels_batch",{"dcids":["gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Female","gender"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"measurementMethod"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"unit"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/Health"],"property":"memberOf","max_results":40,"deadline":"set"}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], - ["FakeGraph.node_labels_batch",{"dcids":["BirthEvent","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Count_Person","LiveBirth","count","country/ETH","measuredValue","medicalStatus"]}], - ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], - ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.node_arcs_batch",{"dcids":["LifeExpectancy_Person","dc/g/Health"],"deadline":"set"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female","LifeExpectancy_Person"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], ["FakeGraph.node_labels_batch",{"dcids":["Female","FertilityRate_Person_Female","Person","country/ETH","dc/base/WorldDevelopmentIndicators","fertilityRate","gender","measuredValue"]}], @@ -1141,7 +1141,7 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.node_labels_batch",{"dcids":["dc/base/WorldDevelopmentIndicators"]}], ["FakeGraph.node_arcs_batch",{"dcids":["FertilityRate_Person_Female"]}], - ["FakeGraph.node_labels_batch",{"dcids":["gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Female","gender"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"measurementMethod"}], @@ -1156,7 +1156,7 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.node_labels_batch",{"dcids":["dc/base/WorldDevelopmentIndicators"]}], ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], - ["FakeGraph.node_labels_batch",{"dcids":["activitySource"]}], + ["FakeGraph.node_labels_batch",{"dcids":["GrossDomesticProduction","activitySource"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/USA"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"measurementMethod"}], @@ -1171,10 +1171,10 @@ ], "test_dynamic_refusal_picks_lowest_index_even_when_it_finishes_last": [ ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], - ["FakeGraph.node_labels_batch",{"dcids":["activitySource"]}], + ["FakeGraph.node_labels_batch",{"dcids":["GrossDomesticProduction","activitySource"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ZZZ_NOT_A_PLACE_1"]}], ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], - ["FakeGraph.node_labels_batch",{"dcids":["activitySource"]}], + ["FakeGraph.node_labels_batch",{"dcids":["GrossDomesticProduction","activitySource"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ZZZ_NOT_A_PLACE_0"]}] ], "test_entries_posted_without_operations_rebuild_parts_without_them": [], @@ -1295,16 +1295,16 @@ ["FakeGraph.node_labels_batch",{"dcids":["BirthEvent","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Count_Person","LiveBirth","count","country/ETH","measuredValue","medicalStatus"]}], ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person"],"entities":["country/ETH"],"needs_dates":true}], + ["FakeGraph.node_labels_batch",{"dcids":["WikidataPopulation","WorldBankSubnationalPopulationEstimate","dc/base/EthiopiaStatistics","dc/base/Subnational_Demographics_Stats","dc/base/WikidataPopulation","dc/base/WorldDevelopmentIndicators"]}], ["FakeGraph.node_labels_batch",{"dcids":["Female","FertilityRate_Person_Female","Person","country/ETH","dc/base/WorldDevelopmentIndicators","fertilityRate","gender","measuredValue"]}], ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person"],"entities":["country/ETH"],"needs_dates":true}], - ["FakeGraph.node_labels_batch",{"dcids":["WikidataPopulation","WorldBankSubnationalPopulationEstimate","dc/base/EthiopiaStatistics","dc/base/Subnational_Demographics_Stats","dc/base/WikidataPopulation","dc/base/WorldDevelopmentIndicators"]}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.node_labels_batch",{"dcids":["dc/base/WorldDevelopmentIndicators"]}], ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], - ["FakeGraph.node_labels_batch",{"dcids":["activitySource"]}], + ["FakeGraph.node_labels_batch",{"dcids":["GrossDomesticProduction","activitySource"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/IND"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"measurementMethod"}], @@ -1313,7 +1313,7 @@ ["FakeGraph.node_subjects_many",{"dcids":["dc/g/EconomicActivity_ActivitySource-GrossDomesticProduction"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction","Amount_EconomicActivity_GrossDomesticProduction_Nominal_AsAFractionOf_Count_Person","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","Amount_EconomicActivity_GrossDomesticProduction_RealValue","Annual_Amount_EconomicActivity_GrossDomesticProduction","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","InflationAdjustedGDP","Nominal_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","Quarterly_Amount_EconomicActivity_GrossDomesticProduction","RealValue_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","dc/g/EconomicActivity_ActivitySource-GrossDomesticProduction"],"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"]}], - ["FakeGraph.node_labels_batch",{"dcids":["medicalStatus"]}], + ["FakeGraph.node_labels_batch",{"dcids":["LiveBirth","medicalStatus"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"measurementMethod"}], @@ -1321,13 +1321,13 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/BirthEvent_MedicalStatus-LiveBirth"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction","Amount_EconomicActivity_GrossDomesticProduction_Nominal","Amount_EconomicActivity_GrossDomesticProduction_Nominal_AsAFractionOf_Count_Person","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","Amount_EconomicActivity_GrossDomesticProduction_RealValue","Annual_Amount_EconomicActivity_GrossDomesticProduction","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","InflationAdjustedGDP","Nominal_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","Quarterly_Amount_EconomicActivity_GrossDomesticProduction","RealValue_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction"],"entities":["country/IND"],"needs_dates":false,"deadline":"set"}], + ["FakeGraph.node_arcs_batch",{"dcids":["Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth","dc/g/BirthEvent_MedicalStatus-LiveBirth"],"deadline":"set"}], ["FakeGraph.node_labels_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal","EconomicActivity","GrossDomesticProduction","Nominal","activitySource","amount","country/IND","measuredValue"]}], ["FakeGraph.node_types_batch",{"dcids":["country/IND"]}], ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"],"entities":["country/IND"],"needs_dates":true}], - ["FakeGraph.node_arcs_batch",{"dcids":["Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth","dc/g/BirthEvent_MedicalStatus-LiveBirth"],"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["FertilityRate_Person_Female"]}], - ["FakeGraph.node_labels_batch",{"dcids":["gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Female","gender"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"measurementMethod"}], @@ -1371,7 +1371,7 @@ ], "test_identity_entry_resolves_definite": [ ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], - ["FakeGraph.node_labels_batch",{"dcids":["activitySource"]}], + ["FakeGraph.node_labels_batch",{"dcids":["GrossDomesticProduction","activitySource"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/IND"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"measurementMethod"}], @@ -1388,7 +1388,7 @@ "test_launch_trailing_tasks_guards_against_relaunching_a_live_parsed_task": [], "test_malformed_entry_fails_whole_request_with_repointed_pointer_and_missing_refs_detail": [ ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], - ["FakeGraph.node_labels_batch",{"dcids":["activitySource"]}], + ["FakeGraph.node_labels_batch",{"dcids":["GrossDomesticProduction","activitySource"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/IND"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"measurementMethod"}], @@ -1396,18 +1396,18 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"],"entities":["country/IND"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/EconomicActivity_ActivitySource-GrossDomesticProduction"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction","Amount_EconomicActivity_GrossDomesticProduction_Nominal_AsAFractionOf_Count_Person","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","Amount_EconomicActivity_GrossDomesticProduction_RealValue","Annual_Amount_EconomicActivity_GrossDomesticProduction","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","InflationAdjustedGDP","Nominal_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","Quarterly_Amount_EconomicActivity_GrossDomesticProduction","RealValue_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","dc/g/EconomicActivity_ActivitySource-GrossDomesticProduction"],"deadline":"set"}], - ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], - ["FakeGraph.node_labels_batch",{"dcids":["activitySource"]}], - ["FakeGraph.node_labels_batch",{"dcids":["country/ZZZ_ALSO_NOT_A_PLACE","country/ZZZ_NOT_A_PLACE"]}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction","Amount_EconomicActivity_GrossDomesticProduction_Nominal","Amount_EconomicActivity_GrossDomesticProduction_Nominal_AsAFractionOf_Count_Person","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","Amount_EconomicActivity_GrossDomesticProduction_RealValue","Annual_Amount_EconomicActivity_GrossDomesticProduction","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","InflationAdjustedGDP","Nominal_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","Quarterly_Amount_EconomicActivity_GrossDomesticProduction","RealValue_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction"],"entities":["country/IND"],"needs_dates":false,"deadline":"set"}], ["FakeGraph.node_labels_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal","EconomicActivity","GrossDomesticProduction","Nominal","activitySource","amount","country/IND","measuredValue"]}], ["FakeGraph.node_types_batch",{"dcids":["country/IND"]}], ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"],"entities":["country/IND"],"needs_dates":true}] + ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"],"entities":["country/IND"],"needs_dates":true}], + ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], + ["FakeGraph.node_labels_batch",{"dcids":["GrossDomesticProduction","activitySource"]}], + ["FakeGraph.node_labels_batch",{"dcids":["country/ZZZ_ALSO_NOT_A_PLACE","country/ZZZ_NOT_A_PLACE"]}] ], "test_n_equals_1_resolves_like_a_single_resubmit": [ ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], - ["FakeGraph.node_labels_batch",{"dcids":["activitySource"]}], + ["FakeGraph.node_labels_batch",{"dcids":["GrossDomesticProduction","activitySource"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/IND"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"measurementMethod"}], @@ -1421,7 +1421,7 @@ ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"],"entities":["country/IND"],"needs_dates":true}], ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], - ["FakeGraph.node_labels_batch",{"dcids":["activitySource"]}], + ["FakeGraph.node_labels_batch",{"dcids":["GrossDomesticProduction","activitySource"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/IND"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"measurementMethod"}], @@ -1548,6 +1548,8 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/BirthEvent_MedicalStatus-LiveBirth"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth","dc/g/BirthEvent_MedicalStatus-LiveBirth"],"deadline":"set"}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person"],"entities":["country/ETH"],"needs_dates":true}], + ["FakeGraph.node_labels_batch",{"dcids":["WikidataPopulation","WorldBankSubnationalPopulationEstimate","dc/base/EthiopiaStatistics","dc/base/Subnational_Demographics_Stats","dc/base/WikidataPopulation","dc/base/WorldDevelopmentIndicators"]}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], ["FakeGraph.node_labels_batch",{"dcids":["BirthEvent","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Count_Person","LiveBirth","count","country/ETH","measuredValue","medicalStatus"]}], ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], @@ -1555,13 +1557,11 @@ ["FakeGraph.node_labels_batch",{"dcids":["Female","FertilityRate_Person_Female","Person","country/ETH","dc/base/WorldDevelopmentIndicators","fertilityRate","gender","measuredValue"]}], ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person"],"entities":["country/ETH"],"needs_dates":true}], - ["FakeGraph.node_labels_batch",{"dcids":["WikidataPopulation","WorldBankSubnationalPopulationEstimate","dc/base/EthiopiaStatistics","dc/base/Subnational_Demographics_Stats","dc/base/WikidataPopulation","dc/base/WorldDevelopmentIndicators"]}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.node_labels_batch",{"dcids":["dc/base/WorldDevelopmentIndicators"]}], ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], - ["FakeGraph.node_labels_batch",{"dcids":["activitySource"]}], + ["FakeGraph.node_labels_batch",{"dcids":["GrossDomesticProduction","activitySource"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/IND"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"measurementMethod"}], @@ -1570,7 +1570,7 @@ ["FakeGraph.node_subjects_many",{"dcids":["dc/g/EconomicActivity_ActivitySource-GrossDomesticProduction"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction","Amount_EconomicActivity_GrossDomesticProduction_Nominal_AsAFractionOf_Count_Person","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","Amount_EconomicActivity_GrossDomesticProduction_RealValue","Annual_Amount_EconomicActivity_GrossDomesticProduction","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","InflationAdjustedGDP","Nominal_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","Quarterly_Amount_EconomicActivity_GrossDomesticProduction","RealValue_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","dc/g/EconomicActivity_ActivitySource-GrossDomesticProduction"],"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"]}], - ["FakeGraph.node_labels_batch",{"dcids":["medicalStatus"]}], + ["FakeGraph.node_labels_batch",{"dcids":["LiveBirth","medicalStatus"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"measurementMethod"}], @@ -1578,14 +1578,14 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/BirthEvent_MedicalStatus-LiveBirth"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction","Amount_EconomicActivity_GrossDomesticProduction_Nominal","Amount_EconomicActivity_GrossDomesticProduction_Nominal_AsAFractionOf_Count_Person","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","Amount_EconomicActivity_GrossDomesticProduction_RealValue","Annual_Amount_EconomicActivity_GrossDomesticProduction","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","InflationAdjustedGDP","Nominal_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","Quarterly_Amount_EconomicActivity_GrossDomesticProduction","RealValue_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction"],"entities":["country/IND"],"needs_dates":false,"deadline":"set"}], + ["FakeGraph.node_arcs_batch",{"dcids":["Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth","dc/g/BirthEvent_MedicalStatus-LiveBirth"],"deadline":"set"}], ["FakeGraph.node_labels_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal","EconomicActivity","GrossDomesticProduction","Nominal","activitySource","amount","country/IND","measuredValue"]}], ["FakeGraph.node_types_batch",{"dcids":["country/IND"]}], ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"],"entities":["country/IND"],"needs_dates":true}], - ["FakeGraph.node_arcs_batch",{"dcids":["Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth","dc/g/BirthEvent_MedicalStatus-LiveBirth"],"deadline":"set"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Mean_BirthWeight_BirthEvent_LiveBirth","Mean_IntervalSinceLastBirth_BirthEvent_LiveBirth","Mean_IntervalSinceLastPregnancyOutcomeNotLiveBirth_BirthEvent_LiveBirth","Mean_LmpGestationalAge_BirthEvent_LiveBirth","Mean_MothersAge_BirthEvent_LiveBirth","Mean_OeGestationalAge_BirthEvent_LiveBirth","Mean_PrePregnancyBMI_BirthEvent_LiveBirth","Mean_PrenatalVisitCount_BirthEvent_LiveBirth"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["FertilityRate_Person_Female"]}], - ["FakeGraph.node_labels_batch",{"dcids":["gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Female","gender"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"measurementMethod"}], @@ -1593,16 +1593,16 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/Health"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["LifeExpectancy_Person","dc/g/Health"],"deadline":"set"}], + ["FakeGraph.node_labels_batch",{"dcids":["BirthEvent","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Count_Person","LiveBirth","count","country/ETH","measuredValue","medicalStatus"]}], + ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female","LifeExpectancy_Person"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], ["FakeGraph.node_labels_batch",{"dcids":["Female","FertilityRate_Person_Female","Person","country/ETH","dc/base/WorldDevelopmentIndicators","fertilityRate","gender","measuredValue"]}], ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":true}], - ["FakeGraph.node_labels_batch",{"dcids":["dc/base/WorldDevelopmentIndicators"]}], - ["FakeGraph.node_labels_batch",{"dcids":["BirthEvent","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","Count_Person","LiveBirth","count","country/ETH","measuredValue","medicalStatus"]}], - ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], - ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"],"entities":["country/ETH"],"needs_dates":true}] + ["FakeGraph.node_labels_batch",{"dcids":["dc/base/WorldDevelopmentIndicators"]}] ], "test_parsed_entry_dcid_pins[one_text_both_members_values_refuses_request_invalid]": [], "test_parsed_entry_dcid_pins[one_text_two_dcids_refuses_request_invalid]": [], @@ -2242,16 +2242,6 @@ ["FakeGraph.node_arcs",{"dcid":"Count_Person_WithDisability_WhiteAlone"}], ["FakeGraph.node_arcs",{"dcid":"Count_Person_WithDisability_AsianAlone"}], ["FakeGraph.node_arcs",{"dcid":"Count_Person_25To64Years_UpperSecondaryEducationOrHigher_Male_AsAFractionOfCount_Person_25To64Years_Male"}], - ["FakeGraph.node_arcs",{"dcid":"FertilityRate_Person_Female"}], - ["FakeGraph.node_arcs",{"dcid":"FertilityRate_Person_Female"}], - ["FakeGraph.node_arcs_batch",{"dcids":["FertilityRate_Person_Female"]}], - ["FakeGraph.node_labels_batch",{"dcids":["gender"]}], - ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], - ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"provenance"}], - ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"measurementMethod"}], - ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"unit"}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":false}], - ["FakeGraph.node_subjects_many",{"dcids":["dc/g/Health"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs",{"dcid":"Count_Person_BelowPovertyLevelInThePast12Months_AsFractionOf_Count_Person"}], ["FakeGraph.node_arcs",{"dcid":"Count_Person_DetailedPrimarySchool"}], ["FakeGraph.node_arcs",{"dcid":"Count_Person_PlaceOfBirthAsia"}], @@ -2723,10 +2713,20 @@ ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_Person_EducationalAttainment_9ThTo12ThGradeNoDiploma","axis":"unit"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person","Count_Person_0To4Years_Female","Count_Person_0To4Years_Male","Count_Person_0To4Years_Rural","Count_Person_10To14Years_Rural","Count_Person_15OrMoreYears","Count_Person_15To19Years_Rural","Count_Person_18OrMoreYears","Count_Person_18OrMoreYears_Civilian","Count_Person_20To24Years_Rural","Count_Person_25OrMoreYears_EducationalAttainment10ThGrade_Female","Count_Person_25OrMoreYears_EducationalAttainment11ThGrade_Female","Count_Person_25OrMoreYears_EducationalAttainment12ThGradeNoDiploma_Female","Count_Person_25OrMoreYears_EducationalAttainment9ThGrade_Female","Count_Person_25OrMoreYears_EducationalAttainmentAssociatesDegree_Female","Count_Person_25OrMoreYears_EducationalAttainmentBachelorsDegreeOrHigher_Female","Count_Person_25OrMoreYears_EducationalAttainmentBachelorsDegree_Female","Count_Person_25OrMoreYears_EducationalAttainmentNoSchoolingCompleted_Female","Count_Person_25OrMoreYears_EducationalAttainmentNoSchoolingCompleted_Male","Count_Person_25OrMoreYears_EducationalAttainmentNurseryTo4ThGrade_Female","Count_Person_25OrMoreYears_EducationalAttainmentSomeCollegeLessThan1Year_Female","Count_Person_25To29Years_Rural","Count_Person_30To34Years_Rural","Count_Person_35To39Years_Rural","Count_Person_40To44Years_Rural","Count_Person_45To49Years_Rural","Count_Person_50To54Years_Rural","Count_Person_55To59Years_Rural","Count_Person_5OrMoreYears_ForeignBorn","Count_Person_5OrMoreYears_ResidesInHousehold","Count_Person_5To9Years_Rural","Count_Person_60To64Years_Rural","Count_Person_65OrMoreYears","Count_Person_65To69Years_Rural","Count_Person_70To74Years_Rural","Count_Person_75To79Years_Rural","Count_Person_80To84Years_Rural","Count_Person_85OrMoreYears_Female","Count_Person_85OrMoreYears_Male","Count_Person_85OrMoreYears_Rural","Count_Person_AmericanIndianOrAlaskaNativeAlone","Count_Person_AsianAlone","Count_Person_AsianOrPacificIslander","Count_Person_BlackOrAfricanAmericanAlone","Count_Person_BlackOrAfricanAmericanAlone_ResidesInGroupQuarters","Count_Person_BornInOtherStateInTheUnitedStates","Count_Person_BornInStateOfResidence","Count_Person_EducationalAttainment1StGrade","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher","Count_Person_EducationalAttainmentKindergarten","Count_Person_EducationalAttainmentLessThanHighSchoolGraduate","Count_Person_EducationalAttainmentNoSchoolingCompleted","Count_Person_EducationalAttainmentNurserySchool","Count_Person_EducationalAttainmentPrimarySchool","Count_Person_EducationalAttainment_9ThTo12ThGradeNoDiploma","Count_Person_EducationalAttainment_LessThan9ThGrade","Count_Person_EducationalAttainment_LessThanHighSchoolDiploma","Count_Person_Employed","Count_Person_Female","Count_Person_Female_AsianAlone","Count_Person_Female_AsianOrPacificIslander","Count_Person_Female_BlackOrAfricanAmericanAlone","Count_Person_Female_ForeignBorn","Count_Person_Female_ForeignBorn_PlaceOfBirthAfrica","Count_Person_Female_ForeignBorn_PlaceOfBirthEurope","Count_Person_Female_ForeignBorn_PlaceOfBirthNorthernWesternEurope","Count_Person_Female_HispanicOrLatino","Count_Person_Female_NonWhite","Count_Person_Female_NotHispanicOrLatino","Count_Person_Female_ResidesInNursingFacilities","Count_Person_Female_TwoOrMoreRaces","Count_Person_Female_WhiteAlone","Count_Person_ForeignBorn_PlaceOfBirthAfrica","Count_Person_HispanicOrLatino","Count_Person_Illiterate","Count_Person_Illiterate_Female","Count_Person_Illiterate_Male","Count_Person_Illiterate_Rural","Count_Person_Illiterate_Urban","Count_Person_Intersex","Count_Person_Literate","Count_Person_Literate_Female","Count_Person_Literate_Male","Count_Person_Literate_Rural","Count_Person_Literate_Urban","Count_Person_Male","Count_Person_Male_ForeignBorn_PlaceOfBirthAfrica","Count_Person_Male_ForeignBorn_PlaceOfBirthEurope","Count_Person_Male_WhiteAlone","Count_Person_Native","Count_Person_NonWhite","Count_Person_OneRace","Count_Person_PlaceOfBirthAsia","Count_Person_PlaceOfBirthEurope","Count_Person_ResidesInGroupQuarters","Count_Person_ResidesInHousehold","Count_Person_ResidesInNoninstitutionalizedGroupQuarters","Count_Person_Rural","Count_Person_Rural_Female","Count_Person_Rural_Male","Count_Person_Unemployed","Count_Person_Unemployed_Female","Count_Person_Unemployed_Male","Count_Person_Upto18Years","Count_Person_Urban","Count_Person_Urban_Female","Count_Person_Urban_Male","Count_Person_WhiteAlone","dc/29vy1f58gdm0g","dc/2xd2ktjs23hk3","dc/4j5t00e5s5el3","dc/jxd329x7k27fb","dc/nm9hcklgg5zb3","dc/scnp6d6sv1jqd","dc/vxhvgzjhwwmj6","dc/xdfcsgf05b4mc","indianCensus/Count_Person_Religion_Buddhist_Female","indianCensus/Count_Person_Religion_Buddhist_Illiterate","indianCensus/Count_Person_Religion_Buddhist_Literate","indianCensus/Count_Person_Religion_Buddhist_Rural","indianCensus/Count_Person_Religion_Buddhist_Urban","indianCensus/Count_Person_Religion_Christian","indianCensus/Count_Person_Religion_Christian_Female","indianCensus/Count_Person_Religion_Christian_Illiterate","indianCensus/Count_Person_Religion_Christian_Literate","indianCensus/Count_Person_Religion_Christian_Rural","indianCensus/Count_Person_Religion_Christian_Urban","indianCensus/Count_Person_Religion_Christian_YearsUpto6","indianCensus/Count_Person_Religion_Christian_YearsUpto6_Urban","indianCensus/Count_Person_Religion_Hindu_Female","indianCensus/Count_Person_Religion_Hindu_Literate","indianCensus/Count_Person_Religion_Hindu_Rural","indianCensus/Count_Person_Religion_Hindu_Urban","indianCensus/Count_Person_Religion_Hindu_YearsUpto6_Urban","indianCensus/Count_Person_Religion_Jain_Literate","indianCensus/Count_Person_Religion_Jain_Urban","indianCensus/Count_Person_Religion_Muslim","indianCensus/Count_Person_Religion_Muslim_Female","indianCensus/Count_Person_Religion_Muslim_Illiterate","indianCensus/Count_Person_Religion_Muslim_Literate","indianCensus/Count_Person_Religion_Muslim_Rural","indianCensus/Count_Person_Religion_Muslim_Urban","indianCensus/Count_Person_Religion_Muslim_YearsUpto6","indianCensus/Count_Person_Religion_Muslim_YearsUpto6_Rural","indianCensus/Count_Person_Religion_Muslim_YearsUpto6_Urban","indianCensus/Count_Person_Religion_Sikh_Illiterate","indianCensus/Count_Person_Religion_Sikh_Literate","indianCensus/Count_Person_Religion_Sikh_Rural","indianCensus/Count_Person_Religion_Sikh_Urban"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/Demographics"],"property":"memberOf","max_results":40,"deadline":"set"}], - ["FakeGraph.node_arcs_batch",{"dcids":["LifeExpectancy_Person","dc/g/Health"],"deadline":"set"}], + ["FakeGraph.node_arcs",{"dcid":"FertilityRate_Person_Female"}], + ["FakeGraph.node_arcs",{"dcid":"FertilityRate_Person_Female"}], + ["FakeGraph.node_arcs_batch",{"dcids":["FertilityRate_Person_Female"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Female","gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], + ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"provenance"}], + ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"measurementMethod"}], + ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"unit"}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":false}], + ["FakeGraph.node_subjects_many",{"dcids":["dc/g/Health"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Count_Company","Count_Person_5To17Years_Poor","Count_Person_AsAFractionOf_Count_Household","Count_Person_InfrastructureProject_BrazilRuralDevelopmentProgram","Count_Person_PerArea","GenderIncomeInequality_Person_15OrMoreYears_WithIncome","GrowthRate_Count_Person","IncrementalCount_Person","LifeExpectancy_Person","MarginOfError_Count_Person","Mean_Age_Person","Mean_Count_Person","Mean_IncrementalCount_Person","Mean_MealCost_Person_FoodSecure","Median_Age_Person","Median_Income_Person","dc/570wbk7b8bv31","dc/9y76nyw44ne76","dc/fkrq43e9fptd3","dc/g/Demographics","dc/qnbqqvcpmlrx8"],"deadline":"set"}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female","LifeExpectancy_Person"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Company","Count_Person","Count_Person_5To17Years_Poor","Count_Person_AsAFractionOf_Count_Household","Count_Person_InfrastructureProject_BrazilRuralDevelopmentProgram","Count_Person_PerArea","GenderIncomeInequality_Person_15OrMoreYears_WithIncome","GrowthRate_Count_Person","IncrementalCount_Person","LifeExpectancy_Person","MarginOfError_Count_Person","Mean_Age_Person","Mean_Count_Person","Mean_IncrementalCount_Person","Mean_MealCost_Person_FoodSecure","Median_Age_Person","Median_Income_Person","dc/570wbk7b8bv31","dc/9y76nyw44ne76","dc/fkrq43e9fptd3","dc/qnbqqvcpmlrx8"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], + ["FakeGraph.node_arcs_batch",{"dcids":["LifeExpectancy_Person","dc/g/Health"],"deadline":"set"}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female","LifeExpectancy_Person"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], ["FakeGraph.node_labels_batch",{"dcids":["Female","FertilityRate_Person_Female","Person","country/ETH","dc/base/WorldDevelopmentIndicators","fertilityRate","gender","measuredValue"]}], ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], @@ -3230,15 +3230,15 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person","Count_Person_0To4Years_Female","Count_Person_0To4Years_Male","Count_Person_0To4Years_Rural","Count_Person_10To14Years_Rural","Count_Person_15OrMoreYears","Count_Person_15To19Years_Rural","Count_Person_18OrMoreYears","Count_Person_18OrMoreYears_Civilian","Count_Person_20To24Years_Rural","Count_Person_25OrMoreYears_EducationalAttainment10ThGrade_Female","Count_Person_25OrMoreYears_EducationalAttainment11ThGrade_Female","Count_Person_25OrMoreYears_EducationalAttainment12ThGradeNoDiploma_Female","Count_Person_25OrMoreYears_EducationalAttainment9ThGrade_Female","Count_Person_25OrMoreYears_EducationalAttainmentAssociatesDegree_Female","Count_Person_25OrMoreYears_EducationalAttainmentBachelorsDegreeOrHigher_Female","Count_Person_25OrMoreYears_EducationalAttainmentBachelorsDegree_Female","Count_Person_25OrMoreYears_EducationalAttainmentNoSchoolingCompleted_Female","Count_Person_25OrMoreYears_EducationalAttainmentNoSchoolingCompleted_Male","Count_Person_25OrMoreYears_EducationalAttainmentNurseryTo4ThGrade_Female","Count_Person_25OrMoreYears_EducationalAttainmentSomeCollegeLessThan1Year_Female","Count_Person_25To29Years_Rural","Count_Person_30To34Years_Rural","Count_Person_35To39Years_Rural","Count_Person_40To44Years_Rural","Count_Person_45To49Years_Rural","Count_Person_50To54Years_Rural","Count_Person_55To59Years_Rural","Count_Person_5OrMoreYears_ForeignBorn","Count_Person_5OrMoreYears_ResidesInHousehold","Count_Person_5To9Years_Rural","Count_Person_60To64Years_Rural","Count_Person_65OrMoreYears","Count_Person_65To69Years_Rural","Count_Person_70To74Years_Rural","Count_Person_75To79Years_Rural","Count_Person_80To84Years_Rural","Count_Person_85OrMoreYears_Female","Count_Person_85OrMoreYears_Male","Count_Person_85OrMoreYears_Rural","Count_Person_AmericanIndianOrAlaskaNativeAlone","Count_Person_AsianAlone","Count_Person_AsianOrPacificIslander","Count_Person_BlackOrAfricanAmericanAlone","Count_Person_BlackOrAfricanAmericanAlone_ResidesInGroupQuarters","Count_Person_BornInOtherStateInTheUnitedStates","Count_Person_BornInStateOfResidence","Count_Person_EducationalAttainment1StGrade","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher","Count_Person_EducationalAttainmentKindergarten","Count_Person_EducationalAttainmentLessThanHighSchoolGraduate","Count_Person_EducationalAttainmentNoSchoolingCompleted","Count_Person_EducationalAttainmentNurserySchool","Count_Person_EducationalAttainmentPrimarySchool","Count_Person_EducationalAttainment_9ThTo12ThGradeNoDiploma","Count_Person_EducationalAttainment_LessThan9ThGrade","Count_Person_EducationalAttainment_LessThanHighSchoolDiploma","Count_Person_Employed","Count_Person_Female","Count_Person_Female_AsianAlone","Count_Person_Female_AsianOrPacificIslander","Count_Person_Female_BlackOrAfricanAmericanAlone","Count_Person_Female_ForeignBorn","Count_Person_Female_ForeignBorn_PlaceOfBirthAfrica","Count_Person_Female_ForeignBorn_PlaceOfBirthEurope","Count_Person_Female_ForeignBorn_PlaceOfBirthNorthernWesternEurope","Count_Person_Female_HispanicOrLatino","Count_Person_Female_NonWhite","Count_Person_Female_NotHispanicOrLatino","Count_Person_Female_ResidesInNursingFacilities","Count_Person_Female_TwoOrMoreRaces","Count_Person_Female_WhiteAlone","Count_Person_ForeignBorn_PlaceOfBirthAfrica","Count_Person_HispanicOrLatino","Count_Person_Illiterate","Count_Person_Illiterate_Female","Count_Person_Illiterate_Male","Count_Person_Illiterate_Rural","Count_Person_Illiterate_Urban","Count_Person_Intersex","Count_Person_Literate","Count_Person_Literate_Female","Count_Person_Literate_Male","Count_Person_Literate_Rural","Count_Person_Literate_Urban","Count_Person_Male","Count_Person_Male_ForeignBorn_PlaceOfBirthAfrica","Count_Person_Male_ForeignBorn_PlaceOfBirthEurope","Count_Person_Male_WhiteAlone","Count_Person_Native","Count_Person_NonWhite","Count_Person_OneRace","Count_Person_PlaceOfBirthAsia","Count_Person_PlaceOfBirthEurope","Count_Person_ResidesInGroupQuarters","Count_Person_ResidesInHousehold","Count_Person_ResidesInNoninstitutionalizedGroupQuarters","Count_Person_Rural","Count_Person_Rural_Female","Count_Person_Rural_Male","Count_Person_Unemployed","Count_Person_Unemployed_Female","Count_Person_Unemployed_Male","Count_Person_Upto18Years","Count_Person_Urban","Count_Person_Urban_Female","Count_Person_Urban_Male","Count_Person_WhiteAlone","dc/29vy1f58gdm0g","dc/2xd2ktjs23hk3","dc/4j5t00e5s5el3","dc/jxd329x7k27fb","dc/nm9hcklgg5zb3","dc/scnp6d6sv1jqd","dc/vxhvgzjhwwmj6","dc/xdfcsgf05b4mc","indianCensus/Count_Person_Religion_Buddhist_Female","indianCensus/Count_Person_Religion_Buddhist_Illiterate","indianCensus/Count_Person_Religion_Buddhist_Literate","indianCensus/Count_Person_Religion_Buddhist_Rural","indianCensus/Count_Person_Religion_Buddhist_Urban","indianCensus/Count_Person_Religion_Christian","indianCensus/Count_Person_Religion_Christian_Female","indianCensus/Count_Person_Religion_Christian_Illiterate","indianCensus/Count_Person_Religion_Christian_Literate","indianCensus/Count_Person_Religion_Christian_Rural","indianCensus/Count_Person_Religion_Christian_Urban","indianCensus/Count_Person_Religion_Christian_YearsUpto6","indianCensus/Count_Person_Religion_Christian_YearsUpto6_Urban","indianCensus/Count_Person_Religion_Hindu_Female","indianCensus/Count_Person_Religion_Hindu_Literate","indianCensus/Count_Person_Religion_Hindu_Rural","indianCensus/Count_Person_Religion_Hindu_Urban","indianCensus/Count_Person_Religion_Hindu_YearsUpto6_Urban","indianCensus/Count_Person_Religion_Jain_Literate","indianCensus/Count_Person_Religion_Jain_Urban","indianCensus/Count_Person_Religion_Muslim","indianCensus/Count_Person_Religion_Muslim_Female","indianCensus/Count_Person_Religion_Muslim_Illiterate","indianCensus/Count_Person_Religion_Muslim_Literate","indianCensus/Count_Person_Religion_Muslim_Rural","indianCensus/Count_Person_Religion_Muslim_Urban","indianCensus/Count_Person_Religion_Muslim_YearsUpto6","indianCensus/Count_Person_Religion_Muslim_YearsUpto6_Rural","indianCensus/Count_Person_Religion_Muslim_YearsUpto6_Urban","indianCensus/Count_Person_Religion_Sikh_Illiterate","indianCensus/Count_Person_Religion_Sikh_Literate","indianCensus/Count_Person_Religion_Sikh_Rural","indianCensus/Count_Person_Religion_Sikh_Urban"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/Demographics"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Count_Company","Count_Person_5To17Years_Poor","Count_Person_AsAFractionOf_Count_Household","Count_Person_InfrastructureProject_BrazilRuralDevelopmentProgram","Count_Person_PerArea","GenderIncomeInequality_Person_15OrMoreYears_WithIncome","GrowthRate_Count_Person","IncrementalCount_Person","LifeExpectancy_Person","MarginOfError_Count_Person","Mean_Age_Person","Mean_Count_Person","Mean_IncrementalCount_Person","Mean_MealCost_Person_FoodSecure","Median_Age_Person","Median_Income_Person","dc/570wbk7b8bv31","dc/9y76nyw44ne76","dc/fkrq43e9fptd3","dc/g/Demographics","dc/qnbqqvcpmlrx8"],"deadline":"set"}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Company","Count_Person","Count_Person_5To17Years_Poor","Count_Person_AsAFractionOf_Count_Household","Count_Person_InfrastructureProject_BrazilRuralDevelopmentProgram","Count_Person_PerArea","GenderIncomeInequality_Person_15OrMoreYears_WithIncome","GrowthRate_Count_Person","IncrementalCount_Person","LifeExpectancy_Person","MarginOfError_Count_Person","Mean_Age_Person","Mean_Count_Person","Mean_IncrementalCount_Person","Mean_MealCost_Person_FoodSecure","Median_Age_Person","Median_Income_Person","dc/570wbk7b8bv31","dc/9y76nyw44ne76","dc/fkrq43e9fptd3","dc/qnbqqvcpmlrx8"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["FertilityRate_Person_Female"]}], - ["FakeGraph.node_labels_batch",{"dcids":["gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Female","gender"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/ETH"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"measurementMethod"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"unit"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/Health"],"property":"memberOf","max_results":40,"deadline":"set"}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Company","Count_Person","Count_Person_5To17Years_Poor","Count_Person_AsAFractionOf_Count_Household","Count_Person_InfrastructureProject_BrazilRuralDevelopmentProgram","Count_Person_PerArea","GenderIncomeInequality_Person_15OrMoreYears_WithIncome","GrowthRate_Count_Person","IncrementalCount_Person","LifeExpectancy_Person","MarginOfError_Count_Person","Mean_Age_Person","Mean_Count_Person","Mean_IncrementalCount_Person","Mean_MealCost_Person_FoodSecure","Median_Age_Person","Median_Income_Person","dc/570wbk7b8bv31","dc/9y76nyw44ne76","dc/fkrq43e9fptd3","dc/qnbqqvcpmlrx8"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], ["FakeGraph.node_labels_batch",{"dcids":["1StGrade","AsianAlone","BlackOrAfricanAmericanAlone","BornInStateOfResidence","Civilian","Count_Person","Employed","Female","ForeignBorn","GroupQuarters","HispanicOrLatino","Islam","Literate","Male","NorthernWesternEurope","OneRace","Person","TwoOrMoreRaces","USC_EmployedAndWorking","USC_ForeignBorn","Urban","WhiteAlone","Years15Onwards","Years16Onwards","Years16To64","Years25Onwards","Years3Onwards","Years85Onwards","age","armedForcesStatus","asia","count","country/ETH","dc/base/WorldDevelopmentIndicators","educationalAttainment","employment","europe","gender","literacyStatus","measuredValue","nativity","placeOfBirth","placeOfResidenceClassification","race","religion","residenceType"]}], ["FakeGraph.node_types_batch",{"dcids":["country/ETH"]}], ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], @@ -3254,7 +3254,7 @@ ], "test_resolution_outcome_is_a_200_part_siblings_unaffected": [ ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], - ["FakeGraph.node_labels_batch",{"dcids":["activitySource"]}], + ["FakeGraph.node_labels_batch",{"dcids":["GrossDomesticProduction","activitySource"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/IND"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"measurementMethod"}], @@ -3262,19 +3262,19 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"],"entities":["country/IND"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/EconomicActivity_ActivitySource-GrossDomesticProduction"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction","Amount_EconomicActivity_GrossDomesticProduction_Nominal_AsAFractionOf_Count_Person","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","Amount_EconomicActivity_GrossDomesticProduction_RealValue","Annual_Amount_EconomicActivity_GrossDomesticProduction","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","InflationAdjustedGDP","Nominal_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","Quarterly_Amount_EconomicActivity_GrossDomesticProduction","RealValue_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","dc/g/EconomicActivity_ActivitySource-GrossDomesticProduction"],"deadline":"set"}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction","Amount_EconomicActivity_GrossDomesticProduction_Nominal","Amount_EconomicActivity_GrossDomesticProduction_Nominal_AsAFractionOf_Count_Person","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","Amount_EconomicActivity_GrossDomesticProduction_RealValue","Annual_Amount_EconomicActivity_GrossDomesticProduction","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","InflationAdjustedGDP","Nominal_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","Quarterly_Amount_EconomicActivity_GrossDomesticProduction","RealValue_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction"],"entities":["country/IND"],"needs_dates":false,"deadline":"set"}], + ["FakeGraph.node_labels_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal","EconomicActivity","GrossDomesticProduction","Nominal","activitySource","amount","country/IND","measuredValue"]}], + ["FakeGraph.node_types_batch",{"dcids":["country/IND"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"],"entities":["country/IND"],"needs_dates":true}], ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"]}], - ["FakeGraph.node_labels_batch",{"dcids":["activitySource"]}], + ["FakeGraph.node_labels_batch",{"dcids":["GrossDomesticProduction","activitySource"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/USA"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"measurementMethod"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Amount_EconomicActivity_GrossDomesticProduction_Nominal","axis":"unit"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"],"entities":["country/USA"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/EconomicActivity_ActivitySource-GrossDomesticProduction"],"property":"memberOf","max_results":40,"deadline":"set"}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction","Amount_EconomicActivity_GrossDomesticProduction_Nominal","Amount_EconomicActivity_GrossDomesticProduction_Nominal_AsAFractionOf_Count_Person","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","Amount_EconomicActivity_GrossDomesticProduction_RealValue","Annual_Amount_EconomicActivity_GrossDomesticProduction","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","InflationAdjustedGDP","Nominal_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","Quarterly_Amount_EconomicActivity_GrossDomesticProduction","RealValue_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction"],"entities":["country/IND"],"needs_dates":false,"deadline":"set"}], - ["FakeGraph.node_labels_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal","EconomicActivity","GrossDomesticProduction","Nominal","activitySource","amount","country/IND","measuredValue"]}], - ["FakeGraph.node_types_batch",{"dcids":["country/IND"]}], - ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction_Nominal"],"entities":["country/IND"],"needs_dates":true}], ["FakeGraph.node_arcs_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction","Amount_EconomicActivity_GrossDomesticProduction_Nominal_AsAFractionOf_Count_Person","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","Amount_EconomicActivity_GrossDomesticProduction_RealValue","Annual_Amount_EconomicActivity_GrossDomesticProduction","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","InflationAdjustedGDP","Nominal_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","Quarterly_Amount_EconomicActivity_GrossDomesticProduction","RealValue_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","dc/g/EconomicActivity_ActivitySource-GrossDomesticProduction"],"deadline":"set"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Amount_EconomicActivity_GrossDomesticProduction","Amount_EconomicActivity_GrossDomesticProduction_Nominal","Amount_EconomicActivity_GrossDomesticProduction_Nominal_AsAFractionOf_Count_Person","Amount_EconomicActivity_GrossDomesticProduction_Nominal_PerCapita","Amount_EconomicActivity_GrossDomesticProduction_RealValue","Annual_Amount_EconomicActivity_GrossDomesticProduction","GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","InflationAdjustedGDP","Nominal_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction","Quarterly_Amount_EconomicActivity_GrossDomesticProduction","RealValue_GrowthRate_Amount_EconomicActivity_GrossDomesticProduction"],"entities":["country/USA"],"needs_dates":false,"deadline":"set"}], ["FakeGraph.node_labels_batch",{"dcids":["Amount_EconomicActivity_GrossDomesticProduction_Nominal","EconomicActivity","GrossDomesticProduction","Nominal","activitySource","amount","country/USA","measuredValue"]}], @@ -3300,8 +3300,8 @@ ["FakeGraph.node_arcs_many",{"dcids":["Country"],"properties":["subClassOf"],"deadline":null}], ["FakeGraph.node_arcs_many",{"dcids":["Country"],"properties":["subClassOf"],"deadline":null}], ["FakeGraph.node_arcs_many",{"dcids":["AdministrativeArea"],"properties":["subClassOf"],"deadline":null}], - ["FakeGraph.node_labels_batch",{"dcids":["AsianAlone","AsianOrPacificIslander","BLS_Employed","BachelorsDegreeOrHigher","BlackOrAfricanAmericanAlone","BornInStateOfResidence","Civilian","Employed","Female","HispanicOrLatino","Household","Islam","Literate","Male","NonWhite","USC_ForeignBorn","USC_Native","Urban","WhiteAlone","Years18Onwards","Years25Onwards","Years5Onwards","Years65Onwards","YearsUpto18"]}], ["FakeGraph.node_arcs_many",{"dcids":["AdministrativeArea"],"properties":["subClassOf"],"deadline":null}], + ["FakeGraph.node_labels_batch",{"dcids":["AsianAlone","AsianOrPacificIslander","BLS_Employed","BachelorsDegreeOrHigher","BlackOrAfricanAmericanAlone","BornInStateOfResidence","Civilian","Employed","Female","HispanicOrLatino","Household","Islam","Literate","Male","NonWhite","USC_ForeignBorn","USC_Native","Urban","WhiteAlone","Years18Onwards","Years25Onwards","Years5Onwards","Years65Onwards","YearsUpto18"]}], ["FakeGraph.node_arcs_batch",{"dcids":["Count_BirthEvent_LiveBirth","Count_Person_15To19Years_Female","Count_Person_15To50Years_Female","Count_Person_16To50Years_Female","Count_Person_35To50Years_Female"]}], ["FakeGraph.node_labels_batch",{"dcids":["LiveBirth"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_Person","axis":"provenance"}], @@ -3374,6 +3374,13 @@ ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_Person_5OrMoreYears_ForeignBorn","axis":"measurementMethod"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_Person_5OrMoreYears_ForeignBorn","axis":"unit"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person","Count_Person_18OrMoreYears","Count_Person_18OrMoreYears_Civilian","Count_Person_5OrMoreYears_ForeignBorn","Count_Person_65OrMoreYears","Count_Person_AsianAlone","Count_Person_AsianOrPacificIslander","Count_Person_BlackOrAfricanAmericanAlone","Count_Person_BornInStateOfResidence","Count_Person_EducationalAttainmentBachelorsDegreeOrHigher","Count_Person_Employed","Count_Person_Female","Count_Person_HispanicOrLatino","Count_Person_Literate","Count_Person_Male","Count_Person_Native","Count_Person_NonWhite","Count_Person_ResidesInHousehold","Count_Person_Upto18Years","Count_Person_Urban","Count_Person_WhiteAlone","dc/nm9hcklgg5zb3","indianCensus/Count_Person_Religion_Muslim"],"entities":["country/ETH"],"needs_dates":false}], + ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_AsAFractionOfCount_Person","axis":"provenance"}], + ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_AsAFractionOfCount_Person","axis":"measurementMethod"}], + ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_AsAFractionOfCount_Person","axis":"unit"}], + ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"provenance"}], + ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"measurementMethod"}], + ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"unit"}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_AsAFractionOfCount_Person","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/Demographics"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["Count_Company","Count_Person_5To17Years_Poor","Count_Person_AsAFractionOf_Count_Household","Count_Person_InfrastructureProject_BrazilRuralDevelopmentProgram","Count_Person_PerArea","GenderIncomeInequality_Person_15OrMoreYears_WithIncome","GrowthRate_Count_Person","IncrementalCount_Person","LifeExpectancy_Person","MarginOfError_Count_Person","Mean_Age_Person","Mean_Count_Person","Mean_IncrementalCount_Person","Mean_MealCost_Person_FoodSecure","Median_Age_Person","Median_Income_Person","dc/570wbk7b8bv31","dc/9y76nyw44ne76","dc/fkrq43e9fptd3","dc/g/Demographics","dc/qnbqqvcpmlrx8"],"deadline":"set"}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Company","Count_Person","Count_Person_5To17Years_Poor","Count_Person_AsAFractionOf_Count_Household","Count_Person_InfrastructureProject_BrazilRuralDevelopmentProgram","Count_Person_PerArea","GenderIncomeInequality_Person_15OrMoreYears_WithIncome","GrowthRate_Count_Person","IncrementalCount_Person","LifeExpectancy_Person","MarginOfError_Count_Person","Mean_Age_Person","Mean_Count_Person","Mean_IncrementalCount_Person","Mean_MealCost_Person_FoodSecure","Median_Age_Person","Median_Income_Person","dc/570wbk7b8bv31","dc/9y76nyw44ne76","dc/fkrq43e9fptd3","dc/qnbqqvcpmlrx8"],"entities":["country/ETH"],"needs_dates":false,"deadline":"set"}], @@ -3382,13 +3389,6 @@ ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person"],"entities":["country/ETH"],"needs_dates":true}], ["FakeGraph.node_labels_batch",{"dcids":["WikidataPopulation","WorldBankSubnationalPopulationEstimate","dc/base/EthiopiaStatistics","dc/base/Subnational_Demographics_Stats","dc/base/WikidataPopulation","dc/base/WorldDevelopmentIndicators"]}], - ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_AsAFractionOfCount_Person","axis":"provenance"}], - ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_AsAFractionOfCount_Person","axis":"measurementMethod"}], - ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_AsAFractionOfCount_Person","axis":"unit"}], - ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"provenance"}], - ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"measurementMethod"}], - ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person","axis":"unit"}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_BirthEvent_AsAFractionOfCount_Person","Count_BirthEvent_LiveBirth_AsFractionOf_Count_Person"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.observation_facets_batch",{"stat_vars":["FertilityRate_Person_Female"],"entities":["country/ETH"],"needs_dates":false}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:FertilityRate_Person_Female","axis":"measurementMethod"}], @@ -3435,7 +3435,7 @@ ], "test_window_disagreement_blocks_fold_editing_to_agree_enables_it": [ ["FakeGraph.node_arcs_batch",{"dcids":["Count_Person_Female"]}], - ["FakeGraph.node_labels_batch",{"dcids":["gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Female","gender"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/IND"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_Person_Female","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_Person_Female","axis":"measurementMethod"}], @@ -3443,8 +3443,14 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person_Female"],"entities":["country/IND"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/Person_Gender-Female"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["LifeExpectancy_Person_Female","Median_Age_Person_Female","Median_Income_Person_15OrMoreYears_Female_WithIncome","UnemploymentRate_Person_Female","dc/g/Person_Gender-Female"],"deadline":"set"}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person_Female","LifeExpectancy_Person_Female","Median_Age_Person_Female","Median_Income_Person_15OrMoreYears_Female_WithIncome","UnemploymentRate_Person_Female"],"entities":["country/IND"],"needs_dates":false,"deadline":"set"}], + ["FakeGraph.node_labels_batch",{"dcids":["Count_Person_Female","Female","Person","count","country/IND","dc/base/IndiaCensus","gender","measuredValue"]}], + ["FakeGraph.node_types_batch",{"dcids":["country/IND"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person_Female"],"entities":["country/IND"],"needs_dates":true}], + ["FakeGraph.node_labels_batch",{"dcids":["dc/base/IndiaCensus"]}], ["FakeGraph.node_arcs_batch",{"dcids":["Count_Person_Male"]}], - ["FakeGraph.node_labels_batch",{"dcids":["gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Male","gender"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/IND"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_Person_Male","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_Person_Male","axis":"measurementMethod"}], @@ -3455,14 +3461,8 @@ ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person_Male"],"entities":["country/IND"],"needs_dates":true}], ["FakeGraph.node_labels_batch",{"dcids":["dc/base/IndiaCensus"]}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person_Female","LifeExpectancy_Person_Female","Median_Age_Person_Female","Median_Income_Person_15OrMoreYears_Female_WithIncome","UnemploymentRate_Person_Female"],"entities":["country/IND"],"needs_dates":false,"deadline":"set"}], - ["FakeGraph.node_labels_batch",{"dcids":["Count_Person_Female","Female","Person","count","country/IND","dc/base/IndiaCensus","gender","measuredValue"]}], - ["FakeGraph.node_types_batch",{"dcids":["country/IND"]}], - ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person_Female"],"entities":["country/IND"],"needs_dates":true}], - ["FakeGraph.node_labels_batch",{"dcids":["dc/base/IndiaCensus"]}], ["FakeGraph.node_arcs_batch",{"dcids":["Count_Person_Female"]}], - ["FakeGraph.node_labels_batch",{"dcids":["gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Female","gender"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/IND"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_Person_Female","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_Person_Female","axis":"measurementMethod"}], @@ -3470,14 +3470,8 @@ ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person_Female"],"entities":["country/IND"],"needs_dates":false}], ["FakeGraph.node_subjects_many",{"dcids":["dc/g/Person_Gender-Female"],"property":"memberOf","max_results":40,"deadline":"set"}], ["FakeGraph.node_arcs_batch",{"dcids":["LifeExpectancy_Person_Female","Median_Age_Person_Female","Median_Income_Person_15OrMoreYears_Female_WithIncome","UnemploymentRate_Person_Female","dc/g/Person_Gender-Female"],"deadline":"set"}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person_Female","LifeExpectancy_Person_Female","Median_Age_Person_Female","Median_Income_Person_15OrMoreYears_Female_WithIncome","UnemploymentRate_Person_Female"],"entities":["country/IND"],"needs_dates":false,"deadline":"set"}], - ["FakeGraph.node_labels_batch",{"dcids":["Count_Person_Female","Female","Person","count","country/IND","dc/base/IndiaCensus","gender","measuredValue"]}], - ["FakeGraph.node_types_batch",{"dcids":["country/IND"]}], - ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], - ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person_Female"],"entities":["country/IND"],"needs_dates":true}], - ["FakeGraph.node_labels_batch",{"dcids":["dc/base/IndiaCensus"]}], ["FakeGraph.node_arcs_batch",{"dcids":["Count_Person_Male"]}], - ["FakeGraph.node_labels_batch",{"dcids":["gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Male","gender"]}], ["FakeGraph.node_labels_batch",{"dcids":["country/IND"]}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_Person_Male","axis":"provenance"}], ["FakeGraph.default_facet_policy",{"stat_var":"dcid:Count_Person_Male","axis":"measurementMethod"}], @@ -3488,6 +3482,12 @@ ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person_Male"],"entities":["country/IND"],"needs_dates":true}], ["FakeGraph.node_labels_batch",{"dcids":["dc/base/IndiaCensus"]}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person_Female","LifeExpectancy_Person_Female","Median_Age_Person_Female","Median_Income_Person_15OrMoreYears_Female_WithIncome","UnemploymentRate_Person_Female"],"entities":["country/IND"],"needs_dates":false,"deadline":"set"}], + ["FakeGraph.node_labels_batch",{"dcids":["Count_Person_Female","Female","Person","count","country/IND","dc/base/IndiaCensus","gender","measuredValue"]}], + ["FakeGraph.node_types_batch",{"dcids":["country/IND"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Country"]}], + ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person_Female"],"entities":["country/IND"],"needs_dates":true}], + ["FakeGraph.node_labels_batch",{"dcids":["dc/base/IndiaCensus"]}], ["FakeGraph.node_arcs_batch",{"dcids":["Female","Male"]}], ["FakeGraph.observation_facets_batch",{"stat_vars":["Count_Person_Female","Count_Person_Male"],"entities":["country/IND"],"needs_dates":true}], ["FakeGraph.node_labels_batch",{"dcids":["dc/base/IndiaCensus"]}] diff --git a/tests/fixtures/requests/test_resubmit_place_scope_default.json b/tests/fixtures/requests/test_resubmit_place_scope_default.json index 8201b3ac..c60c3d27 100644 --- a/tests/fixtures/requests/test_resubmit_place_scope_default.json +++ b/tests/fixtures/requests/test_resubmit_place_scope_default.json @@ -29,7 +29,7 @@ ["FakeGraphReader.node_arcs_many",{"dcids":["TEST/ResubmitPlaceScopeDefault_PopulationFemale","TEST/ResubmitPlaceScopeDefault_PopulationOver65"],"properties":["age","constraintProperties","gender","measuredProperty","measurementDenominator","measurementQualifier","observationProperties","populationType","statType","typeOf"],"individual_probe_budget":null,"deadline":null}], ["FakeGraphReader.node_arcs_many",{"dcids":["TEST/ResubmitPlaceScopeDefault_PopulationFemale","TEST/ResubmitPlaceScopeDefault_PopulationOver65"],"properties":["age","constraintProperties","gender","measuredProperty","measurementDenominator","measurementQualifier","observationProperties","populationType","statType","typeOf"],"individual_probe_budget":null,"deadline":null}], ["FakeGraph.node_arcs_batch",{"dcids":["TEST/ResubmitPlaceScopeDefault_PopulationFemale","TEST/ResubmitPlaceScopeDefault_PopulationOver65"]}], - ["FakeGraph.node_labels_batch",{"dcids":["age","gender"]}], + ["FakeGraph.node_labels_batch",{"dcids":["Female","Years65Onwards","age","gender"]}], ["FakeGraph.generation_stamp",{}], ["FakeGraph.source_key",{}], ["FakeSdmxClient.source_key",{}], diff --git a/tests/graph_api/test_members_door.py b/tests/graph_api/test_members_door.py index 5571b9a0..49ca803e 100644 --- a/tests/graph_api/test_members_door.py +++ b/tests/graph_api/test_members_door.py @@ -116,8 +116,10 @@ def _row(provenance: str, dataset: str) -> ProvenanceRow: class _NameReadFails(FakeGraphNodeReader): - def node_names_many(self, dcids, *, deadline=None): - raise GraphReadError("names", upstream_status=503) + def node_arcs_many(self, dcids, *, properties, deadline=None): + if tuple(properties) == ("name",): + raise GraphReadError("names", upstream_status=503) + return super().node_arcs_many(dcids, properties=properties, deadline=deadline) def _surface_app( @@ -280,13 +282,25 @@ def test_a_provenance_lists_the_variables_the_code_index_associates(self, client ) assert (page["complete"], page["gaps"]) == (True, []) + @pytest.mark.parametrize( + "unnamed_arcs", + [ + pytest.param(lambda dcid: {"name": (dcid,)}, id="self-named"), + pytest.param( + lambda dcid: {"name": (dcid,), "alternateName": ("Alternate",)}, + id="self-named-with-alternate-name", + ), + pytest.param(lambda dcid: {"alternateName": ("Alternate",)}, id="alternate-name-only"), + ], + ) def test_a_variables_page_takes_its_labels_from_one_graph_name_read( - self, shared_resolver, serve_catalogue + self, shared_resolver, serve_catalogue, unnamed_arcs ): - """The first variable has a name; the second only a name equal to its dcid.""" - named, self_named, *_ = get_index().variables_by_provenance[_DAC1] + """The first variable has a name. The second has no name but its own dcid, so it + takes no label, and an `alternateName` never becomes one, as on the node door.""" + named, unnamed, *_ = get_index().variables_by_provenance[_DAC1] graph = FakeGraphNodeReader( - nodes={named: {"name": ("Named variable",)}, self_named: {"name": (self_named,)}} + nodes={named: {"name": ("Named variable",)}, unnamed: unnamed_arcs(unnamed)} ) serve_catalogue(_CATALOGUE) app = _surface_app(shared=shared_resolver, policy=_POLICY, graph=graph) @@ -297,7 +311,7 @@ def test_a_variables_page_takes_its_labels_from_one_graph_name_read( assert [item["label"] for item in page["items"]][:2] == ["Named variable", None] assert page["gaps"] == [] assert response.headers["cache-control"].startswith("public") - assert graph.calls["node_names_many"] == 1 + assert graph.calls["node_arcs_many"] == 1 def test_a_failed_label_read_leaves_labels_null_and_is_not_stored( self, shared_resolver, serve_catalogue diff --git a/tests/graph_api/test_node_blocks.py b/tests/graph_api/test_node_blocks.py index c4bee2b3..347c8ce0 100644 --- a/tests/graph_api/test_node_blocks.py +++ b/tests/graph_api/test_node_blocks.py @@ -397,6 +397,24 @@ def test_a_unit_carries_the_words_it_is_named_by(self, dcid, arcs, expected): assert node["unit"] == expected +@pytest.mark.parametrize( + "names", + [ + ("United States", "United States of America"), + ("United States of America", "United States"), + ], +) +def test_a_node_of_several_names_is_labelled_as_every_surface_labels_it(names): + """The label is the node's `display_name`, the name a card and `node_label` give it, in + whatever order the graph returns the names.""" + graph = FakeGraphNodeReader(nodes={"country/USA": {"typeOf": ("Country",), "name": names}}) + + with _client(graph, policy=None) as client: + node = client.get("/api/graph/node/country/USA").json() + + assert node["label"] == "United States of America" + + def test_a_node_with_no_name_names_the_label_gap(): graph = FakeGraphNodeReader(nodes={"dc/base/Foo": {"typeOf": ("SomeOtherType",)}}) diff --git a/tests/origin_catalogue/test_arc_reads.py b/tests/origin_catalogue/test_arc_reads.py index e94356b8..ebcd0848 100644 --- a/tests/origin_catalogue/test_arc_reads.py +++ b/tests/origin_catalogue/test_arc_reads.py @@ -287,10 +287,10 @@ def _labelled_nodes() -> dict[str, Any]: @pytest.mark.parametrize("hazard", ["omitting", "emptying"]) def test_each_node_an_arc_names_takes_the_label_node_label_gives_it(hazard: str) -> None: - """The label is the last `->name` value, or none, and the arc keeps the name `->*` - gives. A node a `->name` read naming several leaves out or answers with no name is - read again alone, as the `->*` and `->typeOf` reads read it, so `Years0`, which has no - name, is read alone before it is held nameless.""" + """The label is the `label_or_self_name` of the `->name` values, or none, and the arc + keeps the name `->*` gives. A node a `->name` read naming several leaves out or answers + with no name is read again alone, as the `->*` and `->typeOf` reads read it, so + `Years0`, which has no name, is read alone before it is held nameless.""" un = FakeUn(_labelled_nodes()) setattr(un, hazard, {"Alcohol": 1}) diff --git a/tests/origin_catalogue/test_graph.py b/tests/origin_catalogue/test_graph.py index 942a8be1..4f950736 100644 --- a/tests/origin_catalogue/test_graph.py +++ b/tests/origin_catalogue/test_graph.py @@ -805,7 +805,7 @@ def test_a_node_an_arc_names_takes_its_label_from_the_labels_and_keeps_its_arc_n graph: _Graph, client: OriginGraphClient, bind_catalogue ) -> None: """The graph's `->*` read names a nameless node by its dcid and a node of several names - by its first, and `node_label` gives the first no label and the second its last name. + by its first, and `node_label` gives the first no label and the second its `display_name`. The release and the tables answer each read as the graph does, with no request.""" age, alcohol = "Years0", "Alcohol" crs = { diff --git a/tests/presentation_store/test_frame_store.py b/tests/presentation_store/test_frame_store.py index f93dae74..5d6cfc09 100644 --- a/tests/presentation_store/test_frame_store.py +++ b/tests/presentation_store/test_frame_store.py @@ -789,6 +789,25 @@ def test_load_resolves_a_frame_with_view_manifest_and_labels(bind_catalogue): ) +@pytest.mark.parametrize( + "names", + [ + ("Ethiopia", "Federal Democratic Republic of Ethiopia"), + ("Federal Democratic Republic of Ethiopia", "Ethiopia"), + ], +) +def test_load_labels_a_node_of_several_names_as_the_node_door_does(bind_catalogue, names): + """A card's label is the node's `display_name`, whatever order the graph gives its names.""" + snapshots, artifacts, frame_ref, _digest = _happy_path_fixture() + graph = _FakeGraphReader(labels={}, arcs={"country/ETH": {"name": names}}) + store = _store(snapshots=snapshots, artifacts=artifacts, graph=graph) + bind_catalogue(FakeCatalogue(build_id="build-1")) + + resolved = store.load(frame_ref=frame_ref) + + assert resolved.labels["country/ETH"] == "Federal Democratic Republic of Ethiopia" + + def test_load_labels_the_group_members_a_coverage_warning_names(): """A member that served nothing holds no row, so the refs a `coverage_below_scope` warning names join the frame's one label batch.""" From 6b2a0a90f70a232485d267dbf0cf61d8e3714e39 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sat, 24 Oct 2026 21:19:56 +0200 Subject: [PATCH 4/7] fix(search): refuse an answer that drops a measure the question names --- docs/conformance-search-api-v1.md | 2 +- src/search/engine/region/phase_rank_bind.py | 175 +++++++++++++++++--- src/search/models/trace.py | 45 +++-- tests/engine/test_qualifier_guard.py | 107 +++++++++++- 4 files changed, 282 insertions(+), 47 deletions(-) diff --git a/docs/conformance-search-api-v1.md b/docs/conformance-search-api-v1.md index 5f447cd3..43362527 100644 --- a/docs/conformance-search-api-v1.md +++ b/docs/conformance-search-api-v1.md @@ -225,7 +225,7 @@ Gap rows state what is needed to close the clause and name the open issue that t | C166 | Posting a related measure's `definition.identity` as `identity` on a dataset_resubmit identity entry answers definite with the same spec_id, electing the StatVars the identity names: the members a Datasets mint of it resolves. | implemented | Pinned for all 18 Macroeconomic Dashboard members with Pakistan data (`tests/fixtures/related_measures_pakistan.json`) by `tests/engine/test_related_measures.py`. | | C167 | The related-measures reads start once materialisation names the elected shape's members, as grounding starts on the single-shape path and the resubmit door and before candidates are grounded on the candidates path, and run beside grounding and assembly. A definite answer whose survivor is another StatVar starts its own once the survivor's StatVars are known, beside its streamed preview and its options. The reads are at most the StatVar's `memberOf` arcs when the answer does not hold them, one `<-memberOf` read capped at 41 subjects per group, then the chosen group's members' arcs beside their summary facets at the Spec's places. A Spec that does not qualify makes none of them. A qualifying answer's read starts as soon as the elected StatVar is known, runs concurrently with the answer's remaining reads, and can delay the answer by at most the 1.0 s `SEARCH_RELATED_MEASURES_TIMEOUT_S` deadline. A read that fails, times out or raises is logged and states none. | implemented | Pinned by `tests/engine/test_related_measures.py` on a virtual clock, by I9 in `tests/engine/test_answer_invariants.py` and by the serial-depth test in `tests/engine/test_first_answer_place_scenarios.py`. | | C92 | entity_options is the inline preview of the options endpoint's surface; slot/facet options are inline-only at v1. A directional role axis a role-bearing query left open is included: its entity_options entry previews the same candidate set POST /api/search/options already resolves through its own for.role parameter. | implemented | No gap. `engine/options/surface.py`'s `_directional_options_set` resolves the door's own for.role surface; entity_options draws its open-role-axis entry from role_axis_facts_for_materialised instead, a separate derivation over the same population. | -| C93 | resolved.trace: resolved sources, the requested window, the default price-basis reading (whether it applied and the StatVars it removed before bind), capability_version, conversion recipe, each slot_filters entry's own provenance (the binder's election, a code row recognised in the query's own text, a value the measure-family election bound on an axis the binder left open, or a value a code-index declared default (code_index.types.CodeSystemDeclaration.default) bound onto an open slot once the query bound another slot into that default's own scoping subtree, e.g. crsTypeOfFlow defaulting to CRSTypeOfFlow10 for an unqualified grants question, or a value a shape slot default (`CodeIndex.shape_slot_defaults`) bound onto an open slot of its population type: medicalStatus defaulting to LiveBirth on an elected BirthEvent shape; an alternate shape keeps the values its representative member declares), and query_qualifiers naming each narrowing term the request's text named, the code rows it names in the packaged index, the slot it bound or, for an unbound value the answer's shape has a slot for, that slot (if any), and its disposition (bound/unbound/no_vocabulary). A record may also carry its `start` and `end` offsets in the variable phrase, its `evidence` (label, synonym, model, uncovered, unavailable or unsupported), the `operation` (union or refinement) that combined its codes with another term's, and the codes it `selected` before that combination. Code matches on one code system bind by the connective between them (`phase_rank_bind._combine_matches`, `qualifier_match.joins`). Matches joined by union connectives (`qualifier_match.UNION_CONNECTIVES`) are a union. A code nesting under another of the union drops, and codes that do not nest bind a set, so "health and education" and "health & education" bind {120, 110}, "health including malaria control" binds 120, and "HIV" and "AIDS" in "HIV/AIDS and population policy, programme, and reproductive health" bind 130 and are bound through the union, which selected 13040 and 130. An ancestor binds a code only through a union. Two terms joined any other way, or by nothing, refine to the narrower code when one nests under the other ("malaria control within health" binds 12262) and otherwise leave the code system to the model's binding. An exclusion or comparison connective (`EXCLUSION_CONNECTIVES`, `COMPARISON_CONNECTIVES`) followed by a code match, across function words alone or, after "compare" or "compared", at any distance, makes the phrase from the connective to that match a no_vocabulary record with evidence `unsupported` ("excluding malaria control", "compare aid to health"), and codes it sets side by side form no union. The code an exclusion governs, and each code a union joins to it, takes no part in the binding, so "health excluding malaria control" binds 120 and the record of "malaria control" is unbound. A code system whose every code an exclusion removes is not forced. "ODA excluding health" keeps the model's binding, which the record of "health" judges. A comparison after an exclusion compares the code the exclusion kept. Only the variable's matches carry offsets. A code the query text or a dropped phrase names that the variable names too is read at the variable's offsets alone, so "health and education ODA" with the query text "health and education ODA to Kenya" still binds {120, 110}. A comparison of two codes of one system that do not nest refuses in either coverage mode. Every other such phrase refuses under `refuse` only, and under `log` is logged as a would-refuse line. Evidence `unavailable` marks a code term the elected shape cannot apply, meaning one naming only codes no shape can bind (`qualifier_match.bindable_codes`, where no index row holds the code or a descendant of it and index rows carry its code system, as with `CRSTypeOfFinance4`, "population"), whose words the shape's own reading leaves unread, or, on a part naming a dataset, one no row of that dataset holds (C78b). A shape's own reading matches, ties and forces only codes some shape can bind. A part naming a dataset refuses qualifier_not_bound on an unavailable record in either coverage mode, with the elected identity as flagged_match. A part naming none does so under `refuse`. Under `log` it logs the would-refuse line for the term and departs it `measure_not_applied` with the `query_qualifier_dropped` warning. Each run of variable words no reader covers beside a code span of the elected shape's reading or a phrase a menu choice reads, or joined to one by a union connective (`qualifier_match.UNION_CONNECTIVES`), is an unbound record with evidence `uncovered` naming that span's rows (`phase_rank_bind._coverage_records`), such as "maternal" in "maternal health ODA" beside sector 120, and "maternal" in "maternal and infant health ODA" joined to "infant health" when the model chooses 12240 for it. A run beside no code span is rated instead. The binder rates, once per pass over the final bindings, how likely the elected shape's series states each word of it the readers leave unread (`phase_election._rate_unread_words`), and each stretch of words it rates below `SearchSettings.stated_word_bar` (0.3) is a no_vocabulary record with evidence `uncovered` naming no rows, such as "refugee" in "refugee population" on the count of all persons and "tax" in "tax revenue as a percent of GDP" on total revenue. A word rated at the bar or above reads as stated. So does a word the binder leaves unrated or a read that fails, and the part then notes a degraded read, so the response cache stores no such answer. The record's flagged match is the elected series. A run's neighbours lie past the amount words, data words and flow verbs beside it, so "maternal" in "maternal data ODA" and "water" in "water spending ODA" are records beside ODA. The readers are the spans of that reading, code and measure values, the terms already recorded, the part's entity text, matched as a code label is across the separators it holds and with either apostrophe ("China" in "Hong Kong SAR, China ODA" and "Ivoire" in "Côte d’Ivoire ODA" are read), a possessive ending included, date phrases, `function_words.FUNCTION_WORDS`, the amount words `function_words.AMOUNT_WORDS` ("spending", "expenditure", "funding", "financing", "total", "figures", "amount", "amounts", "money", "flows"), the nouns naming data itself `function_words.DATA_WORDS` ("data", "statistics", "numbers", "information", "indicators", "series", as with "data" in "health data"), a connective between two code spans, the camelCase words of the shape's five-tuple dcids and its population type name (`phase_rank_bind._series_words`), the words every label of the datasets the part answers from holds, the words those datasets state of the rows the part reads (`phase_rank_bind._dataset_stated_words`, C78b: "bilateral" on the CRS, and on DAC2a or DAC2b where the part's recipient is a country, and "multilateral" there where it names the provider the part leaves unnamed), the flow verbs `function_words.FLOW_VERBS` (received, receive, receives, receiving, given, provided, disbursed, spent, going, goes, went, reported, reports, as with "received" in "net ODA received by Kenya" and "reported" in "total ODA reported to the DAC"), on a shape with a measurement denominator the ratio words `function_words.RATIO_WORDS` (share, ratio, percent, percentage, proportion) and the words of the denominator's noun (`measure_noun.denominator_noun`, "GNI"), the declared plain names of the shape's slots, and the words of the label or a synonym of each code the final bindings apply that no span of that reading and no menu choice names (`phase_rank_bind._unnamed_bound_code_words`: "private" in "private grants" when the answer binds flow type 30, "private development finance", and "loans" under type of finance 420, synonym "loan instrument", while 130, which "reproductive health" names, leaves "policy" in "reproductive health and education policy ODA" unread); the shape's own label is excluded from the readers. On such a shape the ratio words, denominator noun words and function words of a code term read in every code system together are read too, so "ODA as a share of GNI" and "ODA GNI ratio" answer on the GNI share shape, while on an amount shape "as a share of GNI" is a record naming type of finance 2 and "GNI ratio" one naming the ODA code. The five-tuple words of a code term read in every code system together are read on every shape, and a hyphenated word is read when each of its parts is (`phase_rank_bind._holds_word`), so on the grant-equivalent shape (measured property `grantEquivalent`) "grant-equivalent ODA" and "ODA grant equivalents" (DAC2a aid type 201 "ODA grant") answer, while on an amount shape "grant-equivalent" and "grant equivalents" are records naming the ODA code, and "grants" in "ODA grants" stays a record on the grant-equivalent shape. A code span also reads the adjacent words, function words and connectives aside, that the label or a synonym of every code it names holds (`phase_rank_bind._code_span_reach`), and its neighbours lie past them, so "multilateral" beside "imputed", DAC2a aid type 106 "imputed multilateral ODA", is read, and "grants" in "imputed multilateral grants" is a record naming 106. A word of a code term read in every code system together that the shape's reading leaves unread ("multilateral" in "multilateral ODA" on a CRS shape) is such a record too, unless the name of such an unnamed bound code holds every word of it ("loans" in "ODA loans", DAC2a aid type 218, under 420). Such a run, with the code terms beside it, is a phrase the bind call is offered a bounded candidate menu for on each candidate shape (`phase_rank_bind._candidate_menus`, `CodeIndex.candidate_codes`): the codes of the shape's code systems whose labels or synonyms share a word of the run, ranked by the words they share and how rare those are, the first `CANDIDATE_HITS` (8) kept, then the codes of the terms beside it, then the ancestors and children of each, at most `CANDIDATE_LIMIT` (24) codes, less those the shape's family does not hold. A run sharing no word with any code ("maternal") has no menu, and neither has a word of a code term read in every code system together that names a code another shape the part's recall found binds (`ResolveState.shapes_found`, before a named dataset narrows them), one declaring the code's system whose family holds it: "imputed" in "imputed education aid", DAC2a aid type 106, gets no "imputed student cost" menu on a CRS shape when the imputed sectors shape is recalled, and "multilateral" in "multilateral ODA", DAC1 aid type 2000, stays an unavailable term on a part naming the CRS when a DAC1 shape holding 2000 is recalled. With no such shape the run keeps its menu, so "loans" in "ODA loans for education", DAC2a aid type 218, is offered type of finance 420, and "population", type of finance 4, keeps its menu. A shape the query's places do not bind to (`place_roles.bind_place_roles` refuses) has no menu, so "how much do African countries spend on their own health care?" offers the CRS flow shape no "general government and civil society support" for "domestic general government health", and the expenditure series answers. The menus precede bind, so the labels of bound codes read nothing for them. The bind prompt lists each menu under its phrase, every code of it whatever the 20-value taxonomy cap shows, and a term inside the phrase on a code system the menu names leaves fixed_context; a prompt with no menu is unchanged. A code the model binds that exactly one menu offers, on the code system of its binding, reads the phrase when it is a lexical hit of the menu or a descendant of one and, for each term inside the phrase on that system, is that term's code or lies under it, or has a label or synonym holding every content word of the term (`phase_rank_bind._menu_choices`, `_holds_every_word`). 41081 "environmental education and training" chosen for "environment education" reads the phrase in place of 110, though it lies under 410. 15180 chosen for "girls education" lies outside 110 and holds no "education", so it reads nothing. "education" binds 110 and "girls" keeps its record. A chosen hit reads the phrase as the furthest of its ancestors among the menu's hits whose labels or synonyms share every word of the phrase its own share (`phase_rank_bind._broadest_reading`), so 421 "standard loan" chosen for "education loans" reads as 420, whose synonym "loan instrument" shares "loans" too, and 13010 chosen for "population" reads as 130. Such a code forces in place of the terms inside the phrase on that system, combined with the other terms by the connective between them, the phrase records it with evidence `model`, and no uncovered or unavailable record falls inside the phrase. The terms' own codes, their children and the ancestors of hits read nothing and force nothing, so the terms force as they would with no menu. 121 chosen for "infant health", whose one hit is 12240, gives way to the 120 "health" names, and "infant" keeps its record. "contraceptive services ODA" binds 13030 family planning when the model chooses it from the menu its synonym "contraceptive delivery" puts it on, and the rendered sentence names it; "population policies and reproductive health" binds 130 whether the model chooses 13010 or 130 for "population policies". A phrase with no such choice keeps its terms. `SearchSettings.qualifier_coverage` (env `SEARCH_QUALIFIER_COVERAGE`, default `refuse`) decides what the records do. Under `refuse` they enter query_qualifiers and the part refuses qualifier_not_bound with the near match as flagged_match; under `log` election vetting logs each once (`qualifier_coverage would_refuse phrase= codes=`), the records stay out of query_qualifiers, and an unread word of a code term departs `measure_not_applied` with the `query_qualifier_dropped` warning. The mode enters the response cache's scope document | partial | `conversion` is always absent: conversion is recorded per series in `resolved.observations.series[].conversion`, since one Spec's series can convert under different recipes. Issue: none. No gap on default_reading: pinned by `tests/engine/test_measure_query_collapse_pipeline.py`. No gap on query_qualifiers: pinned by `tests/engine/test_qualifier_trace.py` and, for the disposition and coverage rules, `tests/engine/test_qualifier_guard.py`; the coverage mode's cache key by `tests/engine/test_response_cache_keys.py`. No gap on the `declared_default` provenance value: pinned end to end by `tests/engine/test_declared_default_e2e.py` and, for the trigger's own conditions in isolation, `tests/engine/test_bind_repair.py`'s `TestApplyDeclaredDefaults*` classes; the shape slot default is pinned by `tests/engine/test_shape_slot_defaults.py`. | +| C93 | resolved.trace: resolved sources, the requested window, the default price-basis reading (whether it applied and the StatVars it removed before bind), capability_version, conversion recipe, each slot_filters entry's own provenance (the binder's election, a code row recognised in the query's own text, a value the measure-family election bound on an axis the binder left open, or a value a code-index declared default (code_index.types.CodeSystemDeclaration.default) bound onto an open slot once the query bound another slot into that default's own scoping subtree, e.g. crsTypeOfFlow defaulting to CRSTypeOfFlow10 for an unqualified grants question, or a value a shape slot default (`CodeIndex.shape_slot_defaults`) bound onto an open slot of its population type: medicalStatus defaulting to LiveBirth on an elected BirthEvent shape; an alternate shape keeps the values its representative member declares), and query_qualifiers naming each narrowing term the request's text named, the code rows it names in the packaged index, the slot it bound or, for an unbound value the answer's shape has a slot for, that slot (if any), and its disposition (bound/unbound/no_vocabulary). A record may also carry its `start` and `end` offsets in the variable phrase, its `evidence` (label, synonym, model, uncovered, unavailable or unsupported), the `operation` (union or refinement) that combined its codes with another term's, and the codes it `selected` before that combination. Code matches on one code system bind by the connective between them (`phase_rank_bind._combine_matches`, `qualifier_match.joins`). Matches joined by union connectives (`qualifier_match.UNION_CONNECTIVES`) are a union. A code nesting under another of the union drops, and codes that do not nest bind a set, so "health and education" and "health & education" bind {120, 110}, "health including malaria control" binds 120, and "HIV" and "AIDS" in "HIV/AIDS and population policy, programme, and reproductive health" bind 130 and are bound through the union, which selected 13040 and 130. An ancestor binds a code only through a union. Two terms joined any other way, or by nothing, refine to the narrower code when one nests under the other ("malaria control within health" binds 12262) and otherwise leave the code system to the model's binding. An exclusion or comparison connective (`EXCLUSION_CONNECTIVES`, `COMPARISON_CONNECTIVES`) followed by a code match, across function words alone or, after "compare" or "compared", at any distance, makes the phrase from the connective to that match a no_vocabulary record with evidence `unsupported` ("excluding malaria control", "compare aid to health"), and codes it sets side by side form no union. The code an exclusion governs, and each code a union joins to it, takes no part in the binding, so "health excluding malaria control" binds 120 and the record of "malaria control" is unbound. A code system whose every code an exclusion removes is not forced. "ODA excluding health" keeps the model's binding, which the record of "health" judges. A comparison after an exclusion compares the code the exclusion kept. Only the variable's matches carry offsets. A code the query text or a dropped phrase names that the variable names too is read at the variable's offsets alone, so "health and education ODA" with the query text "health and education ODA to Kenya" still binds {120, 110}. A comparison of two codes of one system that do not nest refuses in either coverage mode. Every other such phrase refuses under `refuse` only, and under `log` is logged as a would-refuse line. Evidence `unavailable` marks a code term the elected shape cannot apply, meaning one naming only codes no shape can bind (`qualifier_match.bindable_codes`, where no index row holds the code or a descendant of it and index rows carry its code system, as with `CRSTypeOfFinance4`, "population"), whose words the shape's own reading leaves unread, or, on a part naming a dataset, one no row of that dataset holds (C78b). A shape's own reading matches, ties and forces only codes some shape can bind. A part naming a dataset refuses qualifier_not_bound on an unavailable record in either coverage mode, with the elected identity as flagged_match. A part naming none does so under `refuse`. Under `log` it logs the would-refuse line for the term and departs it `measure_not_applied` with the `query_qualifier_dropped` warning. Each run of variable words no reader covers beside a code span of the elected shape's reading or a phrase a menu choice reads, or joined to one by a union connective (`qualifier_match.UNION_CONNECTIVES`), is an unbound record with evidence `uncovered` naming that span's rows (`phase_rank_bind._coverage_records`), such as "maternal" in "maternal health ODA" beside sector 120, and "maternal" in "maternal and infant health ODA" joined to "infant health" when the model chooses 12240 for it. On an elected shape the measure index describes, one whose answer carries a value of the measure index's populationType axis, a word names a matchable measure value when every name of the value's reading holds it (the label and each synonym of the value and of each alias of it, `MeasureIndex.reading_set`), no span of the variable names the value, and the answer over the elected shape and the final bindings does not carry it (`phase_rank_bind._uncarried_measure_words`). A function word names no measure value, and neither does a word inside a span read in every code system together or a word of a name of a measure value the answer carries, matchable or not. On any other shape no word names a measure value. In a run beside no code span such a word is unread without a rating, on a shape with or without a named member, and each stretch of unread words holding one is an unbound record with evidence `uncovered` naming those measure values, such as "mobilised" in "private sector investment mobilised", which names amountMobilised ("amount mobilised", "private finance mobilised") on a financial-flow amount series, where "private" and "finance" name nothing. "inflation" names no RealValue, since "real terms" lacks it though the alias "inflation adjusted" holds it. A run beside a code span that holds such a word names the measure values beside the code rows, so "mobilised" in "health ODA mobilised" names CRSTypeOfFlow10 and amountMobilised. The run's other words are rated. The binder rates, once per pass over the final bindings, how likely the elected shape's series states each such word the readers leave unread (`phase_election._rate_unread_words`), and each stretch of words it rates below `SearchSettings.stated_word_bar` (0.3) is a no_vocabulary record with evidence `uncovered` naming no rows, such as "refugee" in "refugee population" on the count of all persons and "tax" in "tax revenue as a percent of GDP" on total revenue. A word rated at the bar or above reads as stated. So does a word the binder leaves unrated or a read that fails, and the part then notes a degraded read, so the response cache stores no such answer. The record's flagged match is the elected series. A run's neighbours lie past the amount words, data words and flow verbs beside it, so "maternal" in "maternal data ODA" and "water" in "water spending ODA" are records beside ODA. The readers are the spans of that reading, code and measure values, the terms already recorded, the part's entity text, matched as a code label is across the separators it holds and with either apostrophe ("China" in "Hong Kong SAR, China ODA" and "Ivoire" in "Côte d’Ivoire ODA" are read), a possessive ending included, date phrases, `function_words.FUNCTION_WORDS`, the amount words `function_words.AMOUNT_WORDS` ("spending", "expenditure", "funding", "financing", "total", "figures", "amount", "amounts", "money", "flows"), the nouns naming data itself `function_words.DATA_WORDS` ("data", "statistics", "numbers", "information", "indicators", "series", as with "data" in "health data"), a connective between two code spans, the camelCase words of the shape's five-tuple dcids and its population type name (`phase_rank_bind._series_words`), the words every label of the datasets the part answers from holds, the words those datasets state of the rows the part reads (`phase_rank_bind._dataset_stated_words`, C78b: "bilateral" on the CRS, and on DAC2a or DAC2b where the part's recipient is a country, and "multilateral" there where it names the provider the part leaves unnamed), the flow verbs `function_words.FLOW_VERBS` (received, receive, receives, receiving, given, provided, disbursed, spent, going, goes, went, reported, reports, as with "received" in "net ODA received by Kenya" and "reported" in "total ODA reported to the DAC"), on a shape with a measurement denominator the ratio words `function_words.RATIO_WORDS` (share, ratio, percent, percentage, proportion) and the words of the denominator's noun (`measure_noun.denominator_noun`, "GNI"), the declared plain names of the shape's slots, and the words of the label or a synonym of each code the final bindings apply that no span of that reading and no menu choice names (`phase_rank_bind._unnamed_bound_code_words`: "private" in "private grants" when the answer binds flow type 30, "private development finance", and "loans" under type of finance 420, synonym "loan instrument", while 130, which "reproductive health" names, leaves "policy" in "reproductive health and education policy ODA" unread); the shape's own label is excluded from the readers. On such a shape the ratio words, denominator noun words and function words of a code term read in every code system together are read too, so "ODA as a share of GNI" and "ODA GNI ratio" answer on the GNI share shape, while on an amount shape "as a share of GNI" is a record naming type of finance 2 and "GNI ratio" one naming the ODA code. The five-tuple words of a code term read in every code system together are read on every shape, and a hyphenated word is read when each of its parts is (`phase_rank_bind._holds_word`), so on the grant-equivalent shape (measured property `grantEquivalent`) "grant-equivalent ODA" and "ODA grant equivalents" (DAC2a aid type 201 "ODA grant") answer, while on an amount shape "grant-equivalent" and "grant equivalents" are records naming the ODA code, and "grants" in "ODA grants" stays a record on the grant-equivalent shape. A code span also reads the adjacent words, function words and connectives aside, that the label or a synonym of every code it names holds (`phase_rank_bind._code_span_reach`), and its neighbours lie past them, so "multilateral" beside "imputed", DAC2a aid type 106 "imputed multilateral ODA", is read, and "grants" in "imputed multilateral grants" is a record naming 106. A word of a code term read in every code system together that the shape's reading leaves unread ("multilateral" in "multilateral ODA" on a CRS shape) is such a record too, unless the name of such an unnamed bound code holds every word of it ("loans" in "ODA loans", DAC2a aid type 218, under 420). Such a run, with the code terms beside it, is a phrase the bind call is offered a bounded candidate menu for on each candidate shape (`phase_rank_bind._candidate_menus`, `CodeIndex.candidate_codes`): the codes of the shape's code systems whose labels or synonyms share a word of the run, ranked by the words they share and how rare those are, the first `CANDIDATE_HITS` (8) kept, then the codes of the terms beside it, then the ancestors and children of each, at most `CANDIDATE_LIMIT` (24) codes, less those the shape's family does not hold. A run sharing no word with any code ("maternal") has no menu, and neither has a word of a code term read in every code system together that names a code another shape the part's recall found binds (`ResolveState.shapes_found`, before a named dataset narrows them), one declaring the code's system whose family holds it: "imputed" in "imputed education aid", DAC2a aid type 106, gets no "imputed student cost" menu on a CRS shape when the imputed sectors shape is recalled, and "multilateral" in "multilateral ODA", DAC1 aid type 2000, stays an unavailable term on a part naming the CRS when a DAC1 shape holding 2000 is recalled. With no such shape the run keeps its menu, so "loans" in "ODA loans for education", DAC2a aid type 218, is offered type of finance 420, and "population", type of finance 4, keeps its menu. A shape the query's places do not bind to (`place_roles.bind_place_roles` refuses) has no menu, so "how much do African countries spend on their own health care?" offers the CRS flow shape no "general government and civil society support" for "domestic general government health", and the expenditure series answers. The menus precede bind, so the labels of bound codes read nothing for them. The bind prompt lists each menu under its phrase, every code of it whatever the 20-value taxonomy cap shows, and a term inside the phrase on a code system the menu names leaves fixed_context; a prompt with no menu is unchanged. A code the model binds that exactly one menu offers, on the code system of its binding, reads the phrase when it is a lexical hit of the menu or a descendant of one and, for each term inside the phrase on that system, is that term's code or lies under it, or has a label or synonym holding every content word of the term (`phase_rank_bind._menu_choices`, `_holds_every_word`). 41081 "environmental education and training" chosen for "environment education" reads the phrase in place of 110, though it lies under 410. 15180 chosen for "girls education" lies outside 110 and holds no "education", so it reads nothing. "education" binds 110 and "girls" keeps its record. A chosen hit reads the phrase as the furthest of its ancestors among the menu's hits whose labels or synonyms share every word of the phrase its own share (`phase_rank_bind._broadest_reading`), so 421 "standard loan" chosen for "education loans" reads as 420, whose synonym "loan instrument" shares "loans" too, and 13010 chosen for "population" reads as 130. Such a code forces in place of the terms inside the phrase on that system, combined with the other terms by the connective between them, the phrase records it with evidence `model`, and no uncovered or unavailable record falls inside the phrase. The terms' own codes, their children and the ancestors of hits read nothing and force nothing, so the terms force as they would with no menu. 121 chosen for "infant health", whose one hit is 12240, gives way to the 120 "health" names, and "infant" keeps its record. "contraceptive services ODA" binds 13030 family planning when the model chooses it from the menu its synonym "contraceptive delivery" puts it on, and the rendered sentence names it; "population policies and reproductive health" binds 130 whether the model chooses 13010 or 130 for "population policies". A phrase with no such choice keeps its terms. `SearchSettings.qualifier_coverage` (env `SEARCH_QUALIFIER_COVERAGE`, default `refuse`) decides what the records do. Under `refuse` they enter query_qualifiers and the part refuses qualifier_not_bound with the near match as flagged_match; under `log` election vetting logs each once (`qualifier_coverage would_refuse phrase= codes=`), the records stay out of query_qualifiers, and an unread word of a code term departs `measure_not_applied` with the `query_qualifier_dropped` warning. The mode enters the response cache's scope document | partial | `conversion` is always absent: conversion is recorded per series in `resolved.observations.series[].conversion`, since one Spec's series can convert under different recipes. Issue: none. No gap on default_reading: pinned by `tests/engine/test_measure_query_collapse_pipeline.py`. No gap on query_qualifiers: pinned by `tests/engine/test_qualifier_trace.py` and, for the disposition and coverage rules, `tests/engine/test_qualifier_guard.py`; the coverage mode's cache key by `tests/engine/test_response_cache_keys.py`. No gap on the `declared_default` provenance value: pinned end to end by `tests/engine/test_declared_default_e2e.py` and, for the trigger's own conditions in isolation, `tests/engine/test_bind_repair.py`'s `TestApplyDeclaredDefaults*` classes; the shape slot default is pinned by `tests/engine/test_shape_slot_defaults.py`. | | C94 | resolved.confidence: the same coarse band as C56 | implemented | | ## Facets / unit binding diff --git a/src/search/engine/region/phase_rank_bind.py b/src/search/engine/region/phase_rank_bind.py index 004b3f0b..07ff75c7 100644 --- a/src/search/engine/region/phase_rank_bind.py +++ b/src/search/engine/region/phase_rank_bind.py @@ -19,7 +19,7 @@ from code_index.query import CodeIndex, fold_word from code_index.shared import get_index, get_measure_index from code_index.tokens import Token, text_matches, tokenize, word_tokens -from code_index.types import CodeRow, DatasetDeclaration, MeasureAxis, StatedWord +from code_index.types import CodeRow, DatasetDeclaration, MeasureAxis, MeasureValueRow, StatedWord from origin_catalogue import current_catalogue from search.engine.bind import ( CandidateMenu, @@ -132,7 +132,11 @@ without_open_axis_gaps, without_unasked_axes, ) -from search.engine.region.qualifier_readings import match_query_readings, reading_for_shape +from search.engine.region.qualifier_readings import ( + answer_reads_measure, + match_query_readings, + reading_for_shape, +) from search.engine.region.result import RegionResult from search.engine.region.seating import is_expansion, seat_menu from search.engine.region.sentence_events import _build_shape_event @@ -2874,15 +2878,22 @@ def _coverage_records( """The variable's words no reader covers on the elected shape, each run as one record with evidence ``"uncovered"``. They catch restrictions the answer would otherwise drop silently. A run beside a recognised code is ``unbound``, naming that code's rows: - "maternal" in "maternal health ODA" binds nothing beside health, sector 120. In any - other run the readers only nominate, and the binder rates how likely the shape's - series states each word (`ResolveState.stated_word_ratings`). Each stretch of words it - rates below `SearchSettings.stated_word_bar` is ``no_vocabulary``, naming none: - "refugee" in "refugee population" on the count of all persons, "tax" in "tax revenue - as a percent of GDP" on total revenue. A word it rates at the bar or above, or does - not rate, is read. Either way a record refuses the part, and the reading it found is - its flagged match. A shape none of whose members has a name states no words of its - measure, so on it only a run beside a code is a record. + "maternal" in "maternal health ODA" binds nothing beside health, sector 120. On a + shape the measure index describes, a word naming a matchable measure value the answer + does not carry, one every name of the value's reading holds + (`_uncarried_measure_words`), is unread, and its record names the value too: + "mobilised" in "private sector investment mobilised" on an amount shape names the + amount mobilised, and so does "mobilised" beside ODA in "health ODA mobilised". The + readers only nominate the run's other words, and the binder + rates how likely the shape's series states each (`ResolveState.stated_word_ratings`). + Each stretch of unread words is a record, ``unbound`` naming the measure values its + words name, else ``no_vocabulary`` naming none: "refugee" in "refugee population" on + the count of all persons, "tax" in "tax revenue as a percent of GDP" on total revenue, + each rated below `SearchSettings.stated_word_bar`. A word it rates at the bar or + above, or does not rate, is read. Either way a record refuses the part, and the + reading it found is its flagged match. A shape none of whose members has a name states + no words of its measure, so on it only a run beside a code and a measure value's word + are records. The readers are the spans of the elected shape's own reading (code and measure values), the terms in `recorded`, a phrase a menu choice reads among them @@ -2938,7 +2949,8 @@ def _coverage_reading( ) -> _CoverageReading: """`_coverage_records` with the verdict of each word of the variable, in query order: the name of the reader that reads it (`_Reading.verdicts`), else, in a run beside no - code span on a shape with a named member, ``"stated"`` when the binder rates the + code span on a shape with a named member, for a word naming no measure value the + answer lacks (`_uncarried_measure_words`), ``"stated"`` when the binder rates the shape's series stating it at `SearchSettings.stated_word_bar` or above (`ResolveState.stated_word_ratings`), ``"unrated"`` when it gives the word no rating, which counts as stated, and ``"unread"`` when it rates it below, else ``"unread"``.""" @@ -3012,30 +3024,49 @@ def _coverage_reading( verdicts = dict(reading.verdicts) ratings = state.stated_word_ratings bar = settings().stated_word_bar + measure_values = _uncarried_measure_words(state, tokens) records: list[ResolvedQualifier] = [] for run in reading.runs: if run.adjacent or run.joined: start, end = tokens[run.first].start, tokens[run.last].end records.append( - _uncovered_record(variable[start:end], start, end, run.adjacent or run.joined) + _uncovered_record( + variable[start:end], + start, + end, + run.adjacent or run.joined, + _run_measure_values(measure_values, run.first, run.last), + ) ) continue - if measure_words is None: - continue - # A word beside no code is unread only on the binder's rating, so each stretch of - # the run it rates below the bar is one record. + # A word beside no code naming a measure value the answer does not carry is unread + # whatever the binder would rate it. Any other is unread only on the binder's + # rating, on a shape with a named member. Each stretch of unread words is one + # record. unread: list[tuple[int, int]] = [] for position in range(run.first, run.last + 1): - rating = ratings.get(tokens[position].text.casefold()) - if rating is None or rating >= bar: - verdicts[position] = "unrated" if rating is None else "stated" - elif unread and unread[-1][1] == position - 1: + if position not in measure_values: + if measure_words is None: + continue + rating = ratings.get(tokens[position].text.casefold()) + if rating is None or rating >= bar: + verdicts[position] = "unrated" if rating is None else "stated" + continue + if unread and unread[-1][1] == position - 1: unread[-1] = (unread[-1][0], position) else: unread.append((position, position)) for first, last in unread: start, end = tokens[first].start, tokens[last].end - records.append(_uncovered_record(variable[start:end], start, end, ())) + records.append( + _uncovered_record( + variable[start:end], + start, + end, + (), + _run_measure_values(measure_values, first, last), + ) + ) records.extend( _uncovered_record(variable[run.start : run.end], run.start, run.end, [run.span]) @@ -3050,6 +3081,90 @@ def _coverage_reading( ) +def _uncarried_measure_words( + state: ResolveState, tokens: Sequence[Token] +) -> dict[int, tuple[GraphRef, ...]]: + """The matchable measure values each word of the variable, whose tokens are `tokens`, + names that the answer over the elected shape and the final bindings does not carry + (`answer_reads_measure`), by token position. Only a shape the measure index describes, + one whose answer carries a value of its population type axis (the financial flow, the + financial obligation), has rival measure values to name, so on any other a word names + none. + + A word names a value when every name of its reading holds it, folded as the code index + folds words (`fold_word`): the label and each synonym of the value and of each alias of + it (`MeasureIndex.reading_set`). No span of the variable may name the value. "mobilised" + names `amountMobilised`, "amount mobilised", "amounts mobilised from the private sector" + and "private finance mobilised", where "private" and "finance" name nothing, and + "inflation" names no `RealValue`, which "real terms" names beside its alias "inflation + adjusted". A function word names none, and neither does a word inside a span of the + variable read in every code system together, which is that span's + (`ResolveState.variable_spans_in_every_system`), nor a word of a name of a measure value + the answer carries, matchable or not: "financial" on a shape whose population type is + the financial flow names no financial obligation.""" + measure_index = get_measure_index() + shape, bindings = state.shape_draft, state.bindings + spans = state.variable_spans_in_every_system + spanned = frozenset().union( + *( + measure_index.reading_set(row.dcid) + for span in spans + for row in span.rows + if isinstance(row, MeasureValueRow) + ) + ) + described = False + carried: set[str] = set() + named: dict[str, list[GraphRef]] = {} + for value in measure_index.values: + if value.same_reading_as is not None: + continue + reading = measure_index.reading_set(value.dcid) + names = [ + {fold_word(word.text) for word in word_tokens(text)} + for row in measure_index.values + if row.axis == value.axis and row.dcid in reading + for text in (row.label, *row.synonyms) + ] + match = QualifierMatch( + code_system=value.axis, + code_dcid=value.dcid, + label=value.label, + source="label", + matched_text=value.label, + ) + if answer_reads_measure(shape, match, bindings): + described |= value.axis == "populationType" + carried.update(*names) + elif value.matchable and value.dcid not in spanned: + for word in set.intersection(*names): + named.setdefault(word, []).append(GraphRef(ref=value.dcid, label=value.label)) + if not described: + return {} + return { + position: tuple(values) + for position, token in enumerate(tokens) + if token.kind == "word" + and token.text.casefold() not in FUNCTION_WORDS + and not any(span.start <= token.start and token.end <= span.end for span in spans) + and (word := fold_word(token.text)) not in carried + and (values := named.get(word)) + } + + +def _run_measure_values( + measure_values: Mapping[int, Sequence[GraphRef]], first: int, last: int +) -> list[GraphRef]: + """The measure values the words at token positions `first`..`last` name + (`_uncarried_measure_words`), once each, in query order.""" + values = { + ref.ref: ref + for position in range(first, last + 1) + for ref in measure_values.get(position, ()) + } + return list(values.values()) + + def _occurrences(tokens: Sequence[Token], texts: Iterable[str | None]) -> list[tuple[int, int]]: """The character range of each occurrence among `tokens` of each text of `texts` (`_token_occurrences`), skipping an empty or absent one.""" @@ -3760,16 +3875,22 @@ def _unsupported_records( def _uncovered_record( - text: str, start: int, end: int, spans: Sequence[QualifierSpan] + text: str, + start: int, + end: int, + spans: Sequence[QualifierSpan], + measure_values: Sequence[GraphRef] = (), ) -> ResolvedQualifier: """The record of the uncovered words `text` at `start`..`end`: ``unbound``, naming the - code rows of `spans` once each, or ``no_vocabulary`` when `spans` name none.""" + code rows of `spans` once each and the `measure_values` its words name + (`_uncarried_measure_words`), or ``no_vocabulary`` when they name none.""" named = { row.dcid: GraphRef(ref=row.dcid, label=row.label) for span in spans for row in span.rows if isinstance(row, CodeRow) } + named.update((ref.ref, ref) for ref in measure_values) return ResolvedQualifier( text=text, named=list(named.values()), @@ -3831,8 +3952,10 @@ def _resolve_query_qualifiers(state: ResolveState) -> list[ResolvedQualifier]: system records nothing more, and no unavailable term inside it is recorded. 9. Under ``SearchSettings.qualifier_coverage="refuse"``, the variable's words no reader covers beside a recognised code, each run ``unbound`` with evidence - ``"uncovered"``, the words beside none that the binder rates unread, each stretch - ``no_vocabulary`` with that evidence, and every other exclusion or comparison + ``"uncovered"``, the words beside none that name a measure value the answer does + not carry or that the binder rates unread, each stretch with that evidence, + ``unbound`` naming the measure values its words name, else ``no_vocabulary``, + and every other exclusion or comparison phrase (`_coverage_records`). In log mode election vetting logs them instead (`phase_election._repair_bindings`) and the answer reads without them. 10. Each value the bind drafted on a property the elected shape has no slot for and a diff --git a/src/search/models/trace.py b/src/search/models/trace.py index 8de5eb32..6138123a 100644 --- a/src/search/models/trace.py +++ b/src/search/models/trace.py @@ -32,19 +32,22 @@ class ResolvedQualifier(StrictModel): `query_text`, or from an admitted qualifier tie (see `_resolve_query_qualifiers`). `text` is the verbatim span of whichever source named it. `named` is every code row -- any system -- that span - names in the packaged index; - empty when the text names no code at all (a `no_vocabulary` term). `slot` + names in the packaged index, and every measure value it names that the + answer does not carry (a dcid of the index's measure values, such as + `amountMobilised`); + empty when the text names neither (a `no_vocabulary` term). `slot` is the constraint property dcid of the `slot_filters` entry whose binding consumed the term, or, on an `"unbound"` value the answer's shape has a slot for but leaves unmet, that slot's property; `None` otherwise. `disposition` is `"bound"` when some final binding on a named row's code system covers - it (that row or a descendant), `"unbound"` when a row was named but no - binding covers it, and `"no_vocabulary"` when the term names no code at - all: an LLM-reported phrase (`bind`'s own `unplaced` output) that no - taxonomy in this deployment's index can express, an exclusion or comparison - phrase (`"unsupported"`), or words beside no code the binder rated as not - stated on the elected shape (`"uncovered"`). + it (that row or a descendant), `"unbound"` when a code row was named but + no binding covers it or a measure value was named that the answer does not + carry, and `"no_vocabulary"` when the term names neither: an LLM-reported + phrase (`bind`'s own `unplaced` output) that no taxonomy in this + deployment's index can express, an exclusion or comparison phrase + (`"unsupported"`), or words beside no code the binder rated as not stated + on the elected shape (`"uncovered"`). `start` and `end` locate the term in the part's variable phrase. `evidence` says what read it, and `None` means a code's label or synonym matched the term as the query @@ -57,10 +60,14 @@ class ResolvedQualifier(StrictModel): (`phase_rank_bind._model_records`); `named` holds the chosen codes. - `"uncovered"` marks words no reader covers on the elected shape (`phase_rank_bind._coverage_records`). Beside a recognised code they are recorded - `"unbound"` with that code's rows as `named`. Beside no code they are recorded - `"no_vocabulary"` with `named` empty when the binder rates them below - `SearchSettings.stated_word_bar` as not stated by the shape's series; a word it rates - at the bar or above, or leaves unrated, reads as stated. + `"unbound"` with that code's rows as `named`. On a shape the measure index describes, + a word that every name of a measure value's reading holds names that value when no + span names it and the answer does not carry it, and its record adds the value to + `named`. Beside no code such a word is recorded `"unbound"` naming the value, whatever + the binder rates it; "mobilised" names `amountMobilised` on an amount series. Other + words beside no code are recorded `"no_vocabulary"` with `named` empty when the + binder rates them below `SearchSettings.stated_word_bar` as not stated by the shape's + series; a word it rates at the bar or above, or leaves unrated, reads as stated. - `"unavailable"` marks a code term the elected shape cannot apply: one naming only codes no shape can bind, or, on a part naming a dataset, one no row of that dataset holds (`phase_rank_bind._unavailable_qualifiers`), recorded `"unbound"`. @@ -81,7 +88,8 @@ class ResolvedQualifier(StrictModel): named: list[GraphRef] = Field( description=( "every code row the span names in the packaged index, any code " - "system; empty when none did" + "system, and every measure value it names that the answer does not " + "carry; empty when none did" ) ) slot: str | None = Field( @@ -95,9 +103,9 @@ class ResolvedQualifier(StrictModel): disposition: Literal["bound", "unbound", "no_vocabulary"] = Field( description=( '"bound" when a final binding covers a named row (or a ' - 'descendant of one); "unbound" when a row was named but no ' - 'binding covers it; "no_vocabulary" when the term names no code ' - "at all" + 'descendant of one); "unbound" when a code row was named but no ' + "binding covers it, or a measure value was named that the answer " + 'does not carry; "no_vocabulary" when the term names neither' ) ) start: int | None = Field( @@ -115,8 +123,9 @@ class ResolvedQualifier(StrictModel): description=( 'what read the term: a code label or synonym, the bind model, "uncovered" ' "for words no reader covers on the elected shape, unbound beside a recognised " - "code and no_vocabulary beside none when the binder rated them as not stated " - 'by its series, "unavailable" for a code no table holds, "unsupported" for a ' + "code or naming a measure value the answer does not carry, and no_vocabulary " + "beside none when the binder rated them as not stated by its series, " + '"unavailable" for a code no table holds, "unsupported" for a ' "distinction no binding expresses" ), ) diff --git a/tests/engine/test_qualifier_guard.py b/tests/engine/test_qualifier_guard.py index 7efca5e3..f0139eb9 100644 --- a/tests/engine/test_qualifier_guard.py +++ b/tests/engine/test_qualifier_guard.py @@ -34,6 +34,7 @@ from search.engine.interpret import Recall from search.engine.qualifier_match import QualifierSpan from search.engine.quantity import NamedDenominator +from search.engine.region.phase_answer import _assemble_carve_outs from search.engine.region.phase_election import _repair_bindings from search.engine.region.phase_rank_bind import ( _coverage_reading, @@ -70,10 +71,14 @@ _value_requirements, unbind_unrealisable, ) -from search.engine.retrieval.retrieve import BoundValuesUnrealisable, MemberConflict +from search.engine.retrieval.retrieve import ( + BoundValuesUnrealisable, + Materialised, + MemberConflict, +) from search.engine.role_text import DoorRole from search.engine.shape import ShapeDraft, SlotKeyDraft -from search.models import GraphRef, NoData, NoDataPart, ResolvedQualifier, Spec +from search.models import CoverageBare, GraphRef, NoData, NoDataPart, ResolvedQualifier, Spec from tests.conftest import minimal_spec from tests.engine._binders import FakeAsk, RatingBinder, keep_the_pick from tests.engine._builders import place_sets @@ -913,6 +918,104 @@ def test_the_binder_rates_the_words_left_unrated_once_the_bindings_settle( assert lost.reads == degraded +# A CRS amount series of financial flows, the population type the measure index describes, +# whose name states none of the words the rows below ask about. +_PSI_SHAPE = replace( + _answering(_shape(*_CRS_SLOTS), "amount", "provenance/OECDCreditorReportingSystem"), + pop_type_dcid="FinancialFlow", + sv_arc_facts={"psi_member": {"name": {"nodes": [{"value": "Gross disbursements"}]}}}, +) +_PSI_BOUND = (("crsTypeOfFlow", ["CRSTypeOfFlow60"]),) +_ODA_BOUND = ("crsTypeOfFlow", ["CRSTypeOfFlow10"]) +_HEALTH_ODA_BOUND = (_ODA_BOUND, ("crsPurpose", ["CRSPurposeCode120"])) + + +@pytest.mark.parametrize( + ("variable", "shape", "bound", "refused"), + [ + ( + "private sector investment mobilised", + _PSI_SHAPE, + _PSI_BOUND, + [("mobilised", ["amountMobilised"])], + ), + # Beside a code span the record names the measure value beside the code's rows. + ( + "health ODA mobilised", + _PSI_SHAPE, + _HEALTH_ODA_BOUND, + [("mobilised", ["CRSTypeOfFlow10", "amountMobilised"])], + ), + # A series measuring the amount mobilised carries it. + ( + "private sector investment mobilised", + replace(_PSI_SHAPE, meas_prop_dcid="amountMobilised"), + _PSI_BOUND, + [], + ), + # "inflation" is no word of "real terms", so it names no RealValue beside its alias + # "inflation adjusted", and "grant equivalent" is a span naming its value. + ("inflation", _PSI_SHAPE, (), []), + ("grant equivalent ODA", _PSI_SHAPE, (_ODA_BOUND,), []), + # A shape outside the measure family has no rival measure value to name. + ("seasonally adjusted unemployment", _READER_TAX, (), []), + ("inflation rate", _READER_TAX, (), []), + ("debt service obligations", _READER_TAX, (), []), + ("financial inclusion", _READER_TAX, (), []), + ("full time equivalent teachers", _READER_TAX, (), []), + ("capital flow", _READER_TAX, (), []), + ("private sector investment mobilised", _READER_TAX, (), []), + ], + ids=[ + "mobilised-refuses", + "mobilised-beside-a-code", + "mobilised-carried", + "alias-word", + "spanned-value", + "off-family-adjusted", + "off-family-inflation", + "off-family-obligations", + "off-family-financial", + "off-family-equivalent", + "off-family-flow", + "off-family-mobilised", + ], +) +def test_a_word_naming_a_measure_value_the_answer_lacks_refuses_whatever_its_rating( + search_settings, variable, shape, bound, refused +): + """A word every name of a matchable measure value holds, which no span names, refuses + ``qualifier_not_bound`` on a financial-flow answer that does not carry the value, + though the binder would rate it stated, and the binder is not asked about it: + "mobilised" names the amount mobilised on an amount series bound to flow type 60, whose + name reads "private" and "sector".""" + search_settings(qualifier_coverage="refuse") + state = _state( + variable=variable, + shape_draft=shape, + bindings=[ + _binding(prop, kind="value" if len(dcids) == 1 else "set", value_dcids=dcids) + for prop, dcids in bound + ], + unplaced=[], + entities=(), + ) + state.capped_shapes = [shape] + state.binder = RatingBinder(GeminiBinder()) + _run(state) + spec = Spec.model_validate(minimal_spec()) + materialised = Materialised( + sv_dcids=["psi_member"], has_data=True, coverage=CoverageBare(has_data=True) + ) + result = asyncio.run(_assemble_carve_outs(state, materialised, spec)) + + terms = result.unbound_qualifiers if result is not None else () + assert (result is not None and result.no_data_reason == "qualifier_not_bound") == bool(refused) + assert [(q.text, [ref.ref for ref in q.named]) for q in terms] == refused + if refused: + assert all("mobilised" not in request.words for request in state.binder.requests) + + @pytest.mark.parametrize( ("rating", "records"), [(0.1, ["refugee"]), (0.9, [])], ids=["below-the-bar", "above"] ) From 6c65b075c3c32ec6d51866dabb44baa1be0ac46f Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sat, 24 Oct 2026 21:35:57 +0200 Subject: [PATCH 5/7] fix: judge sampled sole values on the walked family, and never answer a 202 done (#123) * fix(search): judge a sole value claimed from a sample against the walked family _validate_sole_option_bindings knew whether a sole-value claim rested on the whole family or on a recall sample, and dropped that knowledge, so the member walk read the family filtered by a claim it never tested. A one-member sample asserting gender=Female answered the female series where the question asked for the total. A claim from an incomplete population now carries sampled=True (excluded from every dump and schema). The walk reads sampled slots open, and _judge_sampled_claims confirms each claim on the walked members or refutes it and unbinds the slot. An unsettled walk leaves the claim standing under membership_unverified. No extra member read. Refs #101 (cherry picked from commit 8214f72fdad8df9065b5c624c9a2fcb95800d401) * test(eval): expect the DFO GNI series for a plain GNI question dq-dac-016 "GNI of Denmark" expected the World Bank family base.gni, but the DFO series dev answers with carries the World Bank indicator NY.GNP.MKTP.CD, and DAC1 is its only provenance for Denmark. Pin the dac1.gni family to that series, point the case at it, and drop the known failure, whose issue number did not exist. (cherry picked from commit f98b2b8994c8d38de2f3dc5cd2251c36f5441e8b) * fix(datasets): reset every done operation a submission finds A submission reaches the operation store only when the subject's head could not answer it, yet create_or_get returned a done row at another version unchanged. Two renderings of one over-budget subject make that deterministic: a csv mint after an xlsx rebuild answered 202 done, and its poll served the old version. A client following the contract also raises on a 202 done, which carries no manifest. create_or_get now resets every done row to pending and enqueues it again, and the run serves what the head rule gives at run time. This removes the rejected-version plumbing through submit_over_budget, current and spec_id, and the refresh door's done branch, two-pass loop and RefreshRebuildUnservable. C187, the datasets README and datasets-sync-ceiling state the rule. (cherry picked from commit 37b2e6ff660ebcf0c6485acd4634988daa1ff35c) * fix(datasets): serve the head when a full backlog refuses its submission A submission that found its operation just completed reset it and went through backlog admission again, so with a full backlog it answered 503 although the head already served the result. serve_mint and the definition door's _materialise_and_record now catch OperationBacklogFull, read and judge the head once more under their own head rule, and answer from it when it serves; otherwise the refusal stands. Admission itself is unchanged, and the recheck costs one head read on the 503 path only. (cherry picked from commit 877a2ea20558a79c12b004eb08c4e96bd1082d52) * fix(search): keep a place's own name when another shares its words 0b5ea7733 renamed an entity to the span the user wrote whenever the span's ambiguity settled it. For "aid from Congo to Congo" both ends became "Congo", and since the parsed door reads one text as one place per part, every option pinned both ends to the same country. _recover_entity_surfaces now renames a name to its span only when no other name in the part would carry the same text, so the two ends keep the extractor's names and independent choices, while "ODA disbursements from Germany to Congo" still reads "Congo". (cherry picked from commit 5d42bbd6ba3bf4f0b159abebf331554d141bce4b) * fix(datasets): pace only a failed operation at the floor, and state the backlog recheck * test(live-refresh): give the composed worker the head store the deployed worker has * fix(search): bind the value an unexported family is told apart by over a sampled claim bind_shared_values binds the value an unexported family shares on a property the exported rows also carry, since that value is what tells the family apart (IDS debt service: financialFlowType=ServicingPayment on a five-tuple the index shares with CRS). _unexported_shared_values counted a sampled claim as already bound and skipped the slot, so only the sample backed the claim; the walk then read the slot open, listed interest payments too, and refuted it, and "total debt service of Kenya" served both flows unbound (eval dq-ids-001, 002, 014, 020, 035). A sampled claim now leaves the slot open to the rule, and a claim of the shared value is kept with its flag cleared. * build: pin spec-kit 0d19172, which reads proxy refusals as problem documents --- corpus/dac.yaml | 6 +- corpus/families/dac.yaml | 6 +- docs/conformance-datasets-api-v1.md | 2 +- docs/datasets-sync-ceiling.md | 11 +- packages/dfo-data-tools/pyproject.toml | 8 +- pyproject.toml | 10 +- src/datasets_api/README.md | 6 +- src/datasets_api/current.py | 115 +++++++++++----- src/datasets_api/doors/refresh.py | 90 ++++--------- src/datasets_api/kinds/spec_id.py | 54 +++++--- src/datasets_api/operation_store.py | 60 +++------ src/datasets_api/operations/submit.py | 17 +-- src/search/engine/bind.py | 8 ++ src/search/engine/entry/raw_text.py | 15 ++- src/search/engine/region/phase_election.py | 40 +++++- .../engine/region/phase_place_materialise.py | 85 +++++++++++- src/search/engine/retrieval/bound_values.py | 45 ++++++- tests/dataset_store/test_live.py | 38 ++---- tests/datasets_api/refusal_wire_golden.json | 62 ++++----- tests/datasets_api/test_meta_doors.py | 10 +- tests/datasets_api/test_operation_fencing.py | 1 - tests/datasets_api/test_operation_store.py | 97 ++----------- .../test_operation_submit_door.py | 10 -- tests/datasets_api/test_refresh_door.py | 11 +- .../test_refusal_wire_equivalence.py | 6 +- tests/datasets_api/test_task_client.py | 1 - tests/datasets_api/test_worker.py | 2 - tests/datasets_api/test_worker_app.py | 1 - tests/datasets_api/test_worker_e2e.py | 127 +++++++++++++++++- tests/engine/test_choice_options.py | 60 ++++++++- tests/engine/test_membership_authority.py | 46 ++++++- tests/engine/test_place_scope_default.py | 62 +++++++-- tests/engine/test_raw_text_entity_mentions.py | 39 ++++++ tests/engine/test_role_class_shapes.py | 84 ++++++++++-- tests/engine/test_sole_option_validation.py | 10 +- tests/live_refresh/composed_app.py | 1 + uv.lock | 24 ++-- 37 files changed, 844 insertions(+), 426 deletions(-) diff --git a/corpus/dac.yaml b/corpus/dac.yaml index f43be829..6f6715ef 100644 --- a/corpus/dac.yaml +++ b/corpus/dac.yaml @@ -161,11 +161,9 @@ - id: dq-dac-016 query: GNI of Denmark - why: DAC1 carries GNI only as the denominator of the ODA ratio; a plain GNI question elects the World Bank series. + why: The DFO series carries the World Bank indicator NY.GNP.MKTP.CD, and DAC1 serves it for Denmark. tags: {theme: macro, entity: country, direction: subject, unit: default, time: none, operation: none, breakdown: none, door: raw_text, phrasing: casual, overlap: [dac1], probe: [source_election]} - # Known failure: "GNI of Denmark" returns candidates, not a definite answer. - known_failure: {issue: 1249, checks: [disposition]} - expect: {definite: {measure: {family: base.gni}, entities: [{dcid: country/DNK, role: observationAbout}]}} + expect: {definite: {measure: {family: dac1.gni}, entities: [{dcid: country/DNK, role: observationAbout}]}} - id: dq-dac-017 query: oda gni ratio swedan diff --git a/corpus/families/dac.yaml b/corpus/families/dac.yaml index 780149a9..8f624d60 100644 --- a/corpus/families/dac.yaml +++ b/corpus/families/dac.yaml @@ -59,8 +59,10 @@ dac1.private_mobilised: shape: {measured_property: amountMobilised} dac1.gni: origin: dfo - dataset: {gl: DAC1 provenance} - shape: {gl: GNI series about the provider} + dataset: provenance/OECDDAC1 + shape: {population_type: EconomicUnit, measured_property: primaryIncome, stat_type: measuredValue, measurement_qualifier: Nominal, measurement_denominator: null} + fixed: {fixedCapitalConsumptionBasis: GrossOfFixedCapitalConsumption} + stat_vars: [PrimaryIncome_EconomicUnit_GrossOfFixedCapitalConsumption_Nominal] dac2a.oda_net_received: origin: dfo dataset: provenance/OECDDAC2A diff --git a/docs/conformance-datasets-api-v1.md b/docs/conformance-datasets-api-v1.md index 6ac3543b..53b6d127 100644 --- a/docs/conformance-datasets-api-v1.md +++ b/docs/conformance-datasets-api-v1.md @@ -176,7 +176,7 @@ Five routes are outside the V1 contract text. The code registry's mappings for ` | C93 | A mint or definition request within the sync budget (`DATASETS_SYNC_BUDGET_ROWS`, `DATASETS_SYNC_BUDGET_REQUESTS`) completes synchronously with 200; a request over budget answers 202 Accepted with a `Location` header and an `OperationEnvelope` instead of running inline | implemented | | | C94 | GET /api/datasets/operations/{operation_id} polls the operation: `pending`/`running` answers 200 with progress, `done` answers 200 with the same manifest shape a synchronous mint returns, listing the artifact for the rendering its request posted plus statistics artifacts (a `done` operation whose rendering the snapshot lists no artifact for is failed `internal_error`, retryable, and answered `operation_failed`), `failed` answers `operation_failed` (500) with the operation's refusal code in `cause.code` and, for a deterministic failure, the `detail`, `pointer`, `alternatives`, `issues`, `constraint_id`, `ceiling_kind`, `count`, `cap` and `cause.entities` of the refusal the worker stored; the capability document reports `operation_envelope_active: true` | implemented | | | C94a | An unknown operation id refuses `operation_not_found`, 404. A `done` operation's poll checks no caller. A caller who can compute an operation id from a request can also submit that request, so this route serves the manifest on the same terms a synchronous mint or the public snapshot route already would, with no principal check of its own and none of the publish door's allowlist | implemented | | -| C187 | A 202 operation reply reads `pending` or `running`, apart from the two cases below. A submission reaches the operation store only after the head check has rejected the subject's head, so `OperationStore.create_or_get` resets a `done` operation whose `content_version` is the rejected head's, or any `done` operation when the subject has no head, to `pending` under a new submission and enqueues it again. A `done` operation at another version completed after the submitter read the head, and the 202 reads `done` with a poll that serves that version's manifest. A deterministic failure is returned unchanged, and the 202 reads `failed` | implemented | Pinned by `tests/datasets_api/test_operation_store.py`, `tests/dataset_store/test_live.py` and `tests/datasets_api/test_worker_e2e.py`. | +| C187 | A 202 operation reply reads `pending` or `running`, or `failed` for a deterministic failure. A submission reaches the operation store only when the subject's head could not answer it, so `OperationStore.create_or_get` resets every `done` operation it finds to `pending` under a new submission and enqueues it again, whatever version the row holds, and the run serves what the head rule gives at run time. A retryable failure, a `running` operation whose lease has ended and a `pending` operation past the deadline reset the same way. A deterministic failure is returned unchanged | implemented | Pinned by `tests/datasets_api/test_operation_store.py`, `tests/dataset_store/test_live.py` and `tests/datasets_api/test_worker_e2e.py`. | | C178 | A definition's operation stores `member_dcids`, the StatVars the submitting door resolved as sorted bare dcids, and hashes them into the operation id, so two submissions of one spec share an operation only when their doors resolved the same members. The worker resolves the members again from the identity alone and compares them with `member_dcids`. When they differ, the graph changed after submission and the operation fails `upstream_version_changed`, retryable, answered on the poll as `operation_failed` with `cause.code` `upstream_version_changed`, and a resubmission resolves the new members under a new operation id. A member change never resets a `done` operation; C187 states when one resets | implemented | Pinned by `tests/datasets_api/test_worker.py`. | ## The pin response diff --git a/docs/datasets-sync-ceiling.md b/docs/datasets-sync-ceiling.md index 85c35c50..f5c58e5f 100644 --- a/docs/datasets-sync-ceiling.md +++ b/docs/datasets-sync-ceiling.md @@ -62,11 +62,14 @@ set is never returned to a submission that resolved another. `0024_operation_attempt_fencing.sql`) stores the request identity, rendering options, the admission coverage stamp, catalogue build and estimate, the lifecycle state (`pending`, `running`, `done`, `failed`), and the dispatch and fencing columns described below. A submit that finds an -existing row returns it. A `failed` row marked `retryable`, a `running` row whose lease has ended, -or a `pending` row past the operation deadline (see [Abandoned runs](#abandoned-runs)) is reset to -`pending` as a new submission, with `submission_count` incremented. A new row or a reset first +existing row returns it. A `done` row, a `failed` row marked `retryable`, a `running` row whose +lease has ended, or a `pending` row past the operation deadline (see +[Abandoned runs](#abandoned-runs)) is reset to `pending` as a new submission, with +`submission_count` incremented. A `done` row resets because a submission reaches the store only +when the subject's head could not answer it, and the rerun serves what the head gives at run +time. A new row or a reset first counts the other live rows (every `pending` or `running` row not abandoned) under an advisory -lock and refuses `service_unavailable` (503, retryable) at `DATASETS_OPERATION_BACKLOG_MAX` (20). +lock and refuses `service_unavailable` (503, retryable) at `DATASETS_OPERATION_BACKLOG_MAX` (20). A backlog refusal re-reads the subject's head, and when the head serves the request it answers 200 from the head. Every mint and definition door call that reaches this point raises `OperationSubmitted`, caught by the door itself. The response is `202 Accepted` with a `Location` header pointing at `/api/datasets/operations/{operation_id}` and a body carrying `operation_id`, `status`, `poll`, and diff --git a/packages/dfo-data-tools/pyproject.toml b/packages/dfo-data-tools/pyproject.toml index 7db3cebd..3508f332 100644 --- a/packages/dfo-data-tools/pyproject.toml +++ b/packages/dfo-data-tools/pyproject.toml @@ -88,10 +88,10 @@ graph-release = ["google-cloud-spanner>=3.57", "dfo-data-tools[coverage]"] [tool.uv.sources] # See the root pyproject.toml's dfo-spec-kit source comment for the # pin-consistency invariant this rev must satisfy. -dfo-spec-kit-frame-grammar = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "ba976a2a74173e1c31b6c03d802cbcf71a022273", subdirectory = "python/frame-grammar" } -dfo-spec-kit-components = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "ba976a2a74173e1c31b6c03d802cbcf71a022273", subdirectory = "python/dfo-spec-kit-components" } -dfo-spec-kit-provenance-docs = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "ba976a2a74173e1c31b6c03d802cbcf71a022273", subdirectory = "python/provenance-docs" } -dfo-spec-kit-descriptions = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "ba976a2a74173e1c31b6c03d802cbcf71a022273", subdirectory = "python/descriptions" } +dfo-spec-kit-frame-grammar = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "0d19172e3695c311ed48ff43a885a9995869e6a5", subdirectory = "python/frame-grammar" } +dfo-spec-kit-components = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "0d19172e3695c311ed48ff43a885a9995869e6a5", subdirectory = "python/dfo-spec-kit-components" } +dfo-spec-kit-provenance-docs = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "0d19172e3695c311ed48ff43a885a9995869e6a5", subdirectory = "python/provenance-docs" } +dfo-spec-kit-descriptions = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "0d19172e3695c311ed48ff43a885a9995869e6a5", subdirectory = "python/descriptions" } [build-system] requires = ["uv_build>=0.11,<0.12"] diff --git a/pyproject.toml b/pyproject.toml index 41378941..b44cf731 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -241,16 +241,18 @@ dfo-data-tools = { workspace = true } # The registry refuses a part's `source_code` on search.resolve with # `source_code_not_held`, `source_code_ambiguous`, and # `source_code_shares_variable`. -# The pin is ba976a2 on spec-kit's dev branch. It registers +# The pin is 0d19172 on spec-kit's dev branch. It registers # slot_default_not_applied, the three source-code refusals and # whole_total_available, retires starved_axis_binding_restored, and holds the # client contract `dfo-spec-kit-client-contract` reads, whose ref format admits # a `without_party_totals` rendering and whose warning codes include -# `ref_digest_changed`, and it registers the `timed_out` no_data reason. +# `ref_digest_changed`, and it registers the `timed_out` no_data reason. The +# client contract reads the website proxy's refusals as problem documents and +# refuses a 202 that reads `done`. # Follow-up: move all six to a real spec-kit tag once one is cut, so a future # force-push or GC on spec-kit's dev can't silently break this lock. -dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "ba976a2a74173e1c31b6c03d802cbcf71a022273", subdirectory = "python/spec-kit" } -dfo-spec-kit-client-contract = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "ba976a2a74173e1c31b6c03d802cbcf71a022273", subdirectory = "python/client-contract" } +dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "0d19172e3695c311ed48ff43a885a9995869e6a5", subdirectory = "python/spec-kit" } +dfo-spec-kit-client-contract = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "0d19172e3695c311ed48ff43a885a9995869e6a5", subdirectory = "python/client-contract" } [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/datasets_api/README.md b/src/datasets_api/README.md index 42d6ae09..0640166c 100644 --- a/src/datasets_api/README.md +++ b/src/datasets_api/README.md @@ -141,7 +141,7 @@ Synchronous materialisation admits on an exact `series_coverage`-based estimate ## Async operations and the worker -A mint or definition request over the sync budget is recorded as an operation (`datasets.operation`) instead of running inline: the door answers `202 Accepted` with a `Location` header and an `OperationEnvelope` (`operation_id`, `status`, `poll`, `retry_after_ms`). The operation id is derived deterministically from the request identity (the subject id and, for a definition, the members the submitting door resolved, which the worker resolves again, failing the operation `upstream_version_changed`, retryable, when they differ, so a resubmission starts a new operation for the new members), the origin catalogue build bound to the request, and the rendering key of `options` (format/groupby/pivot/columns/...), so two requests for the same subject that differ only in rendering get distinct operations, a rendering-caused deterministic failure never fails a resubmit asking for a different rendering, and a new catalogue build, which can admit different provenances, starts a new operation. A coverage build leaves the id unchanged, so a done operation stays reachable. The row records that build as `origin_catalogue_build_id`. An estimate over `DATASETS_ASYNC_MAX_ROWS` or `DATASETS_ASYNC_MAX_REQUESTS` refuses `over_ceiling` before any operation is recorded, and a new operation at `DATASETS_OPERATION_BACKLOG_MAX` (default 20) live rows, the `pending` and `running` rows that are not abandoned, refuses `service_unavailable` (503, retryable, `Retry-After`); a resubmit of an already-recorded operation is exempt. One Cloud Tasks task is created per submission: the submit commits the `pending` row, creates the task, then records `dispatched_at`. An enqueue that fails or times out leaves `dispatched_at` null, and a poll 30 seconds or more after the submission creates the task again under the same name. Cloud Tasks deduplicates by name, and `AlreadyExists` counts as success. `GET /api/datasets/operations/{op_id}` polls it: `pending`/`running` answers 200 with progress, `done` answers 200 with the same manifest shape a synchronous mint returns, each artifact under its download link, listing the artifact for the rendering its request posted plus statistics artifacts, and `failed` answers `operation_failed` (500) with the operation's refusal code in `cause.code` and its stored `retryable` flag. A deterministic failure also stores the refusal the worker classified (`datasets.operation.refusal`), and the poll body carries its `detail`, `pointer`, `alternatives`, `issues` and ceiling fields. A `done` operation whose rendering the snapshot lists no artifact for (a bundle linked only as incomplete, a bundle of a superseded rendition, an expired working-tier artifact) is failed `internal_error`, retryable, by that poll and answered `operation_failed`, so a resubmit of the identical request re-runs it. The poll carries no principal check in any state: a `done` manifest is the one the public mint returns and the public snapshot route lists for any caller. A `running` operation whose lease has ended (`claimed_at + lease_seconds`) is failed `internal_error`, retryable, by the next poll when the task's final delivery claimed it. One claimed by an earlier delivery polls as `running` until the next delivery claims it. That operation, and a `pending` one, are failed the same way once `DATASETS_OPERATION_DEADLINE_SECONDS` have passed since the dispatch, or the submission when no dispatch is recorded. A task delivery that arrives after that deadline fails the operation the same way without running it, and the daily `datasets-maintenance age-operations` job does so for every such operation, polled or not. A resubmit of the identical request resets a failed-retryable operation, or a `running` one whose lease has ended, to a new submission. Terminal operations stay pollable for `operation_retention_days` (published on `/api/datasets/capabilities`); the same job then deletes `done` and `failed` rows last updated longer ago, and a poll for one answers `operation_not_found`. +A mint or definition request over the sync budget is recorded as an operation (`datasets.operation`) instead of running inline: the door answers `202 Accepted` with a `Location` header and an `OperationEnvelope` (`operation_id`, `status`, `poll`, `retry_after_ms`). The operation id is derived deterministically from the request identity (the subject id and, for a definition, the members the submitting door resolved, which the worker resolves again, failing the operation `upstream_version_changed`, retryable, when they differ, so a resubmission starts a new operation for the new members), the origin catalogue build bound to the request, and the rendering key of `options` (format/groupby/pivot/columns/...), so two requests for the same subject that differ only in rendering get distinct operations, a rendering-caused deterministic failure never fails a resubmit asking for a different rendering, and a new catalogue build, which can admit different provenances, starts a new operation. A coverage build leaves the id unchanged, so a done operation stays reachable. The row records that build as `origin_catalogue_build_id`. An estimate over `DATASETS_ASYNC_MAX_ROWS` or `DATASETS_ASYNC_MAX_REQUESTS` refuses `over_ceiling` before any operation is recorded, and a new operation, or the reset of a recorded one, at `DATASETS_OPERATION_BACKLOG_MAX` (default 20) other live rows, the `pending` and `running` rows that are not abandoned, refuses `service_unavailable` (503, retryable, `Retry-After`) unless the subject's head, read again after the refusal, now answers the request, as it does when the operation's own run completed in between; a resubmit that joins a live operation is exempt. One Cloud Tasks task is created per submission: the submit commits the `pending` row, creates the task, then records `dispatched_at`. An enqueue that fails or times out leaves `dispatched_at` null, and a poll 30 seconds or more after the submission creates the task again under the same name. Cloud Tasks deduplicates by name, and `AlreadyExists` counts as success. `GET /api/datasets/operations/{op_id}` polls it: `pending`/`running` answers 200 with progress, `done` answers 200 with the same manifest shape a synchronous mint returns, each artifact under its download link, listing the artifact for the rendering its request posted plus statistics artifacts, and `failed` answers `operation_failed` (500) with the operation's refusal code in `cause.code` and its stored `retryable` flag. A deterministic failure also stores the refusal the worker classified (`datasets.operation.refusal`), and the poll body carries its `detail`, `pointer`, `alternatives`, `issues` and ceiling fields. A `done` operation whose rendering the snapshot lists no artifact for (a bundle linked only as incomplete, a bundle of a superseded rendition, an expired working-tier artifact) is failed `internal_error`, retryable, by that poll and answered `operation_failed`, so a resubmit of the identical request re-runs it. The poll carries no principal check in any state: a `done` manifest is the one the public mint returns and the public snapshot route lists for any caller. A `running` operation whose lease has ended (`claimed_at + lease_seconds`) is failed `internal_error`, retryable, by the next poll when the task's final delivery claimed it. One claimed by an earlier delivery polls as `running` until the next delivery claims it. That operation, and a `pending` one, are failed the same way once `DATASETS_OPERATION_DEADLINE_SECONDS` have passed since the dispatch, or the submission when no dispatch is recorded. A task delivery that arrives after that deadline fails the operation the same way without running it, and the daily `datasets-maintenance age-operations` job does so for every such operation, polled or not. A resubmit of the identical request resets a `done` operation, a failed-retryable one, or a `running` one whose lease has ended, to a new submission. Terminal operations stay pollable for `operation_retention_days` (published on `/api/datasets/capabilities`); the same job then deletes `done` and `failed` rows last updated longer ago, and a poll for one answers `operation_not_found`. A dedicated worker process, built from the same image with `DFO_WORKER_SURFACE=1`, mounts a single internal route (`POST /internal/datasets/operations/{operation_id}/run`) that Cloud Tasks dispatches to, verified by Cloud Run IAM plus an app-level OIDC check. It claims the operation for the submission the task names, with a static lease of 1,620 seconds (`claimed_at + lease_seconds`, under the task's 1,800-second dispatch deadline). A task from an earlier submission answers 200 `superseded`, and a delivery that finds another attempt inside its lease answers 409. Each claim increments the operation's attempt counter, and every later write of the run is fenced on that attempt, so a worker whose lease ended and was reclaimed can no longer write. The operation's own writes carry the attempt in their `WHERE` clause, and the head advance checks it under a share lock on the operation row inside the head's transaction (`HeadFence`), so a run that lost the operation after its version and artifact writes leaves the head and `datasets.materialisation` unchanged and answers 200 `superseded`. The run has a work deadline 1,500 seconds after the claim, checked before the version record and before each artifact render and upload, so a run past it releases without writing. The head advance has no deadline check, since the fence covers one that falls after the deadline and inside the lease. The run takes one materialise permit from the worker process's single-permit semaphore. It reads the published coverage pointer and loads a moved build before it starts, re-estimates the request against that snapshot, records the estimate on the operation, fails `over_ceiling` if the fresh estimate now exceeds the async ceilings, and sizes its fetch guard from it. The run completes the operation only when a snapshot read of the completed version, narrowed to the request's rendering, lists an artifact other than statistics. Otherwise it takes the transient-failure path below. Each finished chunk writes `rows_written`, fenced on the attempt. A refused write stops the run at its next chunk, and it writes nothing more and answers 200 `superseded`. A worker that dies mid-run leaves its lease to end, after which the task's next delivery reclaims it, or, when the dead delivery was the task's last, the next poll fails it retryable. A transient failure before the operation's last Cloud Tasks attempt (`X-CloudTasks-TaskRetryCount` against `DATASETS_OPERATION_MAX_ATTEMPTS`, 3 by default, mirroring the queue's own `retry_config.max_attempts`) releases the operation back to `pending` before answering non-2xx, so the next retry claims immediately. On the last attempt it marks the operation failed instead. `docs/datasets-sync-ceiling.md` covers the full admission, operation, and worker mechanics, including the env vars each side needs and the measured writer memory numbers. @@ -179,9 +179,9 @@ The dataset and rendering come from stored records. A version with no `base_cont A groupby sum that leaves the frame unchanged, as over a dataset with one contributing StatVar, and a party-totals cut that leaves nothing out have no version of their own, and their file is linked under the raw version. A raw version linking such a file and no plain requested-format file refreshes as that rendering. One linking both stands for two renderings. While that rendering of the current data is still the plain frame the refresh answers with it and links both files under it, and once it has a version of its own the refresh refuses `rendering_not_recorded`. -A queued rebuild that is done answers its version only while a snapshot read still lists its rendering's file, the check the operations poll makes. When the file has left storage the refresh fails the row, submits it again, and answers `pending`. +A rebuild over the inline budget submits the request's operation. The submission resets a `done` operation and enqueues it again, so the refresh answers `pending`, and the worker's run advances the head, so a later refresh answers from that head. -Refusals: `refresh_not_permitted` (403) for any caller but the live-refresh principal, checked before the body is read. `spec_not_found` (404) for an unregistered id and `snapshot_not_found` (404) for an unknown version of a registered one. `frame_not_refreshable` (422) for a derived (`ops_`) frame. `rendering_not_recorded` (422) for a derived version with no linked file recording its options, for a raw version that links a plain file and one or more groupby sums or party-totals cuts once those renderings of the current data differ, and for a raw version with no plain file, or a derived version, that links files of two or more different options, whose records do not say which the frame is. A cut and a groupby that left the cut frame unchanged share one version and are two such options. `request_invalid` (400) for a malformed body. `service_unavailable` (503, retryable) when the refresh door's executor is full, a materialise slot is not free within `DATASETS_MATERIALISE_WAIT_SECONDS`, a shared rebuild is still running at the request deadline, the operations backlog is full, or a done rebuild's file has left storage again after the refresh submitted it once more. `internal_error` (500) when the queued rebuild failed without a retry. A rebuild that runs inline refuses as `GET /api/datasets/{id}` does. +Refusals: `refresh_not_permitted` (403) for any caller but the live-refresh principal, checked before the body is read. `spec_not_found` (404) for an unregistered id and `snapshot_not_found` (404) for an unknown version of a registered one. `frame_not_refreshable` (422) for a derived (`ops_`) frame. `rendering_not_recorded` (422) for a derived version with no linked file recording its options, for a raw version that links a plain file and one or more groupby sums or party-totals cuts once those renderings of the current data differ, and for a raw version with no plain file, or a derived version, that links files of two or more different options, whose records do not say which the frame is. A cut and a groupby that left the cut frame unchanged share one version and are two such options. `request_invalid` (400) for a malformed body. `service_unavailable` (503, retryable) when the refresh door's executor is full, a materialise slot is not free within `DATASETS_MATERIALISE_WAIT_SECONDS`, a shared rebuild is still running at the request deadline, or the operations backlog is full. `internal_error` (500) when the queued rebuild failed without a retry. A rebuild that runs inline refuses as `GET /api/datasets/{id}` does. The route shares the definition door's clients and deadline (`DATASETS_DEFINITION_TIMEOUT_SECONDS`) and runs on the work executor (`DATASETS_WORK_MAX_CONCURRENCY`) beside mints and rebuilds, so a burst of refreshes occupies work threads only and leaves the read executor's threads to head hits. A refresh arriving when every work thread is busy refuses `service_unavailable` (503, retryable). diff --git a/src/datasets_api/current.py b/src/datasets_api/current.py index 118e4116..8b69d32c 100644 --- a/src/datasets_api/current.py +++ b/src/datasets_api/current.py @@ -86,7 +86,11 @@ ) from datasets_api.manifest_models import ArtifactEntryModel, Manifest, year_coverage_counted from datasets_api.mint_models import MintOptions -from datasets_api.operation_store import NullOperationStore, OperationStore +from datasets_api.operation_store import ( + NullOperationStore, + OperationBacklogFull, + OperationStore, +) from datasets_api.operations.submit import submit_over_budget from datasets_api.operations.task_client import NullTaskClient, TaskClient from datasets_api.output_options import derive_output_options @@ -1150,6 +1154,32 @@ def _serve_head( ) +def _serve_judged_head( + verdict: _Freshness, + *, + subject_id: str, + deps: DefinitionDeps, + coverage_snapshot: CoverageSnapshot, + options: MintOptions | None, + trace_id: str | None, + load_head: HeadLoader | None, + revoked_under: Sequence[str], +) -> Manifest | str: + """`_serve_head` on the head `verdict` serves, or the reason the call rebuilds.""" + if verdict.head is None: + return cast(str, verdict.rebuild_reason) + return _serve_head( + verdict.head, + subject_id=subject_id, + deps=deps, + coverage_snapshot=coverage_snapshot, + options=options, + trace_id=trace_id, + load_head=load_head, + revoked_under=revoked_under, + ) + + def _labelled(manifest: Manifest, *, trigger: Trigger) -> Manifest: label("trigger", trigger) label("served_from", cast(str, manifest.trace["served_from"])) @@ -1280,6 +1310,9 @@ def serve_current( `admitted`, which sizes the raw-fetch guard. Otherwise an `options` call refuses an estimate over `deps.ceiling` (`refuse_inline_over_ceiling`), and a rendering call refuses Excel over its row cap and submits work over the sync budget as an operation (`submit_over_budget`). + A submission the operation backlog refuses reads the head again and answers from it when it + now serves the call, as it does when that operation's run completed in between, so the + backlog refusal stands only on a head read that follows it. Rebuilds share one run per subject in this process (`deps.rebuilds`). Every other caller waits for that run until its own request deadline, then reads the head again and is served @@ -1356,21 +1389,19 @@ def _check_currency( head, stored=stored, requests=requests, now=now, deps=deps, coverage=snapshot.index ) ) - rebuild_reason = verdict.rebuild_reason - if verdict.head is not None: - served = _serve_head( - verdict.head, - subject_id=subject_id, - deps=deps, - coverage_snapshot=snapshot, - options=options, - trace_id=trace_id, - load_head=load_head, - revoked_under=revoked_under, - ) - if not isinstance(served, str): - return served - rebuild_reason = served + answer_from_head = functools.partial( + _serve_judged_head, + subject_id=subject_id, + deps=deps, + coverage_snapshot=snapshot, + options=options, + trace_id=trace_id, + load_head=load_head, + revoked_under=revoked_under, + ) + rebuild_reason = answer_from_head(verdict) + if not isinstance(rebuild_reason, str): + return rebuild_reason rebuild_requests = requests() with deps.rebuilds.turn( subject_id, @@ -1384,20 +1415,38 @@ def _check_currency( current = deps.head_store.read_head(subject_id) if _rebuild_mark(current) != _rebuild_mark(head): return None - label("rebuild_reason", cast(str, rebuild_reason)) - served = _materialise_and_record( - stored, - requests=rebuild_requests, - options=options, - trigger=trigger, - deps=deps, - trace_id=trace_id, - expected_head=head, - coverage_snapshot=snapshot, - admitted=admitted, - progress=progress, - strip_pointer=strip_pointer, - ) + label("rebuild_reason", rebuild_reason) + try: + served = _materialise_and_record( + stored, + requests=rebuild_requests, + options=options, + trigger=trigger, + deps=deps, + trace_id=trace_id, + expected_head=head, + coverage_snapshot=snapshot, + admitted=admitted, + progress=progress, + strip_pointer=strip_pointer, + ) + except OperationBacklogFull: + # The refusal stands only on a head read that follows it. + with phase("head_read"): + current = deps.head_store.read_head(subject_id) + recheck = answer_from_head( + _judge_head( + current, + stored=stored, + requests=requests, + now=deps.now(), + deps=deps, + coverage=snapshot.index, + ) + ) + if isinstance(recheck, str): + raise + return recheck written = served.head if written is None: # A concurrent update won the head. Stamp its confirmation. @@ -1419,10 +1468,8 @@ def _admitted_ceiling( deps: DefinitionDeps, coverage_snapshot: CoverageSnapshot, admitted: Estimate | None, - rejected_head: SubjectHead | None, ) -> int: - """Admit a materialisation and return its raw-fetch guard (`serve_current`). `rejected_head` - is the head this rebuild replaces, for `submit_over_budget`.""" + """Admit a materialisation and return its raw-fetch guard (`serve_current`).""" if admitted is not None: return raw_fetch_guard(admitted, floor=deps.ceiling) estimate = sync_estimate( @@ -1457,7 +1504,6 @@ def _admitted_ceiling( async_max_rows=deps.settings.async_max_rows, async_max_requests=deps.settings.async_max_requests, members=[request.stat_var for request in requests], - rejected_head=rejected_head, ) return raw_fetch_guard(estimate, floor=deps.ceiling) @@ -1492,7 +1538,6 @@ def _materialise_and_record( deps=deps, coverage_snapshot=coverage_snapshot, admitted=admitted, - rejected_head=expected_head, ) with bound_variable_breadth(deps.variable_breadth, refuse_unread=True): return serve_definition( diff --git a/src/datasets_api/doors/refresh.py b/src/datasets_api/doors/refresh.py index 229e4aa9..6dcbb7bb 100644 --- a/src/datasets_api/doors/refresh.py +++ b/src/datasets_api/doors/refresh.py @@ -56,13 +56,11 @@ the head's stored frame and records it. When that cannot be done, or the head's own frame file has expired, the door rebuilds the dataset. -A dataset whose rebuild is always over the inline budget holds no head, because the worker -advances none. Each refresh of it probes and then reads the operation its request submits -(`operation_store.compute_operation_id`): done answers that operation's version. A done -operation answers only while a snapshot read still lists its rendering's file -(`snapshot.rendering_servable`, the check the operations poll makes). When the file has left -storage the door fails the row (`OperationStore.fail_unservable_done`) and submits again, which -re-runs the operation and answers `pending`. +A rebuild over the inline budget submits the request's operation +(`operations.submit.submit_over_budget`). The submission resets a done operation, so the door +answers `pending` for it, and the worker's run serves through `serve_current`, which advances +the head, so a later refresh answers from that head. An operation that failed and will not be +retried refuses non-retryable `internal_error` (500). The door reads the surface's `Deps`, so it shares the definition door's clients, materialise semaphore, and `RebuildFlights`. Refreshes run on the work executor beside mints and rebuilds, so @@ -94,7 +92,6 @@ _served_spec, request_definition_deps, ) -from datasets_api.doors.snapshot import rendering_servable from datasets_api.headers import no_store_headers from datasets_api.manifest_models import Manifest from datasets_api.mint_models import MintOptions @@ -106,7 +103,7 @@ records_plain_file, ) from datasets_api.refusals import classify_datasets_refusal -from datasets_api.render import artifact_rendering_keys, servable_frame_location +from datasets_api.render import servable_frame_location from datasets_api.timings import door_call, phase from dfo_platform import ProblemSpec from dfo_platform.doors import DoorContribution, route_registry_endpoints @@ -187,20 +184,6 @@ def __init__(self) -> None: super().__init__("the frame refresh door is at its concurrent capacity right now; retry") -class RefreshRebuildUnservable(Refusal): - """The queued rebuild is done and its file has left storage, again after this call - submitted it to run once more. Maps to retryable `service_unavailable` (503).""" - - code = "service_unavailable" - retryable = True - - def __init__(self, *, operation_id: str) -> None: - super().__init__( - f"the queued rebuild {operation_id!r} is done but its file is no longer in " - "storage; retry" - ) - - def classify_refresh_exception(exc: BaseException) -> ProblemSpec | None: return classify_datasets_refusal(exc) @@ -555,46 +538,24 @@ def _answer(current_content_version: str) -> FrameCurrent | FrameMoved: ) return FrameMoved(frame=frame) - # A second pass runs only after this call failed a done operation whose file is gone. - for _ in range(2): - try: - manifest = _serve_spec( - stored=served, - options=options, - trigger="refresh", - deps=deps, - trace_id=trace_id, - load_head=_load_head_frame, - ) - except OperationSubmitted as submitted: - operation = submitted.operation - else: - return _answer(manifest.content_version) - # The operation's options are the rendering's, so its version is the current frame. - if operation.state == "failed": - raise RefreshRebuildFailed( - operation_id=operation.id, refusal_code=operation.refusal_code - ) - if operation.state != "done" or operation.content_version is None: - return FramePending( - operation_id=operation.id, retry_after_s=OPERATION_SUBMIT_RETRY_AFTER_MS / 1000 - ) - if rendering_servable( - subject_id=spec_id, - content_version=operation.content_version, - rendering_keys=artifact_rendering_keys( - options, - site=deps.settings.site, - enrichment_digest=deps.code_index.enrichment_digest, - ), - snapshot_reader=deps.snapshot_reader, - now=deps.now, - settings=deps.settings, - ): - return _answer(operation.content_version) - # The file has left storage. Failing the row makes the next submit re-run it. - deps.operation_store.fail_unservable_done(operation.id, attempt=operation.attempt_count) - raise RefreshRebuildUnservable(operation_id=operation.id) + try: + manifest = _serve_spec( + stored=served, + options=options, + trigger="refresh", + deps=deps, + trace_id=trace_id, + load_head=_load_head_frame, + ) + except OperationSubmitted as submitted: + operation = submitted.operation + else: + return _answer(manifest.content_version) + if operation.state == "failed": + raise RefreshRebuildFailed(operation_id=operation.id, refusal_code=operation.refusal_code) + return FramePending( + operation_id=operation.id, retry_after_s=OPERATION_SUBMIT_RETRY_AFTER_MS / 1000 + ) def build_refresh_door( @@ -651,7 +612,6 @@ async def refresh_frame(body: RefreshFrameRequest) -> JSONResponse: FrameNotRefreshable, RenderingNotRecorded, RefreshRebuildFailed, - RefreshRebuildUnservable, RefreshBackpressure, RefreshNotPermitted, ), @@ -659,7 +619,6 @@ async def refresh_frame(body: RefreshFrameRequest) -> JSONResponse: FrameNotRefreshable(spec_id="ops_exemplar"), RenderingNotRecorded(spec_id="spec_exemplar", content_version="cv_exemplar"), RefreshRebuildFailed(operation_id="op_exemplar", refusal_code="over_ceiling"), - RefreshRebuildUnservable(operation_id="op_exemplar"), RefreshBackpressure(), RefreshNotPermitted(presented=False), ), @@ -675,7 +634,6 @@ async def refresh_frame(body: RefreshFrameRequest) -> JSONResponse: "RefreshBackpressure", "RefreshFrameRequest", "RefreshRebuildFailed", - "RefreshRebuildUnservable", "RenderingNotRecorded", "StoredRendering", "build_refresh_door", diff --git a/src/datasets_api/kinds/spec_id.py b/src/datasets_api/kinds/spec_id.py index f4315594..fdf706ab 100644 --- a/src/datasets_api/kinds/spec_id.py +++ b/src/datasets_api/kinds/spec_id.py @@ -65,6 +65,7 @@ from datasets_api.mint_deps import MintDeps from datasets_api.mint_models import MintOptions, Scope from datasets_api.models import CanonicalDefinition +from datasets_api.operation_store import OperationBacklogFull from datasets_api.operations.submit import submit_over_budget from datasets_api.render import linked_under from datasets_api.shape import StatVarShape @@ -429,7 +430,9 @@ def serve_mint( (`current.serve_current_head`). Otherwise admission runs once here. It refuses an Excel request over the row cap, submits work over the sync budget as an operation, and serves the current version (`current.serve_current`) under the admitted estimate, starting from - the head and verdict `serve_current_head` reached, so the head is read once. + the head and verdict `serve_current_head` reached, so the head is read once. A submission + the operation backlog refuses reads the head again and answers from it when it now links + the rendering, as it does when another submission's run completed in between. One coverage snapshot serves the head check, the estimate, the budget decision and the fetch. @@ -466,23 +469,38 @@ def serve_mint( datasets=elected_provenances(requests), ), ) - submit_over_budget( - subject_id=stored.spec_id, - options=options, - estimate=estimate, - coverage_stamp=coverage_snapshot.stamp, - operation_store=deps.operation_store, - task_client=deps.task_client, - within_budget=within_sync_budget( - estimate, - sync_budget_rows=deps.settings.sync_budget_rows, - sync_budget_requests=deps.settings.sync_budget_requests, - ), - async_max_rows=deps.settings.async_max_rows, - async_max_requests=deps.settings.async_max_requests, - members=[request.stat_var for request in requests], - rejected_head=served.head, - ) + try: + submit_over_budget( + subject_id=stored.spec_id, + options=options, + estimate=estimate, + coverage_stamp=coverage_snapshot.stamp, + operation_store=deps.operation_store, + task_client=deps.task_client, + within_budget=within_sync_budget( + estimate, + sync_budget_rows=deps.settings.sync_budget_rows, + sync_budget_requests=deps.settings.sync_budget_requests, + ), + async_max_rows=deps.settings.async_max_rows, + async_max_requests=deps.settings.async_max_requests, + members=[request.stat_var for request in requests], + ) + except OperationBacklogFull: + # The refusal stands only on a head read that follows it. + served = serve_current_head( + stored, + requests=requests, + options=options, + trigger="mint", + deps=definition_deps, + trace_id=trace_id, + coverage_snapshot=coverage_snapshot, + revoked_under=revoked_under, + ) + if isinstance(served, Manifest): + return served + raise return serve_current( stored, resolve_requests=lambda: requests, diff --git a/src/datasets_api/operation_store.py b/src/datasets_api/operation_store.py index 713b9eca..8ba6b03f 100644 --- a/src/datasets_api/operation_store.py +++ b/src/datasets_api/operation_store.py @@ -44,20 +44,18 @@ abandoned row, for the `datasets-maintenance age-operations` job. `create_or_get` returns an existing row unchanged unless it is resettable: a retryable failure, -a `running` row whose lease has ended, whichever delivery claimed it, a `pending` row past -the operation deadline, or a `done` row the submitting rebuild rejected. A submit reaches the -store only after `serve_current` has rejected the subject's head, so `rejected_version` names -that head's content version, or is `None` when the subject has no head. A `done` row at that -version, or any `done` row when there is no head, holds the result the rebuild rejected, so it -resets and the 202 reads `pending`. A `done` row at another version completed after the caller read the head and -stays. A reset of a row a later delivery is still due for moves it to a new -submission, and that delivery answers `superseded`. That preserves the original options and -estimate. A -reset returns the row to `pending`, clears the lease, `final_delivery`, `refusal_code`, -`refusal`, `retryable` and `dispatched_at`, sets `rows_written` to 0, and updates `submitted_at` while incrementing -`submission_count`, so its task gets a new name. Identity, options, admission coverage, and -catalogue build keep their admitted values. The attempt token carries on, so a superseded -attempt's late writes stay fenced. Deterministic failures stay unchanged. +a `running` row whose lease has ended, whichever delivery claimed it, a `pending` row past the +operation deadline, or any `done` row. A submission reaches the store only when the subject's +head could not answer it, and the rerun serves through `serve_current`, which applies the head +rule at run time. So a `done` row resets whatever version it holds, the 202 reads `pending`, and +the run serves what the current head gives for this rendering. A reset of a row a later delivery +is still due for moves it to a new submission, and that delivery answers `superseded`. That +preserves the original options and estimate. A reset returns the row to `pending`, clears the +lease, `final_delivery`, `refusal_code`, `refusal`, `retryable` and `dispatched_at`, sets +`rows_written` to 0, and updates `submitted_at` while incrementing `submission_count`, so its +task gets a new name. Identity, options, admission coverage, and catalogue build keep their +admitted values. The attempt token carries on, so a superseded attempt's late writes stay +fenced. Deterministic failures stay unchanged. A new row or a reset takes `pg_advisory_xact_lock`, re-reads the id under the lock, and counts live rows other than its own: every `pending` or `running` row that is not abandoned, which @@ -152,16 +150,12 @@ # such row past the deadline is abandoned. _CLAIMABLE = f"((state = 'pending' OR {_LEASE_ENDED}) AND {_QUEUED_SINCE} >= %s)" -# A done row at the parameter, the rejected head's content version, or any done row when it is -# NULL. -_DONE_REJECTED = "(state = 'done' AND content_version = COALESCE(%s, content_version))" - -# Rows `create_or_get` resets, at the parameters now, the untracked-lease cutoff, the deadline -# cutoff and the rejected head's content version: a retryable failure, a running row whose lease -# ended, a pending row past the deadline, or a done row the submitting rebuild rejected. +# Rows `create_or_get` resets, at the parameters now, the untracked-lease cutoff and the deadline +# cutoff: a retryable failure, a running row whose lease ended, a pending row past the deadline, +# or any done row. _RESETTABLE = ( f"((state = 'failed' AND retryable) OR {_LEASE_ENDED} OR {_PENDING_PAST_DEADLINE} " - f"OR {_DONE_REJECTED})" + "OR state = 'done')" ) # Rows the backlog counts, at `_ABANDONED`'s parameters: everything in flight except an @@ -318,14 +312,11 @@ def create_or_get( coverage_stamp: str, origin_catalogue_build_id: str, estimate: Estimate, - rejected_version: str | None, ) -> Operation: """Creates or reuses the operation keyed by identity, catalogue build, and rendering. A new row commits `pending` with no dispatch. Existing rows remain unchanged unless - resettable. `rejected_version` is the content version of the head the submitting - rebuild rejected, or `None` when the subject has no head; a `done` row at that version, - or any `done` row without a head, resets. Raises `OperationBacklogFull` when a new row + resettable; a `done` row always is. Raises `OperationBacklogFull` when a new row would exceed the configured backlog limit. """ ... @@ -415,7 +406,6 @@ def create_or_get( coverage_stamp: str, origin_catalogue_build_id: str, estimate: Estimate, - rejected_version: str | None, ) -> Operation: raise NotImplementedError( "NullOperationStore cannot create an operation it could never find again" @@ -550,7 +540,6 @@ def _resettable( now: datetime, *, deadline: timedelta, - rejected_version: str | None, ) -> bool: """`_RESETTABLE` on a row already read.""" operation = _operation_from_row(row) @@ -558,10 +547,7 @@ def _resettable( (operation.state == "failed" and bool(operation.retryable)) or operation.lease_ended(now) or operation.abandoned(now, deadline=deadline) - or ( - operation.state == "done" - and (rejected_version is None or operation.content_version == rejected_version) - ) + or operation.state == "done" ) @@ -613,7 +599,6 @@ def create_or_get( coverage_stamp: str, origin_catalogue_build_id: str, estimate: Estimate, - rejected_version: str | None, ) -> Operation: id = compute_operation_id( request_identity=request_identity, @@ -628,9 +613,7 @@ def _run() -> Mapping[str, object] | None: existing = tx.fetch_one( f"SELECT {_COLUMNS} FROM datasets.operation WHERE id = %s", (id,) ) - if existing is not None and not _resettable( - existing, now, deadline=self._deadline, rejected_version=rejected_version - ): + if existing is not None and not _resettable(existing, now, deadline=self._deadline): return existing # Lock before counting or inserting a new row or resetting one. tx.execute("SELECT pg_advisory_xact_lock(%s)", (_BACKLOG_LOCK_KEY,)) @@ -638,9 +621,7 @@ def _run() -> Mapping[str, object] | None: existing = tx.fetch_one( f"SELECT {_COLUMNS} FROM datasets.operation WHERE id = %s", (id,) ) - if existing is not None and not _resettable( - existing, now, deadline=self._deadline, rejected_version=rejected_version - ): + if existing is not None and not _resettable(existing, now, deadline=self._deadline): return existing # The row being reset keeps its own place, so only the others count. backlog_row = tx.fetch_one( @@ -687,7 +668,6 @@ def _run() -> Mapping[str, object] | None: now, now - _UNTRACKED_LEASE, now - self._deadline, - rejected_version, ), ) return tx.fetch_one( diff --git a/src/datasets_api/operations/submit.py b/src/datasets_api/operations/submit.py index e5c46320..b40e5435 100644 --- a/src/datasets_api/operations/submit.py +++ b/src/datasets_api/operations/submit.py @@ -29,7 +29,6 @@ from datasets_api.origin import catalogue_build_id from datasets_api.schema import OperationEnvelope, OperationResource from datasets_api.timings import phase -from datasets_api.versions import SubjectHead from series_coverage.index import Estimate # The shortest poll interval an operation advertises, and the refresh door's fixed retry interval. @@ -63,15 +62,12 @@ def submit_over_budget( async_max_rows: int, async_max_requests: int, members: Sequence[str], - rejected_head: SubjectHead | None, ) -> None: """Returns when the caller's sync-budget decision allows inline work. `members` are the StatVar refs a door resolved, `dcid:`-prefixed or bare. The operation - stores them as `member_dcids`. `rejected_head` is the head `serve_current` rejected before - this rebuild, or `None` when the subject has none. A done operation at its version, which - could serve nothing, restarts (`OperationStore.create_or_get`), so the 202 reads pending or - running. + stores them as `member_dcids`. A done operation restarts (`OperationStore.create_or_get`), + so the 202 reads pending or running, or failed for a deterministic failure. Otherwise, async row or request limits raise `OverCeilingError` before a row or task exists. `OperationBacklogFull` also propagates. The function records or reuses the operation, @@ -94,7 +90,6 @@ def submit_over_budget( coverage_stamp=coverage_stamp, origin_catalogue_build_id=catalogue_build_id(), estimate=estimate, - rejected_version=rejected_head.content_version if rejected_head is not None else None, ) with phase("dispatch"): operation = dispatch_if_due( @@ -139,10 +134,10 @@ def poll_interval_ms(operation: Operation, *, now: datetime) -> int: time since its current submission (`submitted_at`), held between `OPERATION_SUBMIT_RETRY_AFTER_MS` and `OPERATION_POLL_MAX_MS`. A repeat submission that joins a long build gets the slow value, a resubmission that restarts a failed build starts - at the floor, and a waiting browser backs off as the build runs. A done or failed - operation answers its next poll at once, so it gets the shortest interval. + at the floor, and a waiting browser backs off as the build runs. A failed operation + answers its next poll at once, so it gets the shortest interval. """ - if operation.state in ("done", "failed"): + if operation.state == "failed": return OPERATION_SUBMIT_RETRY_AFTER_MS age_ms = (now - operation.submitted_at) // timedelta(milliseconds=1) return min(OPERATION_POLL_MAX_MS, max(OPERATION_SUBMIT_RETRY_AFTER_MS, age_ms // 4)) @@ -160,7 +155,7 @@ def operation_submitted_resource( submitted: OperationSubmitted, *, now: datetime ) -> OperationResource: """Builds the 202 resource from the returned operation state, which may already be running or - terminal. Progress stays unset in the submit response and appears on polls. + failed. Progress stays unset in the submit response and appears on polls. """ operation = submitted.operation return OperationResource( diff --git a/src/search/engine/bind.py b/src/search/engine/bind.py index 29865f45..f336d84b 100644 --- a/src/search/engine/bind.py +++ b/src/search/engine/bind.py @@ -136,6 +136,14 @@ class SlotBindingDraft(BaseModel): # every dump. A ``sole_option`` draft of the reply claims that every member asserts # its value; one the engine wrote is bound by the rule that wrote it. drafted_by: SkipJsonSchema[Literal["reply", "engine"]] = Field(default="engine", exclude=True) + # Whether a ``sole_option`` claim of the reply rests on a recall sample that does not + # refute it (`phase_election._validate_sole_option_bindings`), so the walk of the + # family reads the slot open and confirms or refutes the claim + # (`phase_place_materialise._judge_sampled_claims`). False for a claim the whole family + # backs, for one the value an unexported family is told apart by backs + # (`bound_values.bind_shared_values`), and for every draft the validator did not judge. + # Left out of the schema and of every dump, as `drafted_by` is. + sampled: SkipJsonSchema[bool] = Field(default=False, exclude=True) @model_validator(mode="after") def _names_its_values(self) -> SlotBindingDraft: diff --git a/src/search/engine/entry/raw_text.py b/src/search/engine/entry/raw_text.py index 60c5920a..45bf1a05 100644 --- a/src/search/engine/entry/raw_text.py +++ b/src/search/engine/entry/raw_text.py @@ -9,6 +9,7 @@ import dataclasses import logging import re +from collections import Counter from collections.abc import AsyncGenerator, Collection, Sequence from contextlib import aclosing @@ -350,7 +351,11 @@ async def _recover_entity_surfaces( `query` (`_grounded`) is read for the qualifier. A name that settles on its surface's ambiguity takes the surface as its text (`surface_resolution.ambiguous_surface_names`), so "Congo" in "ODA to Congo", which the extraction names "Democratic Republic of the - Congo", asks its choice in the query's words. + Congo", asks its choice in the query's words. It does so only when no other name in + the query would carry the same text: the parsed door reads one text as one place per + part, so "aid from Congo to Congo", which the extraction names "Republic of the Congo" + and "Democratic Republic of the Congo", keeps both names and asks each end's choice on + its own. """ # Each entity reads the spans the ones before it recovered, so two never take one word. spanned = list(entities) @@ -396,9 +401,15 @@ async def _recover_entity_surfaces( recovered.append(entity) kept = _without_absorbed(recovered, query=query) surfaces = _entity_surfaces(kept, query=query) - as_written = await _await_bounded( + ambiguous = await _await_bounded( ambiguous_surface_names(surfaces, resolver=resolver), deadline=deadline ) + # The text each name would carry on the wire: its surface when renamed, else itself. + claims = Counter( + _normalize(surfaces[key] if key in ambiguous else key) + for key in dict.fromkeys(_entity_key(entity) for entity in kept) + ) + as_written = {key for key in ambiguous if claims[_normalize(surfaces[key])] == 1} return [ entity.model_copy(update={"text": surfaces[key], "span": surfaces[key]}) if (key := _entity_key(entity)) in as_written diff --git a/src/search/engine/region/phase_election.py b/src/search/engine/region/phase_election.py index a852afda..a384e5ae 100644 --- a/src/search/engine/region/phase_election.py +++ b/src/search/engine/region/phase_election.py @@ -99,6 +99,7 @@ shape_invariants, shape_query, sole_shared_value, + without_values, ) from search.engine.retrieval.member_facts import MemberFacts, asserted_constraint_properties from search.engine.retrieval.standard import ( @@ -289,7 +290,9 @@ def _settle_bindings( ``sole_option`` value (`_claim_single_member_slots`). A ``sole_option`` claim on a value the shape's population shows the slot a reading without leaves the slot unbound (`_validate_sole_option_bindings`), as does a value the - measure's headline series already means (`_unbind_implied_values`). One value drafted + measure's headline series already means (`_unbind_implied_values`). A claim a recall + sample alone backs stays bound, marked `SlotBindingDraft.sampled`, until the walk of the + family confirms or refutes it (`phase_place_materialise._judge_sampled_claims`). One value drafted on two properties no member holds together keeps the property a measure noun pins (`_keep_the_noun_pinned_property`). An operation over a category slot opens that slot, and so does a breakdown phrase naming it ("by @@ -415,6 +418,24 @@ def _settle_sole_options( ] +def sampled_claims(bindings: Sequence[SlotBindingDraft]) -> dict[str, str]: + """The value each bound ``sole_option`` claim of the reply in `bindings` that rests on a + recall sample (`SlotBindingDraft.sampled`) claims, by its property.""" + return { + b.property_dcid: b.value_dcids[0] + for b in bindings + if b.sampled and b.kind == "value" and b.property_dcid is not None + } + + +def without_sampled_claims(bindings: Sequence[SlotBindingDraft]) -> list[SlotBindingDraft]: + """`bindings` with the slot of each claim resting on a recall sample + (`sampled_claims`) left unbound: a sampled claim selects no member until the walk of + the family confirms it.""" + sampled = sampled_claims(bindings) + return without_values(bindings, sampled.keys()) if sampled else list(bindings) + + def _claim_single_member_slots( bindings: list[SlotBindingDraft], shape_draft: ShapeDraft ) -> list[SlotBindingDraft]: @@ -426,9 +447,11 @@ def _claim_single_member_slots( where the taxonomy holds only ``Years0``. The claim is the one the prompt asks of the reply, so it is stamped as the reply's (`SlotBindingDraft.drafted_by`) and every reader of the reply's claims judges it. A taxonomy read from a recall sample can show one value - the family's total does not hold (a population shape whose sample holds one gender), - which `_validate_sole_option_bindings` refutes. A ``requested_all`` draft asks for the - slot in full and stays.""" + the family's total does not hold (a population shape whose sample holds one gender). + `_validate_sole_option_bindings` refutes the claim where the sample shows it, and + otherwise marks it `SlotBindingDraft.sampled`, so the walk of the family reads the slot + open and refutes it on the total (`phase_place_materialise._judge_sampled_claims`). A + ``requested_all`` draft asks for the slot in full and stays.""" taxonomy = shape_draft.slot_taxonomy or {} claimed: list[SlotBindingDraft] = [] for draft in bindings: @@ -483,8 +506,11 @@ def _validate_sole_option_bindings( by-creditor member, which asserts only the flow type every member shares, is the total and asserts none of them. A member that asserts none but narrows on another property is no reading without the value: "infant mortality rate" binds ``age=Years0`` - beside members that state an age group classification instead. A shape with no - members passes every claim through. + beside members that state an age group classification instead. A claim the sample + does not refute is kept as `SlotBindingDraft.sampled`: a member outside the sample may + still lack the value, and the walk of the family, which reads the slot open + (`without_sampled_claims`), settles it (`phase_place_materialise._judge_sampled_claims`). + A shape with no members passes every claim through. """ population, index_sourced = five_tuple_member_facts(shape_draft) if not population: @@ -507,6 +533,8 @@ def _validate_sole_option_bindings( binding = binding.model_copy( update={"kind": "unbound", "value_dcids": [], "evidence": "mentioned"} ) + elif not complete: + binding = binding.model_copy(update={"sampled": True}) validated.append(binding) return validated diff --git a/src/search/engine/region/phase_place_materialise.py b/src/search/engine/region/phase_place_materialise.py index 4a15a467..b97a8472 100644 --- a/src/search/engine/region/phase_place_materialise.py +++ b/src/search/engine/region/phase_place_materialise.py @@ -81,6 +81,8 @@ _settle_served_dataset, narrowed_to_breakdown, record_coverage, + sampled_claims, + without_sampled_claims, ) from search.engine.region.phase_intake import not_admitted_entity from search.engine.region.phase_shared import ( @@ -143,6 +145,8 @@ narrowed_to, shape_population_arcs, shape_query, + sole_shared_value, + without_values, ) from search.engine.retrieval.facts import ( read_constraints, @@ -272,6 +276,12 @@ async def _phase_place_bindings(state: ResolveState) -> RegionResult | None: the rule the read applies (`unrealisable.unbind_unrealisable`), and the family is settled again, so the place context is built once, on the family of the bindings the answer states. `membership_unverified` then says only that a family could not be read. + + The walk reads each slot a recall sample alone claimed open + (`phase_election.without_sampled_claims`), so the family it settles holds the members + the claim would exclude, and the claim is confirmed or refuted on that family + (`_judge_sampled_claims`). A family no walk settles leaves the claim standing on the + sample, under `membership_unverified`. """ _note_named_places(state) @@ -295,7 +305,10 @@ async def _phase_place_bindings(state: ResolveState) -> RegionResult | None: # member graph to walk. Members it does not settle still answer, under a warning. role_keys = bound.role_keys role_query = shape_query( - state.shape_draft, state.bindings, role_keys.bound, open_role_keys=role_keys.open + state.shape_draft, + without_sampled_claims(state.bindings), + role_keys.bound, + open_role_keys=role_keys.open, ) recalled = state.shape_draft try: @@ -347,7 +360,10 @@ async def _phase_place_bindings(state: ResolveState) -> RegionResult | None: recalled = state.shape_draft relaxed = True role_query = shape_query( - recalled, state.bindings, role_keys.bound, open_role_keys=role_keys.open + recalled, + without_sampled_claims(state.bindings), + role_keys.bound, + open_role_keys=role_keys.open, ) try: resolution = await asyncio.to_thread( @@ -361,9 +377,13 @@ async def _phase_place_bindings(state: ResolveState) -> RegionResult | None: except (RoleSignatureError, FamilyNotResolvableError) as exc: return _election_refusal(state, exc) state.shape_draft = resolution.value + state.walked = resolution.walked + if refuted := _judge_sampled_claims(state, role_keys): + if (refused := await _unbind_refuted_claims(state, refuted, role_keys)) is not None: + return refused + relaxed = True if not resolution.authority.settled: state.warnings.append(membership_unverified_warning()) - state.walked = resolution.walked # The place context below is built for the shape the election returned. bound = bound_after_election(state, recalled, bound, state.shape_draft) if bound == "role_incompatible": @@ -1575,6 +1595,65 @@ async def _unbind( (`phase_election.record_coverage`), as binding repair reads them: "female" in "female aged X" reads no bound code once Sex=Female is unbound.""" unbind_unrealisable(state, unrealisable) + return await _resettle_unbound_part(state, role_keys) + + +def _judge_sampled_claims(state: ResolveState, role_keys: RoleKeys) -> frozenset[str]: + """The properties of the claims a recall sample alone backs + (`phase_election.sampled_claims`) that the family the walk settled refutes, once each + claim it confirms is no longer marked sampled. + + The walk read each claimed slot open (`phase_election.without_sampled_claims`), and its + family (`ShapeDraft.sv_arc_facts` of the shape it settled) confirms a claim when every + member asserts the claimed value (`sole_shared_value`), the rule a claim on a complete + population meets at settle. A confirmed claim keys the walk (`ResolveState.walked`) to + the query binding its value (`WalkedMembers.binding`): every walked member asserts it, + so the walk of that query settles the same members, and the identity minted under the + bindings reads them (`stated_members`). With no family settled (`ResolveState.walked` + is None), every claim stands on its sample.""" + claims = sampled_claims(state.bindings) + if not claims or state.walked is None: + return frozenset() + constraints = [ + read_constraints(arcs) for arcs in (state.shape_draft.sv_arc_facts or {}).values() + ] + confirmed = { + prop for prop, value in claims.items() if sole_shared_value(constraints, prop) == value + } + refuted = frozenset(claims.keys() - confirmed) + if not confirmed: + return refuted + state.bindings = [ + b.model_copy(update={"sampled": False}) if b.property_dcid in confirmed else b + for b in state.bindings + ] + bound_query = shape_query( + state.shape_draft, + without_values(state.bindings, refuted), + role_keys.bound, + open_role_keys=role_keys.open, + ) + for prop in confirmed: + state.walked = state.walked.binding(prop, bound_query.slots[prop]) + return refuted + + +async def _unbind_refuted_claims( + state: ResolveState, refuted: frozenset[str], role_keys: RoleKeys +) -> RegionResult | None: + """Unbind each slot of `refuted`, claims a recall sample backed and the walked family + refutes (`_judge_sampled_claims`), and settle the part again as an unbinding does + (`_resettle_unbound_part`). The request's words never named the value, so the answer + records no requirement and names no departure for it.""" + state.bindings = without_values(state.bindings, refuted) + state.forget_bound(refuted) + return await _resettle_unbound_part(state, role_keys) + + +async def _resettle_unbound_part(state: ResolveState, role_keys: RoleKeys) -> RegionResult | None: + """Settle the dataset the part is served from under the bindings an unbinding leaves, + returning the refusal of a part no table then serves, and rate and record the words the + coverage rule reads on them (`phase_election.record_coverage`).""" if (refused := _settle_served_dataset(state, role_keys)) is not None: return refused await record_coverage(state) diff --git a/src/search/engine/retrieval/bound_values.py b/src/search/engine/retrieval/bound_values.py index 8da1a97c..75c67478 100644 --- a/src/search/engine/retrieval/bound_values.py +++ b/src/search/engine/retrieval/bound_values.py @@ -2245,6 +2245,21 @@ def settles(self, query: FamilyQuery) -> bool: asked (`asked`).""" return walk_key(query) in (self.key, self.asked) + def binding(self, prop: str, slot: SlotBinding) -> WalkedMembers: + """These members keyed with `prop` bound by `slot`, for a walk that read `prop` + open and whose every member asserts the value `slot` binds. The walk of the query + binding it admits exactly those members, so it settles the same members and + family.""" + + def bound(key: WalkKey) -> WalkKey: + five_tuple, slots, role_keys, open_role_keys = key + rebound = {**dict(slots), prop: _walked_binding(slot)} + return five_tuple, tuple(sorted(rebound.items())), role_keys, open_role_keys + + return replace( + self, key=bound(self.key), asked=None if self.asked is None else bound(self.asked) + ) + def stated_members(identity: BaseIdentity, walked: WalkedMembers | None) -> StatedMembers | None: """What a Datasets mint of `identity`, a described identity, states of its members, when @@ -3496,7 +3511,9 @@ def _unexported_shared_values( shape: ShapeDraft, bindings: Sequence[SlotBindingDraft] ) -> dict[str, str]: """Each open slot the exported family shares, mapped to the one value every member of - `shape`'s unexported family holds on it (`bind_shared_values`).""" + `shape`'s unexported family holds on it (`bind_shared_values`). A slot a sampled claim + binds (`SlotBindingDraft.sampled`) counts as open, since the claim selects no member + until something beside its sample backs it.""" members = shape.sv_arc_facts index = get_index() five_tuple = shape_five_tuple(shape) @@ -3514,7 +3531,8 @@ def _unexported_shared_values( closed = { binding.property_dcid for binding in bindings - if binding.kind in ("value", "set") or binding.evidence == "requested_all" + if (binding.kind in ("value", "set") and not binding.sampled) + or binding.evidence == "requested_all" } shared: dict[str, str] = {} for slot in shape.slot_keys: @@ -3550,9 +3568,30 @@ def bind_shared_values( value there that the family's total does not hold. A slot the query asked to see in full (`requested_all`) stays open. + A ``sole_option`` claim of the reply on such a slot that rests on the recalled sample + (`SlotBindingDraft.sampled`) and claims the shared value is backed by this rule, which + reads the value as what tells the family apart, so it is no longer marked sampled and + the walk reads the slot bound, as it reads a value this rule binds. Read open, the walk + of "debt service" lists the interest payments total of the same five-tuple and refutes + the claim (`phase_place_materialise._judge_sampled_claims`), and the answer serves + every flow. A sampled claim of another value stays as it is, for the walk to judge. + Returns the bindings with each such slot bound as a `sole_option` value, and the bindings added. Both follow slot-key order.""" - return _bind_sole_options(shape, bindings, _unexported_shared_values(shape, bindings)) + shared = _unexported_shared_values(shape, bindings) + sampled = { + b.property_dcid: b.value_dcids + for b in bindings + if b.sampled and b.property_dcid is not None + } + backed = {prop for prop, value in shared.items() if sampled.get(prop) == [value]} + if backed: + bindings = [ + b.model_copy(update={"sampled": False}) if b.property_dcid in backed else b + for b in bindings + ] + unclaimed = {prop: value for prop, value in shared.items() if prop not in sampled} + return _bind_sole_options(shape, bindings, unclaimed) def bind_member_values( diff --git a/tests/dataset_store/test_live.py b/tests/dataset_store/test_live.py index 4c805de6..069a76ed 100644 --- a/tests/dataset_store/test_live.py +++ b/tests/dataset_store/test_live.py @@ -1075,7 +1075,6 @@ def test_operation_store_create_or_get_reset_survives_a_concurrent_resubmit_race coverage_stamp=stamp, origin_catalogue_build_id="fake-catalogue", estimate=estimate, - rejected_version=None, ) target = setup_store.create_or_get( request_identity=identity, @@ -1083,7 +1082,6 @@ def test_operation_store_create_or_get_reset_survives_a_concurrent_resubmit_race coverage_stamp=stamp, origin_catalogue_build_id="fake-catalogue", estimate=estimate, - rejected_version=None, ) setup_store.claim(target.id, submission_count=0, lease_seconds=60, final_delivery=False) setup_store.fail(target.id, attempt=1, refusal_code="internal_error", retryable=True) @@ -1112,7 +1110,6 @@ def _loser_call() -> None: coverage_stamp=stamp, origin_catalogue_build_id="fake-catalogue", estimate=estimate, - rejected_version=None, ) # surfaced on the main thread below except BaseException as exc: @@ -1135,7 +1132,6 @@ def _loser_call() -> None: coverage_stamp=stamp, origin_catalogue_build_id="fake-catalogue", estimate=estimate, - rejected_version=None, ) finally: winner_pool.close_all() @@ -1295,7 +1291,6 @@ def _new_operation(store: LiveOperationStore, run_id: str, label: str): coverage_stamp=f"cov-test-live-fence-{run_id}", origin_catalogue_build_id="fake-catalogue", estimate=Estimate(rows=1, requests=1), - rejected_version=None, ) @@ -1386,27 +1381,16 @@ def test_operation_store_complete_keeps_the_runs_confirmed_instant(live_store): assert stored is not None and stored.confirmed_at == confirmed_at -@pytest.mark.parametrize( - ("rejected_version", "state"), - [ - pytest.param("cv1", "pending", id="rejected-head-is-the-done-version"), - pytest.param(None, "pending", id="no-head"), - pytest.param("cv0", "done", id="done-after-the-rejected-head"), - ], -) -def test_operation_store_create_or_get_resets_a_done_row_the_rebuild_rejected( - live_store, rejected_version: str | None, state: str -): - """The reset `UPDATE` binds the rejected head's version on PostgreSQL: a done row at that - version, or any done row with no head, returns to pending, and one at another version - stays.""" +def test_operation_store_create_or_get_resets_a_done_row(live_store): + """The reset `UPDATE`'s `_RESETTABLE` admits a done row on PostgreSQL, so it returns to + pending under a new submission and its next claim takes a new attempt.""" run_id = uuid4().hex t0 = datetime(2026, 1, 1, tzinfo=UTC) operations = LiveOperationStore(store=live_store, now=lambda: t0, backlog_max=_RACE_BACKLOG_MAX) - op = _new_operation(operations, run_id, "rejected") + op = _new_operation(operations, run_id, "done-reset") claimed = operations.claim(op.id, submission_count=0, lease_seconds=30, final_delivery=False) assert claimed is not None - subject_id = f"spec-test-live-rejected-{run_id}" + subject_id = f"spec-test-live-done-reset-{run_id}" _insert_version_record(live_store, subject_id=subject_id, content_version="cv1") assert operations.complete( op.id, @@ -1422,11 +1406,16 @@ def test_operation_store_create_or_get_resets_a_done_row_the_rebuild_rejected( coverage_stamp=op.coverage_stamp, origin_catalogue_build_id="fake-catalogue", estimate=op.estimate, - rejected_version=rejected_version, ) - assert (resubmitted.id, resubmitted.state) == (op.id, state) - assert resubmitted.submission_count == (0 if state == "done" else 1) + assert (resubmitted.id, resubmitted.state, resubmitted.submission_count) == ( + op.id, + "pending", + 1, + ) + reclaimed = operations.claim(op.id, submission_count=1, lease_seconds=30, final_delivery=False) + assert reclaimed is not None + assert reclaimed.attempt_count == claimed.attempt_count + 1 def _materialisation_count(live_store, subject_id: str) -> int: @@ -1822,7 +1811,6 @@ def _subject_operation(store: LiveOperationStore, run_id: str): coverage_stamp="cov-test-live-admission", origin_catalogue_build_id="fake-catalogue", estimate=Estimate(rows=200_000, requests=5), - rejected_version=None, ) diff --git a/tests/datasets_api/refusal_wire_golden.json b/tests/datasets_api/refusal_wire_golden.json index c4b2a9c3..01b7891e 100644 --- a/tests/datasets_api/refusal_wire_golden.json +++ b/tests/datasets_api/refusal_wire_golden.json @@ -375,7 +375,7 @@ "status": 410, "truncated": false }, - "extra-134-FacetNotAvailableError": { + "extra-133-FacetNotAvailableError": { "alternatives": null, "cap": null, "cause": null, @@ -391,7 +391,7 @@ "status": 422, "truncated": false }, - "extra-135-FacetNotAvailableError": { + "extra-134-FacetNotAvailableError": { "alternatives": null, "cap": null, "cause": null, @@ -407,7 +407,7 @@ "status": 422, "truncated": false }, - "extra-136-FacetNotAvailableError": { + "extra-135-FacetNotAvailableError": { "alternatives": null, "cap": null, "cause": null, @@ -433,7 +433,7 @@ "status": 422, "truncated": false }, - "extra-137-DefaultFacetUndefinedError": { + "extra-136-DefaultFacetUndefinedError": { "alternatives": null, "cap": null, "cause": null, @@ -449,7 +449,7 @@ "status": 422, "truncated": false }, - "extra-138-DefaultFacetUndefinedError": { + "extra-137-DefaultFacetUndefinedError": { "alternatives": null, "cap": null, "cause": null, @@ -475,7 +475,7 @@ "status": 422, "truncated": false }, - "extra-139-DefaultFacetUndefinedError": { + "extra-138-DefaultFacetUndefinedError": { "alternatives": null, "cap": null, "cause": null, @@ -501,7 +501,7 @@ "status": 422, "truncated": false }, - "extra-140-DefaultFacetUndefinedError": { + "extra-139-DefaultFacetUndefinedError": { "alternatives": null, "cap": null, "cause": null, @@ -517,7 +517,7 @@ "status": 422, "truncated": false }, - "extra-141-RecipeNotDeclaredError": { + "extra-140-RecipeNotDeclaredError": { "alternatives": null, "cap": null, "cause": { @@ -537,7 +537,7 @@ "status": 422, "truncated": false }, - "extra-142-NonCurrencyUnitError": { + "extra-141-NonCurrencyUnitError": { "alternatives": null, "cap": null, "cause": null, @@ -553,8 +553,8 @@ "status": 422, "truncated": false }, - "extra-143-MixedGrainError": null, - "extra-144-MemberNotResolvable": { + "extra-142-MixedGrainError": null, + "extra-143-MemberNotResolvable": { "alternatives": null, "cap": null, "cause": null, @@ -589,7 +589,7 @@ "status": 422, "truncated": true }, - "extra-145-ColumnUnknownError": { + "extra-144-ColumnUnknownError": { "alternatives": [ { "label": "place.name", @@ -610,7 +610,7 @@ "status": 422, "truncated": false }, - "extra-146-PivotOverColumnCapError": { + "extra-145-PivotOverColumnCapError": { "alternatives": null, "cap": null, "cause": null, @@ -626,7 +626,7 @@ "status": 422, "truncated": false }, - "extra-147-PivotNotUniqueError": { + "extra-146-PivotNotUniqueError": { "alternatives": null, "cap": null, "cause": null, @@ -661,7 +661,7 @@ "status": 422, "truncated": true }, - "extra-148-OverCeilingError": { + "extra-147-OverCeilingError": { "alternatives": null, "cap": null, "cause": null, @@ -677,7 +677,7 @@ "status": 422, "truncated": false }, - "extra-149-DatasetsStoreUnavailable": { + "extra-148-DatasetsStoreUnavailable": { "alternatives": null, "cap": null, "cause": null, @@ -693,7 +693,7 @@ "status": 500, "truncated": false }, - "extra-150-SnapshotSigningFailed": { + "extra-149-SnapshotSigningFailed": { "alternatives": null, "cap": null, "cause": null, @@ -709,7 +709,7 @@ "status": 500, "truncated": false }, - "extra-151-ArtifactInventoryError": { + "extra-150-ArtifactInventoryError": { "alternatives": null, "cap": null, "cause": null, @@ -725,7 +725,7 @@ "status": 503, "truncated": false }, - "extra-152-RefreshNotPermitted": { + "extra-151-RefreshNotPermitted": { "alternatives": null, "cap": null, "cause": null, @@ -773,7 +773,7 @@ "status": 503, "truncated": false }, - "meta-132-OperationNotFound": { + "meta-131-OperationNotFound": { "alternatives": null, "cap": null, "cause": null, @@ -789,7 +789,7 @@ "status": 404, "truncated": false }, - "meta-133-OperationFailed": { + "meta-132-OperationFailed": { "alternatives": null, "cap": null, "cause": { @@ -2456,23 +2456,7 @@ "status": 500, "truncated": false }, - "refresh-129-RefreshRebuildUnservable": { - "alternatives": null, - "cap": null, - "cause": null, - "ceiling_kind": null, - "code": "service_unavailable", - "constraint_id": null, - "count": null, - "detail": "the queued rebuild 'op_exemplar' is done but its file is no longer in storage; retry", - "headers": null, - "issues": null, - "pointer": null, - "retryable": true, - "status": 503, - "truncated": false - }, - "refresh-130-RefreshBackpressure": { + "refresh-129-RefreshBackpressure": { "alternatives": null, "cap": null, "cause": null, @@ -2488,7 +2472,7 @@ "status": 503, "truncated": false }, - "refresh-131-RefreshNotPermitted": { + "refresh-130-RefreshNotPermitted": { "alternatives": null, "cap": null, "cause": null, diff --git a/tests/datasets_api/test_meta_doors.py b/tests/datasets_api/test_meta_doors.py index 6e498c44..237e3a6b 100644 --- a/tests/datasets_api/test_meta_doors.py +++ b/tests/datasets_api/test_meta_doors.py @@ -371,7 +371,6 @@ def test_a_resubmission_is_paced_from_its_current_submission( "coverage_stamp": "cov_stamp_1", "origin_catalogue_build_id": "fake-catalogue", "estimate": _DEFAULT_ESTIMATE, - "rejected_version": None, } created = store.create_or_get(**submission) assert store.claim( @@ -390,9 +389,8 @@ def test_a_resubmission_is_paced_from_its_current_submission( assert resource.operation.retry_after_ms == expected_ms -@pytest.mark.parametrize("state", ["done", "failed"]) -def test_a_finished_operation_is_paced_at_the_floor_whatever_its_age(state: str) -> None: - operation = _operation(state=state) +def test_a_failed_operation_is_paced_at_the_floor_whatever_its_age() -> None: + operation = _operation(state="failed") interval = poll_interval_ms(operation, now=_OP_NOW + timedelta(hours=1)) @@ -579,7 +577,6 @@ def test_failed_operation_poll_with_an_unparseable_stored_refusal_names_the_code coverage_stamp="cov_stamp_1", origin_catalogue_build_id="fake-catalogue", estimate=_DEFAULT_ESTIMATE, - rejected_version=None, ) operation_store.claim(op.id, submission_count=0, lease_seconds=1_500, final_delivery=False) stored = _STORED_REFUSALS["over_ceiling"] @@ -892,7 +889,6 @@ def test_a_done_operation_whose_rendering_is_gone_fails_retryable_and_a_resubmit "coverage_stamp": "cov_stamp_1", "origin_catalogue_build_id": "fake-catalogue", "estimate": _DEFAULT_ESTIMATE, - "rejected_version": None, } op = operation_store.create_or_get(**submission) assert ( @@ -1014,7 +1010,6 @@ def _create(operation_store: LiveOperationStore) -> Operation: coverage_stamp="cov_stamp_1", origin_catalogue_build_id="fake-catalogue", estimate=Estimate(rows=1_000_000, requests=40), - rejected_version=None, ) @@ -1378,7 +1373,6 @@ def test_the_composed_poll_route_reconciles_through_the_surfaces_task_client() - coverage_stamp="cov_stamp_1", origin_catalogue_build_id="fake-catalogue", estimate=Estimate(rows=1_000_000, requests=40), - rejected_version=None, ) task_client = FakeTaskClient() client = make_client( diff --git a/tests/datasets_api/test_operation_fencing.py b/tests/datasets_api/test_operation_fencing.py index b417a9f4..f2f66146 100644 --- a/tests/datasets_api/test_operation_fencing.py +++ b/tests/datasets_api/test_operation_fencing.py @@ -171,7 +171,6 @@ def test_a_crash_between_commit_and_enqueue_is_recovered_by_the_next_poll() -> N async_max_rows=_SETTINGS.async_max_rows, async_max_requests=_SETTINGS.async_max_requests, members=["dcid:sv_a"], - rejected_head=None, ) operation_id = excinfo.value.operation.id crashed = store.get(operation_id) diff --git a/tests/datasets_api/test_operation_store.py b/tests/datasets_api/test_operation_store.py index 7accf16e..55ec6743 100644 --- a/tests/datasets_api/test_operation_store.py +++ b/tests/datasets_api/test_operation_store.py @@ -95,18 +95,17 @@ def test_compute_operation_id_is_deterministic() -> None: assert id_a.startswith("op_") -def test_a_coverage_build_leaves_the_operation_and_its_completion_in_place( +def test_a_coverage_build_leaves_the_operation_and_its_admitted_stamp_in_place( store: LiveOperationStore, ) -> None: """The coverage stamp stays out of the id, so a resubmission after a coverage build finds - the done operation. Its rebuild rejected an older head, so the completion stays.""" + the done operation and resets it under the stamp it was admitted with.""" first = store.create_or_get( request_identity=_IDENTITY, options=_OPTIONS, coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert store.claim(first.id, submission_count=0, lease_seconds=60, final_delivery=False) assert store.complete( @@ -123,12 +122,11 @@ def test_a_coverage_build_leaves_the_operation_and_its_completion_in_place( coverage_stamp="cov_stamp_2", origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version="cv_0", ) assert (after_build.id, after_build.state, after_build.coverage_stamp) == ( first.id, - "done", + "pending", _STAMP, ) @@ -172,7 +170,6 @@ def test_a_completed_operation_is_not_reused_under_a_new_catalogue_build( coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) claimed = store.claim(first.id, submission_count=0, lease_seconds=60, final_delivery=False) assert claimed is not None @@ -190,7 +187,6 @@ def test_a_completed_operation_is_not_reused_under_a_new_catalogue_build( coverage_stamp=_STAMP, origin_catalogue_build_id="20261006T000000Z-ef567890", estimate=_ESTIMATE, - rejected_version=None, ) assert rebuilt.id != first.id @@ -208,7 +204,6 @@ def test_create_or_get_is_idempotent_on_repeat_submit_of_the_same_rendering( coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert first.state == "pending" assert first.rows_written == 0 @@ -221,7 +216,6 @@ def test_create_or_get_is_idempotent_on_repeat_submit_of_the_same_rendering( coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=Estimate(rows=1, requests=1), - rejected_version=None, ) assert second.id == first.id assert second.options == _OPTIONS @@ -237,7 +231,6 @@ def test_create_or_get_starts_a_different_operation_for_a_different_rendering( coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) csv = store.create_or_get( request_identity=_IDENTITY, @@ -245,7 +238,6 @@ def test_create_or_get_starts_a_different_operation_for_a_different_rendering( coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert parquet.id != csv.id @@ -265,7 +257,6 @@ def test_create_or_get_refuses_a_new_operation_at_the_backlog_max() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.create_or_get( request_identity={"input": {"spec_id": "spec_b"}}, @@ -273,7 +264,6 @@ def test_create_or_get_refuses_a_new_operation_at_the_backlog_max() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) with pytest.raises(OperationBacklogFull) as excinfo: @@ -283,7 +273,6 @@ def test_create_or_get_refuses_a_new_operation_at_the_backlog_max() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert excinfo.value.in_flight == 2 assert excinfo.value.limit == 2 @@ -311,7 +300,6 @@ def test_create_or_get_at_the_backlog_max_still_returns_an_existing_operation() coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) second = store.create_or_get( @@ -320,7 +308,6 @@ def test_create_or_get_at_the_backlog_max_still_returns_an_existing_operation() coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert second.id == first.id @@ -340,7 +327,6 @@ def test_create_or_get_backlog_count_excludes_terminal_operations() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(first.id, submission_count=0, lease_seconds=60, final_delivery=False) store.complete( @@ -357,7 +343,6 @@ def test_create_or_get_backlog_count_excludes_terminal_operations() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert second.state == "pending" finally: @@ -390,7 +375,6 @@ def test_create_or_get_backlog_count_excludes_a_run_nothing_will_finish( coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert store.claim( first.id, submission_count=0, lease_seconds=60, final_delivery=final_delivery @@ -404,7 +388,6 @@ def _second(): coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) if admitted: @@ -423,7 +406,6 @@ def _create(store: LiveOperationStore, spec_id: str): coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) @@ -538,7 +520,6 @@ def test_create_or_get_takes_the_advisory_lock_before_counting_on_a_new_row() -> coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) statements = [s.strip().upper() for s in recording.executed] lock_index = next( @@ -561,7 +542,6 @@ def test_create_or_get_skips_the_advisory_lock_for_an_existing_operation() -> No coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) recording.executed.clear() @@ -571,7 +551,6 @@ def test_create_or_get_skips_the_advisory_lock_for_an_existing_operation() -> No coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert not any( @@ -588,7 +567,6 @@ def test_claim_moves_pending_to_running(store: LiveOperationStore) -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) claimed = store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) @@ -609,7 +587,6 @@ def test_claim_refuses_a_live_lease(store: LiveOperationStore) -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=3600, final_delivery=False) @@ -629,7 +606,6 @@ def test_claim_takes_over_a_lease_only_once_it_has_ended(elapsed: int, attempt: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) live_store.claim(op.id, submission_count=0, lease_seconds=30, final_delivery=False) @@ -646,7 +622,6 @@ def test_claim_refuses_a_terminal_operation(store: LiveOperationStore) -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) store.complete( @@ -667,7 +642,6 @@ def test_progress_only_applies_while_running(store: LiveOperationStore) -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert store.progress(op.id, attempt=1, rows_written=100) is None @@ -688,7 +662,6 @@ def test_complete_sets_the_manifest_reference_and_is_running_only( coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert ( @@ -736,7 +709,6 @@ def test_fail_sets_the_refusal_code_and_is_running_only(store: LiveOperationStor coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert store.fail(op.id, attempt=1, refusal_code="excel_limit", retryable=False) is None @@ -774,7 +746,6 @@ def test_fail_stores_the_classified_refusal_beside_its_code(store: LiveOperation coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert op.refusal is None store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) @@ -809,7 +780,6 @@ def test_an_unparseable_stored_refusal_reads_as_none_and_keeps_the_code( coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) store.fail( @@ -841,7 +811,6 @@ def test_fail_without_a_refusal_leaves_the_column_null(store: LiveOperationStore coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) @@ -858,7 +827,6 @@ def test_a_retryable_reset_clears_a_stored_refusal(store: LiveOperationStore) -> coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) store.fail(op.id, attempt=1, refusal_code="upstream_error", retryable=True, refusal=_REFUSAL) @@ -869,7 +837,6 @@ def test_a_retryable_reset_clears_a_stored_refusal(store: LiveOperationStore) -> coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert resubmitted.state == "pending" @@ -884,7 +851,6 @@ def test_fail_persists_retryable_true_for_a_transient_failure(store: LiveOperati coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) @@ -903,7 +869,6 @@ def test_release_moves_running_back_to_pending_and_clears_the_claim( coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert store.release(op.id, attempt=1) is None @@ -937,7 +902,6 @@ def test_release_is_a_no_op_once_terminal(store: LiveOperationStore) -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) store.complete( @@ -963,7 +927,6 @@ def test_fail_unservable_done_fails_only_a_done_row_at_the_polled_attempt( "coverage_stamp": _STAMP, "origin_catalogue_build_id": _BUILD, "estimate": _ESTIMATE, - "rejected_version": None, } op = store.create_or_get(**submission) assert store.fail_unservable_done(op.id, attempt=0) is None @@ -1015,7 +978,6 @@ def _reclaimed(now_box: dict[str, datetime], store: LiveOperationStore) -> str: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) first = store.claim(op.id, submission_count=0, lease_seconds=30, final_delivery=False) assert first is not None and first.attempt_count == 1 @@ -1111,7 +1073,6 @@ def test_claim_refuses_a_delivery_from_an_earlier_submission( coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) store.fail(op.id, attempt=1, refusal_code="internal_error", retryable=True) @@ -1121,7 +1082,6 @@ def test_claim_refuses_a_delivery_from_an_earlier_submission( coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert reset.submission_count == 1 @@ -1145,7 +1105,6 @@ def test_get_returns_a_terminal_operation(store: LiveOperationStore) -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) store.fail(op.id, attempt=1, refusal_code="excel_limit", retryable=False) @@ -1166,7 +1125,6 @@ def test_create_or_get_returns_a_deterministic_failed_row_unchanged() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) store.fail(op.id, attempt=1, refusal_code="excel_limit", retryable=False) @@ -1177,7 +1135,6 @@ def test_create_or_get_returns_a_deterministic_failed_row_unchanged() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert resubmitted.id == op.id @@ -1200,7 +1157,6 @@ def test_create_or_get_resets_a_retryable_failed_row_to_pending() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) store.fail(op.id, attempt=1, refusal_code="internal_error", retryable=True) @@ -1211,7 +1167,6 @@ def test_create_or_get_resets_a_retryable_failed_row_to_pending() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert resubmitted.id == op.id @@ -1232,34 +1187,21 @@ def test_create_or_get_resets_a_retryable_failed_row_to_pending() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert twice_resubmitted.submission_count == 2 finally: fake.close() -@pytest.mark.parametrize( - ("rejected_version", "state"), - [ - pytest.param("cv_1", "pending", id="rejected-head-is-the-done-version"), - pytest.param(None, "pending", id="no-head"), - pytest.param("cv_0", "done", id="done-after-the-rejected-head"), - ], -) -def test_create_or_get_resets_a_done_row_the_rebuild_rejected( - store: LiveOperationStore, rejected_version: str | None, state: str -) -> None: - """A rebuild that rejected the head at the done row's version, or found no head, restarts - the operation, so its 202 has something to wait for. A done row at another version completed - after the caller read the head and stays.""" +def test_create_or_get_resets_a_done_row(store: LiveOperationStore) -> None: + """A submission reaches the store only when the head could not answer it, so a done row + resets whatever version it holds and its next claim takes a new attempt.""" op = store.create_or_get( request_identity=_IDENTITY, options=_OPTIONS, coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) assert store.complete( @@ -1276,15 +1218,16 @@ def test_create_or_get_resets_a_done_row_the_rebuild_rejected( coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=rejected_version, ) - assert (resubmitted.id, resubmitted.state) == (op.id, state) - assert resubmitted.submission_count == (0 if state == "done" else 1) - if state == "pending": - claimed = store.claim(op.id, submission_count=1, lease_seconds=60, final_delivery=False) - assert claimed is not None - assert claimed.attempt_count == 2 + assert (resubmitted.id, resubmitted.state, resubmitted.submission_count) == ( + op.id, + "pending", + 1, + ) + claimed = store.claim(op.id, submission_count=1, lease_seconds=60, final_delivery=False) + assert claimed is not None + assert claimed.attempt_count == 2 def test_create_or_get_reset_of_a_retryable_failed_row_takes_the_backlog_lock() -> None: @@ -1298,7 +1241,6 @@ def test_create_or_get_reset_of_a_retryable_failed_row_takes_the_backlog_lock() coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) store.fail(op.id, attempt=1, refusal_code="internal_error", retryable=True) @@ -1310,7 +1252,6 @@ def test_create_or_get_reset_of_a_retryable_failed_row_takes_the_backlog_lock() coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) statements = [s.strip().upper() for s in recording.executed] @@ -1338,7 +1279,6 @@ def test_create_or_get_reset_survives_a_concurrent_reset_of_the_same_row() -> No coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) store.fail(op.id, attempt=1, refusal_code="internal_error", retryable=True) @@ -1363,7 +1303,6 @@ def _concurrent_reset(conn: sqlite3.Connection) -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert resubmitted.id == op.id @@ -1415,7 +1354,6 @@ def _concurrent_insert(conn: sqlite3.Connection) -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert op.id == id_ @@ -1436,7 +1374,6 @@ def test_create_or_get_reset_refuses_at_the_backlog_max() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) store.claim(op.id, submission_count=0, lease_seconds=60, final_delivery=False) store.fail(op.id, attempt=1, refusal_code="internal_error", retryable=True) @@ -1448,7 +1385,6 @@ def test_create_or_get_reset_refuses_at_the_backlog_max() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) with pytest.raises(OperationBacklogFull): @@ -1458,7 +1394,6 @@ def test_create_or_get_reset_refuses_at_the_backlog_max() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) finally: fake.close() @@ -1494,7 +1429,6 @@ def test_mark_dispatched_records_once_per_submission() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert op.dispatched_at is None @@ -1512,7 +1446,6 @@ def test_mark_dispatched_records_once_per_submission() -> None: coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert (reset.submission_count, reset.dispatched_at) == (1, None) # A late record from submission 0 leaves the new submission undispatched. @@ -1545,7 +1478,6 @@ def test_abandon_fails_a_running_row_only_once_nothing_will_finish_it( coverage_stamp=_STAMP, origin_catalogue_build_id=_BUILD, estimate=_ESTIMATE, - rejected_version=None, ) assert store.abandon(op.id, attempt=0) is None assert store.claim( @@ -1591,7 +1523,6 @@ def test_create_or_get_resets_a_running_row_only_past_its_lease( "coverage_stamp": _STAMP, "origin_catalogue_build_id": _BUILD, "estimate": _ESTIMATE, - "rejected_version": None, } op = store.create_or_get(**submission) assert store.claim( diff --git a/tests/datasets_api/test_operation_submit_door.py b/tests/datasets_api/test_operation_submit_door.py index 7e1fb615..188b721b 100644 --- a/tests/datasets_api/test_operation_submit_door.py +++ b/tests/datasets_api/test_operation_submit_door.py @@ -301,7 +301,6 @@ def _submit( async_max_rows=_SETTINGS.async_max_rows, async_max_requests=_SETTINGS.async_max_requests, members=_MEMBERS, - rejected_head=None, ) return excinfo.value @@ -389,7 +388,6 @@ def test_resubmit_of_a_never_dispatched_pending_row_enqueues_it() -> None: coverage_stamp=stamp, origin_catalogue_build_id="fake-catalogue", estimate=estimate, - rejected_version=None, ) assert operation.state == "pending" @@ -440,7 +438,6 @@ def test_resubmit_of_a_deterministically_failed_operation_stays_failed() -> None coverage_stamp=stamp, origin_catalogue_build_id="fake-catalogue", estimate=estimate, - rejected_version=None, ) operation_store.claim(operation.id, submission_count=0, lease_seconds=60, final_delivery=False) operation_store.fail(operation.id, attempt=1, refusal_code="excel_limit", retryable=False) @@ -458,7 +455,6 @@ def test_resubmit_of_a_deterministically_failed_operation_stays_failed() -> None async_max_rows=_SETTINGS.async_max_rows, async_max_requests=_SETTINGS.async_max_requests, members=_MEMBERS, - rejected_head=None, ) assert excinfo.value.operation.state == "failed" @@ -481,7 +477,6 @@ def _submit_members(operation_store, *, members: list[str]): async_max_rows=_SETTINGS.async_max_rows, async_max_requests=_SETTINGS.async_max_requests, members=members, - rejected_head=None, ) return excinfo.value.operation @@ -539,7 +534,6 @@ def test_over_row_ceiling_refuses_before_any_operation_or_enqueue() -> None: async_max_rows=_SETTINGS.async_max_rows, async_max_requests=_SETTINGS.async_max_requests, members=_MEMBERS, - rejected_head=None, ) assert excinfo.value.code == "over_ceiling" @@ -572,7 +566,6 @@ def test_row_ceiling_at_the_cap_is_admitted() -> None: async_max_rows=_SETTINGS.async_max_rows, async_max_requests=_SETTINGS.async_max_requests, members=_MEMBERS, - rejected_head=None, ) operation = excinfo.value.operation @@ -600,7 +593,6 @@ def test_over_request_ceiling_refuses_before_any_operation_or_enqueue() -> None: async_max_rows=_SETTINGS.async_max_rows, async_max_requests=_SETTINGS.async_max_requests, members=_MEMBERS, - rejected_head=None, ) assert excinfo.value.ceiling_kind == "operation_requests" @@ -631,7 +623,6 @@ def test_request_ceiling_at_the_cap_is_admitted() -> None: async_max_rows=_SETTINGS.async_max_rows, async_max_requests=_SETTINGS.async_max_requests, members=_MEMBERS, - rejected_head=None, ) operation = excinfo.value.operation @@ -656,6 +647,5 @@ def test_over_row_ceiling_takes_priority_over_request_ceiling_reporting() -> Non async_max_rows=_SETTINGS.async_max_rows, async_max_requests=_SETTINGS.async_max_requests, members=_MEMBERS, - rejected_head=None, ) assert excinfo.value.ceiling_kind == "operation_rows" diff --git a/tests/datasets_api/test_refresh_door.py b/tests/datasets_api/test_refresh_door.py index 16c0091c..433e774c 100644 --- a/tests/datasets_api/test_refresh_door.py +++ b/tests/datasets_api/test_refresh_door.py @@ -818,7 +818,8 @@ def test_a_raw_version_linking_both_renderings_refuses_once_they_differ(): def _worker_deps(plain: _PlainRig) -> DefinitionDeps: - """The worker's deps over the rig's stores, graph, SDMX body, and coverage.""" + """The worker's deps over the rig's stores, graph, SDMX body, and coverage. The worker + advances the rig's head, as the deployed worker does.""" rig = plain.rig settings = DatasetsBuildSettings() return DefinitionDeps( @@ -847,6 +848,7 @@ def _worker_deps(plain: _PlainRig) -> DefinitionDeps: coverage=plain.coverage, version_writer=LiveVersionRecordWriter(store=rig.version_store), snapshot_reader=LiveSnapshotReader(store=rig.version_store), + head_store=rig.head_store, head_recheck_interval=timedelta(seconds=settings.head_recheck_seconds), head_max_age=timedelta(seconds=settings.head_max_age_seconds), ) @@ -907,8 +909,9 @@ def test_a_done_rebuild_of_a_kind_column_frame_is_checked_under_the_current_dige monkeypatch: pytest.MonkeyPatch, ): """A rebuild whose rendering `carries_party_kind` is linked by the worker under the - key salted with the serving code index's digest. The refresh checks the same key, so it - answers `moved`; a key without the digest would find nothing and run the row again.""" + key salted with the serving code index's digest. The refresh reads the head the worker + advanced under the same key, so it answers `moved`; a key without the digest would find + nothing and submit the operation again.""" real_frame_options = refresh_module._frame_options monkeypatch.setattr( refresh_module, @@ -981,7 +984,7 @@ def run_queued_rebuild(*, submission_count: int) -> str: resp = rig.refresh(spec_id=plain.spec_id, content_version=plain.content_version) - # The done row is failed and submitted again, so the answer is the rebuild, not its version. + # The submission resets the done row and enqueues it again, so the answer is the rerun. assert resp.status_code == 200, resp.text assert resp.json() == {"status": "pending", "operation_id": operation_id, "retry_after_s": 1.5} assert rig.task_client.enqueued == [(operation_id, 0), (operation_id, 1)] diff --git a/tests/datasets_api/test_refusal_wire_equivalence.py b/tests/datasets_api/test_refusal_wire_equivalence.py index 01576857..a7a6314e 100644 --- a/tests/datasets_api/test_refusal_wire_equivalence.py +++ b/tests/datasets_api/test_refusal_wire_equivalence.py @@ -81,7 +81,6 @@ FrameNotRefreshable, RefreshBackpressure, RefreshRebuildFailed, - RefreshRebuildUnservable, RenderingNotRecorded, ) from datasets_api.doors.register import RegistryBackpressure @@ -435,7 +434,6 @@ def _column_unknown_at_pivot() -> ColumnUnknownError: ("refresh", FrameNotRefreshable(spec_id="ops_exemplar")), ("refresh", RenderingNotRecorded(spec_id="spec_exemplar", content_version="cv_exemplar")), ("refresh", RefreshRebuildFailed(operation_id="op_exemplar", refusal_code="over_ceiling")), - ("refresh", RefreshRebuildUnservable(operation_id="op_exemplar")), ("refresh", RefreshBackpressure()), ("refresh", RefreshNotPermitted(presented=False)), ("meta", OperationNotFound(operation_id="op_exemplar")), @@ -530,12 +528,12 @@ def _column_unknown_at_pivot() -> ColumnUnknownError: _DELIBERATE_DIFFS: dict[str, dict[str, object]] = { # A refusal renders its own alternatives. A facet refusal's alternatives reach the top level # only when it is raised outside aggregation with alternatives, which no raise site does. - "extra-135-FacetNotAvailableError": { + "extra-134-FacetNotAvailableError": { "alternatives": [{"ref": "Exemplar", "label": "Exemplar"}], }, # `/facets/unit` locates the failure in the materialisation request, not the posted body, # so it stays the refusal's `facet_pointer` and leaves the top level. - "extra-137-DefaultFacetUndefinedError": {"pointer": None}, + "extra-136-DefaultFacetUndefinedError": {"pointer": None}, # Publish stores the canonical frame, so its refusals name a stored frame in place of frame1. "publish-105-PublishBytesDiverge": { "detail": "spec_id 'ops_exemplar' content_version 'ocv_exemplar' already holds a stored " diff --git a/tests/datasets_api/test_task_client.py b/tests/datasets_api/test_task_client.py index 332199a4..bee36c6f 100644 --- a/tests/datasets_api/test_task_client.py +++ b/tests/datasets_api/test_task_client.py @@ -99,7 +99,6 @@ def test_a_recovered_create_answered_already_exists_is_recorded() -> None: coverage_stamp="cov_1", origin_catalogue_build_id="fake-catalogue", estimate=Estimate(rows=200_000, requests=5), - rejected_version=None, ) cloud_tasks = _LandsThenTimesOut() client = _client(cloud_tasks) diff --git a/tests/datasets_api/test_worker.py b/tests/datasets_api/test_worker.py index 8727bf47..45c7b212 100644 --- a/tests/datasets_api/test_worker.py +++ b/tests/datasets_api/test_worker.py @@ -170,7 +170,6 @@ def _claimed_operation( coverage_stamp="cov_1", origin_catalogue_build_id="fake-catalogue", estimate=Estimate(rows=200_000, requests=5), - rejected_version=None, ) claimed = operation_store.claim( operation.id, submission_count=0, lease_seconds=lease_seconds, final_delivery=False @@ -520,7 +519,6 @@ def _claimed_operation_with_malformed_options(operation_store: LiveOperationStor coverage_stamp="cov_1", origin_catalogue_build_id="fake-catalogue", estimate=Estimate(rows=200_000, requests=5), - rejected_version=None, ) claimed = operation_store.claim( operation.id, submission_count=0, lease_seconds=30, final_delivery=False diff --git a/tests/datasets_api/test_worker_app.py b/tests/datasets_api/test_worker_app.py index b14918a3..ffa55d0c 100644 --- a/tests/datasets_api/test_worker_app.py +++ b/tests/datasets_api/test_worker_app.py @@ -60,7 +60,6 @@ def _submit(operation_store: OperationStore, *, subject_id: str = _SUBJECT_ID) - coverage_stamp="cov_1", origin_catalogue_build_id="fake-catalogue", estimate=Estimate(rows=200_000, requests=5), - rejected_version=None, ) return operation.id diff --git a/tests/datasets_api/test_worker_e2e.py b/tests/datasets_api/test_worker_e2e.py index d566cd7f..5d831014 100644 --- a/tests/datasets_api/test_worker_e2e.py +++ b/tests/datasets_api/test_worker_e2e.py @@ -15,11 +15,12 @@ from code_index.shared import get_index from dataset_store.clock import Clock, utc_now from dataset_store.snapshot import LiveSnapshotReader +from datasets_api import current as current_module from datasets_api import kinds from datasets_api.admission import MaterialisePermits from datasets_api.config import DatasetsBuildSettings from datasets_api.doors.definition import DefinitionDeps -from datasets_api.operation_store import LiveOperationStore +from datasets_api.operation_store import LiveOperationStore, Operation from datasets_api.operations.oidc import FakeOidcVerifier from datasets_api.operations.task_client import FakeTaskClient from datasets_api.operations.worker_app import build_worker_app @@ -28,6 +29,7 @@ from graph_release.service import ReleaseService from materialise import EnrichmentLookup from materialise.rate_tables import RateTableCache +from series_coverage.index import Estimate from spec_registry import LiveSpecRegistry from spec_registry.reader import LiveSpecReader from statvar_members import MemberEnumerator @@ -51,12 +53,15 @@ _PRINCIPAL = "e2e-test-principal" -def _sdmx_transport(requests: list[httpx.Request]): +def _sdmx_transport(requests: list[httpx.Request], body: Callable[[], str] = lambda: _CSV_BODY): + """Answers every SDMX request with `body()`, read per request so a test can change the + data between runs.""" + def handler(request: httpx.Request) -> httpx.Response: requests.append(request) return httpx.Response( 200, - text=_CSV_BODY, + text=body(), headers={"content-type": "application/vnd.sdmx.data+csv;version=2.0.0"}, ) @@ -254,6 +259,122 @@ def test_a_resubmit_after_the_head_aged_out_restarts_the_done_operation( assert rebuilt is not None and rebuilt.confirmed_at == clock["t"] +def test_a_resubmit_after_another_rendering_moved_the_head_serves_the_new_head( + monkeypatch: pytest.MonkeyPatch, +): + """Rendering A completes at V1. The head ages out, the data changes, and rendering B's + operation rebuilds the head to V2, which does not link A. A resubmit of A resets its done + operation, so the 202 reads pending and the poll serves V2. Before, the 202 read done and + the poll served V1.""" + clock = {"t": datetime(2026, 10, 24, tzinfo=UTC)} + data = {"body": _CSV_BODY} + runs: list[Callable[[], None]] = [] + api_client, _operation_store, forwarded = api_and_worker( + graph=_graph(), + sdmx=lambda: SdmxClient( + base_url=_BASE_URL, transport=_sdmx_transport([], lambda: data["body"]) + ), + coverage=_coverage_service, + now=lambda: clock["t"], + deferred=runs, + ) + monkeypatch.setattr(kinds.spec_id, "within_sync_budget", lambda estimate, **_: False) + body_a = {**_HAPPY_PATH_BODY, "options": {"bundle": False}} + body_b = {**_HAPPY_PATH_BODY, "options": {"bundle": False, "format": "parquet"}} + + first_a = api_client.post("/api/datasets", json=body_a) + assert first_a.status_code == 202, first_a.text + runs.pop()() + op_a = first_a.json()["operation"]["operation_id"] + v1 = api_client.get(first_a.headers["location"]).json()["manifest"]["content_version"] + + clock["t"] += timedelta(seconds=DatasetsBuildSettings().head_max_age_seconds + 1) + data["body"] = _CSV_BODY.replace(",2016,2000,", ",2016,3000,") + first_b = api_client.post("/api/datasets", json=body_b) + assert first_b.status_code == 202, first_b.text + runs.pop()() + v2 = api_client.get(first_b.headers["location"]).json()["manifest"]["content_version"] + assert v2 != v1 + + again_a = api_client.post("/api/datasets", json=body_a) + + assert again_a.status_code == 202, again_a.text + assert again_a.json()["operation"]["status"] == "pending" + assert forwarded[-1] == (op_a, 1) + runs.pop()() + polled = api_client.get(again_a.headers["location"]) + assert polled.status_code == 200, polled.text + assert polled.json()["manifest"]["content_version"] == v2 + + +def _complete_then_fill_backlog( + monkeypatch: pytest.MonkeyPatch, + operation_store: LiveOperationStore, + runs: list[Callable[[], None]], +) -> None: + """Delays the next submission between its head read and the store: the deferred run + completes its operation, and another submission takes the backlog's one place.""" + create_or_get = operation_store.create_or_get + + def _delayed(**submission) -> Operation: + runs.pop()() + create_or_get( + request_identity={"subject_id": "spec_other", "member_dcids": []}, + options={}, + coverage_stamp="stamp_other", + origin_catalogue_build_id="build_other", + estimate=Estimate(rows=1, requests=1), + ) + return create_or_get(**submission) + + monkeypatch.setattr(operation_store, "create_or_get", _delayed) + + +@pytest.mark.parametrize("door", ["mint", "definition"]) +def test_a_submission_that_finds_its_operation_just_completed_serves_the_head_when_the_backlog_is_full( + monkeypatch: pytest.MonkeyPatch, door: str +): + """A second submission reads the head before the first one's run completes and reaches the + store after it, with the backlog full. It answers 200 from the head that run advanced, and + resubmits nothing. Before, it reset the done operation into backlog admission and answered + 503 for a result the head served.""" + monkeypatch.setenv("DATASETS_OPERATION_BACKLOG_MAX", "1") + clock = {"t": datetime(2026, 10, 24, tzinfo=UTC)} + runs: list[Callable[[], None]] = [] + api_client, operation_store, forwarded = api_and_worker( + graph=_graph(), + sdmx=lambda: SdmxClient(base_url=_BASE_URL, transport=_sdmx_transport([])), + coverage=_coverage_service, + now=lambda: clock["t"], + deferred=runs, + ) + if door == "mint": + monkeypatch.setattr(kinds.spec_id, "within_sync_budget", lambda estimate, **_: False) + submit = lambda: api_client.post("/api/datasets", json=_HAPPY_PATH_BODY) # noqa: E731 + else: + minted = api_client.post("/api/datasets", json=_HAPPY_PATH_BODY) + assert minted.status_code == 200, minted.text + # The head ages out, so the definition door rebuilds, over budget, as an operation. + clock["t"] += timedelta(seconds=DatasetsBuildSettings().head_max_age_seconds + 1) + monkeypatch.setattr(current_module, "within_sync_budget", lambda estimate, **_: False) + spec_id = minted.json()["definition"]["spec_id"] + submit = lambda: api_client.get(f"/api/datasets/{spec_id}") # noqa: E731 + + first = submit() + assert first.status_code == 202, first.text + operation_id = first.json()["operation"]["operation_id"] + _complete_then_fill_backlog(monkeypatch, operation_store, runs) + + second = submit() + + assert second.status_code == 200, second.text + done = operation_store.get(operation_id) + assert done is not None and (done.state, done.submission_count) == ("done", 0) + assert second.json()["trace"]["served_from"] == "head" + assert second.json()["content_version"] == done.content_version + assert forwarded == [(operation_id, 0)] + + def _rendered(manifest: dict) -> list[tuple[str, str]]: return sorted( (json.dumps(artifact["rendering"], sort_keys=True), artifact["artifact_digest"]) diff --git a/tests/engine/test_choice_options.py b/tests/engine/test_choice_options.py index 6b8eb431..ed2af3d6 100644 --- a/tests/engine/test_choice_options.py +++ b/tests/engine/test_choice_options.py @@ -27,6 +27,7 @@ _FERTILITY = "fertility rate" _COD = "country/COD" +_COG = "country/COG" _FERTILITY_SV = "FertilityRate_Person_Female" _FLOW_SV = "TESTFIXTURE/ChoiceOptions/Flow" _PLACES = { @@ -86,12 +87,14 @@ def _fertility_client(llm: FakeLLM | _ShapeZeroLLM | None = None) -> TestClient: ) -def _flow_client() -> TestClient: - """A flow StatVar declaring a creditor and a debtor role, over four countries and the - European Union.""" +def _flow_client(llm: _ShapeZeroLLM | None = None) -> TestClient: + """A flow StatVar declaring a creditor and a debtor role, over six countries, two of + them Congos, and the European Union.""" graph = FakeGraph( nodes={ **{dcid: {"label": name, "type": "Country"} for name, dcid in _PLACES.items()}, + _COD: {"label": "Congo [DRC]", "type": "Country"}, + _COG: {"label": "Congo [Republic]", "type": "Country"}, _FLOW_SV: { "label": "Test flow", "arcs": { @@ -108,7 +111,7 @@ def _flow_client() -> TestClient: resolve=dict(_PLACES), obs={}, ) - return _client(graph, _ShapeZeroLLM()) + return _client(graph, llm or _ShapeZeroLLM()) def _client(graph: FakeGraph, llm: _ShapeZeroLLM) -> TestClient: @@ -193,12 +196,59 @@ def test_a_name_the_query_words_leave_open_asks_its_choice_in_those_words(): assert no_data["entity_terms"] == ["Congo"] (group,) = no_data["ambiguous_entities"] assert group["text"] == "Congo" - assert {c["ref"] for c in group["candidates"]} == {_COD, "country/COG"} + assert {c["ref"] for c in group["candidates"]} == {_COD, _COG} assert [o["mentions"] for o in no_data["options"]] == [["Congo"], ["Congo"]] assert [e["text"] for e in part["restatement"]["entities"]] == ["Congo"] _assert_options_settle(client, no_data) +class _TwoCongosExtractionLLM(_ShapeZeroLLM): + """Reads every query as aid from the Republic of the Congo to the Democratic Republic + of the Congo, each named in the query "Congo", and elects shape 0 with no bindings.""" + + async def generate_structured(self, *, prompt, system, schema): + if schema is Extraction: + entities = [ + ExtractedEntity( + text="Republic of the Congo", span="Congo", flow_role="origin", role_span="from" + ), + ExtractedEntity( + text="Democratic Republic of the Congo", + span="Congo", + flow_role="destination", + role_span="to", + ), + ] + return Extraction(kind="resolution", variables=["aid"], entities=entities), None + return await super().generate_structured(prompt=prompt, system=system, schema=schema) + + +def test_two_names_the_query_words_alike_ask_each_choice_on_its_own(): + """The parsed door reads one text as one place, so two names that "Congo" leaves open + keep the extraction's names, and each end offers its own choice.""" + client = _flow_client(_TwoCongosExtractionLLM()) + + part = _raw_part(client, "aid from Congo to Congo") + + no_data = part["no_data"] + assert no_data["reason"] == "entity_ambiguous" + names = ["Republic of the Congo", "Democratic Republic of the Congo"] + assert [group["text"] for group in no_data["ambiguous_entities"]] == names + pins = [ + {e["text"]: e.get("dcid") for e in option["data_query"]["parts"][0]["entities"]} + for option in no_data["options"] + ] + assert pins == [ + {names[0]: _COD, names[1]: None}, + {names[0]: _COG, names[1]: None}, + {names[0]: None, names[1]: _COD}, + {names[0]: None, names[1]: _COG}, + ] + for option in no_data["options"]: + response = client.post("/api/search/resolve", json={"input": option["data_query"]}) + assert response.status_code == 200, (option["label"], response.text) + + def test_a_name_the_query_words_settle_keeps_its_name(): """ "DR Congo" names one country, so the part answers under the extraction's name.""" client = _fertility_client(_CongoExtractionLLM("DR Congo")) diff --git a/tests/engine/test_membership_authority.py b/tests/engine/test_membership_authority.py index 935797a5..8df7367a 100644 --- a/tests/engine/test_membership_authority.py +++ b/tests/engine/test_membership_authority.py @@ -53,6 +53,7 @@ MemberEnumerator, MemberGroup, five_tuple_from_identity, + member_digest, ) from tests.datasets_api._graph_doubles import FakeMintGraphReader, _FakeSubjectPage from tests.datasets_api._member_bounds import mint_bounds @@ -65,6 +66,8 @@ _DETECT_QUERY, _EARTH_CHILDREN, _TOTAL_AND_PART_NODES, + _FixedBindLLM, + _OpenSlotsBindLLM, _scope_sdmx, ) from tests.engine.test_place_scope_default import _SV as _POPULATION @@ -710,14 +713,22 @@ def test_a_posted_set_that_is_the_walked_family_is_kept(): ) +@pytest.mark.parametrize( + "llm", + [_FixedBindLLM(), _OpenSlotsBindLLM("gender")], + ids=["slot-left-out", "slot-drafted-unbound"], +) @pytest.mark.parametrize("walk", list(_UNSETTLED_POPULATION_WALKS)) -def test_a_subject_only_answer_whose_walk_settles_nothing_answers_its_recall_and_warns(walk): +def test_a_subject_only_answer_whose_walk_settles_nothing_answers_its_recall_and_warns(walk, llm): """Recall holds the female part alone. With no walk, or one that settles nothing, the part - answers, and the answer carries `membership_unverified`.""" + answers, and the answer carries `membership_unverified`. A reply drafting `gender` + unbound has it claimed as `Female` from the sample, and with no family read the claim + stands on the sample.""" graph = _population_graph(_POPULATION_FEMALE) result = _run_place_less( graph=graph, + llm=llm, sdmx=_scope_sdmx(graph), materialise_graph=_UNSETTLED_POPULATION_WALKS[walk](), ) @@ -727,6 +738,37 @@ def test_a_subject_only_answer_whose_walk_settles_nothing_answers_its_recall_and assert MEMBERSHIP_UNVERIFIED in _codes(result) +def test_a_sampled_claim_the_walked_family_confirms_states_the_members_the_mint_serves(): + """The family holds the female part alone, and a reply drafting `gender` unbound has it + claimed as `Female` from the sample. The walk reads `gender` open and settles the part, + which asserts `Female`, so the claim stands and the walk is keyed to the identity that + binds it: the answer states the digest of the family the mint serves.""" + graph = _population_graph(_POPULATION_FEMALE) + member_graph = member_walk_reader(graph, stat_vars=[_POPULATION_FEMALE]) + + result = _run_place_less( + graph=graph, + llm=_OpenSlotsBindLLM("gender"), + sdmx=_scope_sdmx(graph), + materialise_graph=member_graph, + ) + + assert result.status == "definite", result.no_data_reason + (spec,) = result.specs + assert [sv.ref.ref for sv in spec.resolved.members] == [_POPULATION_FEMALE] + assert MEMBERSHIP_UNVERIFIED not in _codes(result) + minted = resolve_definition_members( + spec.definition.identity.model_dump(mode="json", exclude_none=True), + spec_id=spec.spec_id, + graph=member_graph, + enumerator=MemberEnumerator(), + code_index=get_index(), + bounds=mint_bounds(), + ) + assert [shape.stat_var.removeprefix("dcid:") for shape in minted.shapes] == [_POPULATION_FEMALE] + assert spec.resolved.shape.member_digest == member_digest([_POPULATION_FEMALE]) + + # --------------------------------------------------------------------------- # An alternate # --------------------------------------------------------------------------- diff --git a/tests/engine/test_place_scope_default.py b/tests/engine/test_place_scope_default.py index 55e9c9c4..270b76da 100644 --- a/tests/engine/test_place_scope_default.py +++ b/tests/engine/test_place_scope_default.py @@ -42,7 +42,11 @@ from tests.engine._binders import elected_reply from tests.engine._fake_coverage import empty_coverage_snapshot from tests.engine._fake_sdmx import FakeSdmxClient -from tests.engine._harness import materialise_double_from_obs, member_walk_reader +from tests.engine._harness import ( + materialise_double_from_obs, + member_walk_reader, + with_member_arcs, +) from tests.engine._quantity import NO_QUANTITY, NO_RATING from tests.fixtures import FakeGraph, make_resolver from tests.materialise._scope_availability import ScopeAvailability @@ -80,6 +84,26 @@ async def generate_structured(self, *, prompt, system, schema): return elected_reply(0, also_plausible=None), None +class _OpenSlotsBindLLM: + """Elects the sole candidate shape and drafts each slot of `properties` unbound, as a + reply that names no value of them does.""" + + def __init__(self, *properties: str) -> None: + self.properties = properties + + async def generate_structured(self, *, prompt, system, schema): + if schema is QuantityOutput: + return NO_QUANTITY + if schema is StatedWordsOutput: + return NO_RATING + assert schema.__name__ == "_UniversalBindOutput" + drafts = [ + SlotBindingDraft(axis="how", property_dcid=prop, kind="unbound") + for prop in self.properties + ] + return elected_reply(0, bindings=drafts, also_plausible=None), None + + class _CountingFakeGraph(FakeGraph): """Count presence and observation probes, including requested entity IDs, and record the default scope's availability reads (`availability_reads`).""" @@ -999,10 +1023,18 @@ def test_a_shape_with_a_total_and_its_part_is_scoped_over_the_total_its_mint_ser assert _scope_messages(result) == {None: _EVERY_PLACE_MESSAGE} -def test_a_recalled_part_answers_as_the_total_its_mint_serves(): +@pytest.mark.parametrize( + "llm", + [_FixedBindLLM(), _OpenSlotsBindLLM("gender")], + ids=["slot-left-out", "slot-drafted-unbound"], +) +def test_a_recalled_part_answers_as_the_total_its_mint_serves(llm): """Recall holds the female part alone and `gender` stays unbound. The mint resolves the emitted identity by its walk to the total and its part, and serves the total, so the - answer lists the total and is scoped over the total's own country.""" + answer lists the total and is scoped over the total's own country. A reply drafting + `gender` unbound leaves a slot whose sampled taxonomy holds `Female` alone, so the + engine claims it from the sample, and the walk, which reads the family with the claim + open, refutes it on the total.""" graph = FakeGraph( nodes=_TOTAL_AND_PART_NODES, detect={_DETECT_QUERY: {"svs": [_SV_FEMALE], "cosine_scores": [0.9]}}, @@ -1015,7 +1047,7 @@ def test_a_recalled_part_answers_as_the_total_its_mint_serves(): ) member_graph = member_walk_reader(graph) - result = _run(graph=graph, sdmx=_scope_sdmx(graph), materialise_graph=member_graph) + result = _run(graph=graph, llm=llm, sdmx=_scope_sdmx(graph), materialise_graph=member_graph) assert result.status == "definite", result.no_data_reason (spec,) = result.specs @@ -1068,7 +1100,9 @@ def observation_facets_batch(self, stat_vars, entities, needs_dates=False, **kwa return super().observation_facets_batch(stat_vars, entities, needs_dates=needs_dates) -@pytest.mark.parametrize("answer", ["elected", "alternate", "lone_alternate"]) +@pytest.mark.parametrize( + "answer", ["elected", "elected_slots_drafted", "alternate", "lone_alternate"] +) def test_each_member_is_counted_and_grounded_at_its_own_scope(answer, monkeypatch): """The shape serves two members neither of which dominates the other: `_U_FEMALE` holds data in A1 and `_U_OVER65` in A2. The shape answers as the elected one, as an alternate @@ -1077,7 +1111,12 @@ def test_each_member_is_counted_and_grounded_at_its_own_scope(answer, monkeypatc leaves both open and serves both, each counted at its own country; an alternate binds `_U_FEMALE`'s constraint, so it shows that member, counts its own country alone, and the lone alternate previews its series there. The elected answer's two places preview - through a scope-wide SDMX read, which this test's SDMX double holds no rows for.""" + through a scope-wide SDMX read, which this test's SDMX double holds no rows for. A reply + drafting `gender` and `age` unbound leaves each a slot whose sampled taxonomy holds one + value, so the engine claims both from the sample, and the walk over the members' arcs, + which reads the family with both claims open, refutes each on the member that asserts + the other.""" + elected = answer in ("elected", "elected_slots_drafted") def facet(year: str) -> list[dict]: return [ @@ -1092,7 +1131,7 @@ def facet(year: str) -> list[dict]: obs = {f"{_U_FEMALE}|country/A1": facet("2020"), f"{_U_OVER65}|country/A2": facet("2021")} svs, scores = [_U_FEMALE, _U_OVER65], [0.9, 0.89] - if answer != "elected": + if not elected: svs, scores = [_SV_B, *svs], [0.95, *scores] if answer == "alternate": obs[f"{_SV_B}|country/A3"] = facet("2021") @@ -1117,10 +1156,15 @@ def facet(year: str) -> list[dict]: children=_EARTH_CHILDREN, ) materialise_graph, materialise_sdmx = materialise_double_from_obs(obs) + if answer == "elected_slots_drafted": + materialise_graph = with_member_arcs(materialise_graph, graph) result = _run( graph=graph, - llm=_FixedBindLLM() if answer == "elected" else _CandidatesBindLLM(), + llm={ + "elected": _FixedBindLLM(), + "elected_slots_drafted": _OpenSlotsBindLLM("gender", "age"), + }.get(answer, _CandidatesBindLLM()), sdmx=_scope_sdmx(graph), materialise_graph=materialise_graph, materialise_sdmx=materialise_sdmx, @@ -1133,7 +1177,7 @@ def facet(year: str) -> list[dict]: spec = next(spec for spec in result.specs if spec.resolved.members[0].ref.ref == _U_FEMALE) members, places = ( ([_U_FEMALE, _U_OVER65], ["country/A1", "country/A2"]) - if answer == "elected" + if elected else ([_U_FEMALE], ["country/A1"]) ) assert [sv.ref.ref for sv in spec.resolved.members] == members diff --git a/tests/engine/test_raw_text_entity_mentions.py b/tests/engine/test_raw_text_entity_mentions.py index 44f60ead..d736d039 100644 --- a/tests/engine/test_raw_text_entity_mentions.py +++ b/tests/engine/test_raw_text_entity_mentions.py @@ -623,6 +623,45 @@ def test_a_recovered_span_takes_no_words_another_entity_names( assert found == {entity.text: Resolved(entity=places[entity.text]) for entity in entities} +@pytest.mark.parametrize( + ("query", "entities", "texts"), + [ + pytest.param( + "ODA disbursements from Germany to Congo", + [ExtractedEntity(text="Germany"), ExtractedEntity(text=_COD_NAME, span="Congo")], + ["Germany", "Congo"], + id="one_name_takes_the_words", + ), + pytest.param( + "aid from Congo to Congo", + [ + ExtractedEntity(text=_COG_NAME, span="Congo", flow_role="origin"), + ExtractedEntity(text=_COD_NAME, span="Congo", flow_role="destination"), + ], + [_COG_NAME, _COD_NAME], + id="two_names_keep_their_own", + ), + ], +) +def test_words_that_leave_a_choice_open_become_one_name_s_text( + query: str, entities: list[ExtractedEntity], texts: list[str] +) -> None: + """A name its words leave open reads as those words only when no other name would read + the same, since the parsed door reads one text as one place per part.""" + resolver = _Resolver( + { + "Congo": Ambiguous(candidates=(_COD, _COG)), + "Germany": Resolved(entity=_DEU), + _COD_NAME: Resolved(entity=_COD), + _COG_NAME: Resolved(entity=_COG), + } + ) + + recovered = _recover(entities, query=query, resolver=resolver) + + assert [entity.text for entity in recovered] == texts + + @pytest.mark.parametrize( ("query", "entity", "registered", "answers", "expected"), [ diff --git a/tests/engine/test_role_class_shapes.py b/tests/engine/test_role_class_shapes.py index 31eeb6c4..c4dc432b 100644 --- a/tests/engine/test_role_class_shapes.py +++ b/tests/engine/test_role_class_shapes.py @@ -903,6 +903,69 @@ def test_a_recalled_index_row_does_not_take_the_debt_shapes_family(): } == {"financialFlowType": "ServicingPayment"} +_IDS_SERVICE = "Amount_FinancialFlow_ServicingPayment_Nominal" +_IDS_INTEREST = "Amount_FinancialFlow_InterestPayment_Nominal" + + +def _ids_flow(label: str, flow_type: str) -> dict: + return _node(label, _RECIPIENT, population_type="FinancialFlow", financialFlowType=flow_type) + + +@pytest.mark.parametrize( + "draft", + [ + SlotBindingDraft( + axis="how", + property_dcid="financialFlowType", + kind="value", + value_dcids=["ServicingPayment"], + evidence="sole_option", + ), + SlotBindingDraft(axis="how", property_dcid="financialFlowType", kind="unbound"), + ], + ids=["claimed-by-the-reply", "claimed-at-settle"], +) +def test_debt_service_keeps_the_flow_type_its_family_is_told_apart_by(draft): + """ "Total debt service of Kenya", as dq-ids-001 asks of Ghana, recalls the IDS debt service total + alone, and the reply binds its one flow type as the slot's sole option, or leaves the + slot unbound and settle claims it. The recalled sample is not the whole family, so the + claim rests on it. The family shares its five-tuple with the CRS rows the index + exports, and the flow type is what tells it apart from them (`bind_shared_values`), so + the walk reads the flow type bound. Read open, the walk lists the IDS interest payments + total beside it, refutes the claim and answers every flow to Kenya.""" + recalled = {_IDS_SERVICE: _ids_flow("Debt service on external debt", "ServicingPayment")} + interest = {_IDS_INTEREST: _ids_flow("Interest payments on external debt", "InterestPayment")} + walked = {**recalled, **interest} + obs = {f"{dcid}|country/KEN": _OBS for dcid in walked} + + class _DraftLLM: + async def generate_structured(self, *, prompt, system, schema): + if schema is StatedWordsOutput: + return NO_RATING + if schema is QuantityOutput: + return NO_QUANTITY + return elected_reply(0, bindings=[draft], also_plausible=None), None + + result = _run( + entities=["Kenya"], + obs=obs, + query="total debt service of Kenya", + nodes=recalled, + value_nodes=interest, + llm=_DraftLLM(), + options=EngineOptions(materialise_graph=_walk_reader(walked)), + coverage=_one_role_coverage(walked, obs), + ) + + assert result.status == "definite" + (spec,) = result.specs + assert [sv.ref.ref for sv in spec.resolved.members] == [_IDS_SERVICE] + assert { + slot.property: getattr(slot.binding, "ref", None) for slot in spec.definition.identity.slots + } == {"financialFlowType": "ServicingPayment"} + assert "membership_unverified" not in {w.code for w in result.warnings} + + def test_an_unindexed_shape_on_a_five_tuple_the_index_carries_keeps_its_own_slot_keys(): """By-creditor debt members declare the roles CRS rows declare. The index holds none of them, so the shape is the debt family and reads no index row.""" @@ -2992,8 +3055,11 @@ def test_a_family_with_no_total_refuses_once_its_sole_option_drafts_go(monkeypat declares a creditor where the others declare the debtor alone. Every member conflicts on a sole-option draft, so all three go, and the members left under the named value declare role sets the election cannot order: the part refuses - `role_signature_incompatible`, as the mint refuses the identity, with no draft still - bound and no value refused on.""" + `role_signature_incompatible`, as the mint refuses the identity, and no value is refused + on. The recalled sample does not refute the drafts, so each stands as a sampled claim. + The walk reads every sampled claim open, meets those role sets at once and refuses + with nothing unbound. With no walk the read unbinds all three drafts, leaving no draft + bound.""" nodes = { _RELAX_BONDS: _node( "External debt stocks, bonds", @@ -3028,12 +3094,14 @@ def unbind(state, unrealisable): assert (result.status, result.no_data_reason) == ("no_data", "role_signature_incompatible") assert result.flagged_match is None assert not result.unbound_qualifiers - (state,) = states - assert { - b.property_dcid: b.value_dcids - for b in state.bindings - if b.kind == "value" and b.property_dcid is not None - } == {"positionType": ["AmountOutstanding"]} + assert [ + { + b.property_dcid: b.value_dcids + for b in state.bindings + if b.kind == "value" and b.property_dcid is not None + } + for state in states + ] == ([] if walked else [{"positionType": ["AmountOutstanding"]}]) @pytest.mark.parametrize("walked", [True, False], ids=["walk-point", "read"]) diff --git a/tests/engine/test_sole_option_validation.py b/tests/engine/test_sole_option_validation.py index 0f1938f4..68860c5d 100644 --- a/tests/engine/test_sole_option_validation.py +++ b/tests/engine/test_sole_option_validation.py @@ -164,6 +164,7 @@ def test_a_sole_option_claim_on_an_index_family_stands_only_when_every_row_asser assert (validated == binding) is kept assert validated.kind == ("value" if kept else "unbound") + assert validated.sampled is False # --------------------------------------------------------------------------- @@ -244,6 +245,7 @@ def test_a_sole_option_claim_on_a_held_family_stands_only_when_every_member_asse assert (validated == binding) is kept assert validated.kind == ("value" if kept else "unbound") + assert validated.sampled is False # --------------------------------------------------------------------------- @@ -371,7 +373,8 @@ def test_a_sole_option_claim_on_a_recalled_sample_falls_to_a_total_lacking_the_v the total. A sample that splits on what a flow is net of holds no total, and the net transfers member stating no concessionality is the reading without one beside the member that states it. The shape holds no held family and the index carries none of - its rows.""" + its rows, so a claim the sample does not refute stands marked sampled, for the walk of + the family to judge.""" shape = replace( _shape("UnindexedFlow"), slot_keys=( @@ -399,8 +402,9 @@ def test_a_sole_option_claim_on_a_recalled_sample_falls_to_a_total_lacking_the_v *_, validated = _validate_sole_option_bindings([*stated, binding], shape) - assert (validated == binding) is kept - assert validated.kind == ("value" if kept else "unbound") + assert (validated.kind, validated.value_dcids, validated.sampled) == ( + ("value", ["IDSConcessional"], True) if kept else ("unbound", [], False) + ) # --------------------------------------------------------------------------- diff --git a/tests/live_refresh/composed_app.py b/tests/live_refresh/composed_app.py index b402e2e3..0b754038 100644 --- a/tests/live_refresh/composed_app.py +++ b/tests/live_refresh/composed_app.py @@ -971,6 +971,7 @@ def compose( rate_cache=rate_cache, coverage=coverage, version_writer=version_writer, + head_store=head_store, snapshot_reader=snapshot_reader, head_recheck_interval=timedelta(seconds=settings.head_recheck_seconds), head_max_age=timedelta(seconds=settings.head_max_age_seconds), diff --git a/uv.lock b/uv.lock index ac6ea317..9770cabf 100644 --- a/uv.lock +++ b/uv.lock @@ -724,10 +724,10 @@ requires-dist = [ { name = "dfo-data-tools", extras = ["generation"], marker = "extra == 'resolve'", editable = "packages/dfo-data-tools" }, { name = "dfo-data-tools", extras = ["sdmx"], marker = "extra == 'coverage'", editable = "packages/dfo-data-tools" }, { name = "dfo-data-tools", extras = ["sdmx"], marker = "extra == 'formats'", editable = "packages/dfo-data-tools" }, - { name = "dfo-spec-kit-components", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdfo-spec-kit-components&rev=ba976a2a74173e1c31b6c03d802cbcf71a022273" }, - { name = "dfo-spec-kit-descriptions", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdescriptions&rev=ba976a2a74173e1c31b6c03d802cbcf71a022273" }, - { name = "dfo-spec-kit-frame-grammar", extras = ["arrow"], git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=ba976a2a74173e1c31b6c03d802cbcf71a022273" }, - { name = "dfo-spec-kit-provenance-docs", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fprovenance-docs&rev=ba976a2a74173e1c31b6c03d802cbcf71a022273" }, + { name = "dfo-spec-kit-components", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdfo-spec-kit-components&rev=0d19172e3695c311ed48ff43a885a9995869e6a5" }, + { name = "dfo-spec-kit-descriptions", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdescriptions&rev=0d19172e3695c311ed48ff43a885a9995869e6a5" }, + { name = "dfo-spec-kit-frame-grammar", extras = ["arrow"], git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=0d19172e3695c311ed48ff43a885a9995869e6a5" }, + { name = "dfo-spec-kit-provenance-docs", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fprovenance-docs&rev=0d19172e3695c311ed48ff43a885a9995869e6a5" }, { name = "fastapi", marker = "extra == 'surface'", specifier = ">=0.138" }, { name = "google-auth", extras = ["requests"], marker = "extra == 'generation'", specifier = ">=2.55" }, { name = "google-auth", extras = ["requests"], marker = "extra == 'sdmx'", specifier = ">=2.55" }, @@ -753,7 +753,7 @@ provides-extras = ["sdmx", "formats", "surface", "resolve", "generation", "cover [[package]] name = "dfo-spec-kit" version = "0.2.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=ba976a2a74173e1c31b6c03d802cbcf71a022273#ba976a2a74173e1c31b6c03d802cbcf71a022273" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=0d19172e3695c311ed48ff43a885a9995869e6a5#0d19172e3695c311ed48ff43a885a9995869e6a5" } dependencies = [ { name = "rfc8785" }, ] @@ -761,12 +761,12 @@ dependencies = [ [[package]] name = "dfo-spec-kit-client-contract" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fclient-contract&rev=ba976a2a74173e1c31b6c03d802cbcf71a022273#ba976a2a74173e1c31b6c03d802cbcf71a022273" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fclient-contract&rev=0d19172e3695c311ed48ff43a885a9995869e6a5#0d19172e3695c311ed48ff43a885a9995869e6a5" } [[package]] name = "dfo-spec-kit-components" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdfo-spec-kit-components&rev=ba976a2a74173e1c31b6c03d802cbcf71a022273#ba976a2a74173e1c31b6c03d802cbcf71a022273" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdfo-spec-kit-components&rev=0d19172e3695c311ed48ff43a885a9995869e6a5#0d19172e3695c311ed48ff43a885a9995869e6a5" } dependencies = [ { name = "dfo-spec-kit-descriptions" }, ] @@ -774,7 +774,7 @@ dependencies = [ [[package]] name = "dfo-spec-kit-descriptions" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdescriptions&rev=ba976a2a74173e1c31b6c03d802cbcf71a022273#ba976a2a74173e1c31b6c03d802cbcf71a022273" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdescriptions&rev=0d19172e3695c311ed48ff43a885a9995869e6a5#0d19172e3695c311ed48ff43a885a9995869e6a5" } dependencies = [ { name = "rfc8785" }, ] @@ -782,7 +782,7 @@ dependencies = [ [[package]] name = "dfo-spec-kit-frame-grammar" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=ba976a2a74173e1c31b6c03d802cbcf71a022273#ba976a2a74173e1c31b6c03d802cbcf71a022273" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=0d19172e3695c311ed48ff43a885a9995869e6a5#0d19172e3695c311ed48ff43a885a9995869e6a5" } dependencies = [ { name = "dfo-spec-kit" }, { name = "pyarrow" }, @@ -792,7 +792,7 @@ dependencies = [ [[package]] name = "dfo-spec-kit-provenance-docs" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fprovenance-docs&rev=ba976a2a74173e1c31b6c03d802cbcf71a022273#ba976a2a74173e1c31b6c03d802cbcf71a022273" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fprovenance-docs&rev=0d19172e3695c311ed48ff43a885a9995869e6a5#0d19172e3695c311ed48ff43a885a9995869e6a5" } dependencies = [ { name = "markdown-it-py" }, ] @@ -2938,7 +2938,7 @@ requires-dist = [ { name = "dfo-data-tools", extras = ["resolve", "sdmx", "coverage"], marker = "extra == 'graph'", editable = "packages/dfo-data-tools" }, { name = "dfo-data-tools", extras = ["sdmx"], marker = "extra == 'sdmx'", editable = "packages/dfo-data-tools" }, { name = "dfo-data-tools", extras = ["surface", "resolve", "coverage", "sdmx"], marker = "extra == 'engine'", editable = "packages/dfo-data-tools" }, - { name = "dfo-spec-kit", marker = "extra == 'engine'", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=ba976a2a74173e1c31b6c03d802cbcf71a022273" }, + { name = "dfo-spec-kit", marker = "extra == 'engine'", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=0d19172e3695c311ed48ff43a885a9995869e6a5" }, { name = "fastapi", marker = "extra == 'api'", specifier = ">=0.138" }, { name = "google-auth", marker = "extra == 'datasets'", specifier = ">=2.34" }, { name = "google-auth", extras = ["requests"], marker = "extra == 'eval'", specifier = ">=2.34" }, @@ -2969,7 +2969,7 @@ lint = [ { name = "ruff", specifier = ">=0.15" }, ] test = [ - { name = "dfo-spec-kit-client-contract", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fclient-contract&rev=ba976a2a74173e1c31b6c03d802cbcf71a022273" }, + { name = "dfo-spec-kit-client-contract", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fclient-contract&rev=0d19172e3695c311ed48ff43a885a9995869e6a5" }, { name = "frictionless", specifier = ">=5.19.0" }, { name = "pytest", specifier = ">=8" }, { name = "pytest-split", specifier = ">=0.10.0" }, From 2a7a490c49805bd116f899443a7135d10853c34e Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sat, 24 Oct 2026 22:17:58 +0200 Subject: [PATCH 6/7] fix(datasets): read the entity refs a warning will carry before any mint writes them (#128) WarningModel gains optional `unserved_entities` and `omitted_entities` (`{count, refs}`) and `omitted_entity_years` (`{count, entities: [{ref, years}]}`), and #120's member model is renamed CountedRefsModel with the same stored keys. Stored bodies are extra="forbid", so this reader deploys before the writer that emits the fields. Nothing writes them yet. --- src/datasets_api/manifest_models.py | 71 +++++++++++++++++++---- tests/datasets_api/test_warning_model.py | 72 +++++++++++++++++++----- 2 files changed, 120 insertions(+), 23 deletions(-) diff --git a/src/datasets_api/manifest_models.py b/src/datasets_api/manifest_models.py index a8b69d2a..2f5cf548 100644 --- a/src/datasets_api/manifest_models.py +++ b/src/datasets_api/manifest_models.py @@ -257,21 +257,48 @@ def sources_by_ref(self) -> dict[str, CitationSource]: return by_ref -class UnservedMembersModel(BaseModel): - """The members of a group of one kind that served nothing (`materialise.codes.UnservedMembers`).""" +class CountedRefsModel(BaseModel): + """The things of one kind a warning counts: how many, and the `dcid:` refs of the first + five, or ten for a conversion's left-out entities (`materialise.codes.CountedRefs`). The message states the count and names none, so a + reader labels the refs.""" model_config = ConfigDict(extra="forbid") - count: int = Field(description="how many members of this kind served nothing") + count: int = Field(description="how many there are") refs: list[str] = Field( - description="the `dcid:` refs of the first five of them, in request order" + description=( + "the `dcid:` refs of the first five of them, ten for omitted_entities: a " + "group's members in request order, entities sorted" + ) + ) + + +class EntityYearsModel(BaseModel): + """One entity a conversion left rows out of, and the years of those rows + (`materialise.codes.EntityYears`).""" + + model_config = ConfigDict(extra="forbid") + ref: str = Field(description="the entity's `dcid:` ref") + years: list[str] = Field( + description="the years left out, as runs of consecutive years, `1990-2001` or `2004`" ) +class CountedEntityYearsModel(BaseModel): + """The entities a conversion left rows out of, each with its years: how many, and the + first ten, sorted (`materialise.codes.CountedEntityYears`). The message states the count + and names none, so a reader labels the refs.""" + + model_config = ConfigDict(extra="forbid") + count: int = Field(description="how many entities there are") + entities: list[EntityYearsModel] = Field(description="the first ten of them, sorted") + + class WarningModel(BaseModel): - """One manifest warning. Only `stat_vars_summed` sets its three structured fields, and only - a group's `coverage_below_scope` its three member fields. A `None` field is left out of - every dump, so other codes' warnings, on the wire and in stored version bodies, stay - exactly `{code, severity, message}`. + """One manifest warning. Only `stat_vars_summed` sets its three structured fields, a + group's `coverage_below_scope` its three member fields, `coverage_below_scope` + `unserved_entities`, `currency_unmapped_omitted` `omitted_entities`, and + `deflator_missing_omitted` `omitted_entity_years`. A `None` field is left out of every dump, so other codes' warnings, on + the wire and in stored version bodies, stay exactly `{code, severity, message}`. """ model_config = ConfigDict(extra="forbid") @@ -312,7 +339,7 @@ class WarningModel(BaseModel): "group requested" ), ) - off_cadence_members: UnservedMembersModel | None = Field( + off_cadence_members: CountedRefsModel | None = Field( default=None, exclude_if=_is_none, description=( @@ -321,7 +348,7 @@ class WarningModel(BaseModel): "leave out; absent when there are none" ), ) - out_of_window_members: UnservedMembersModel | None = Field( + out_of_window_members: CountedRefsModel | None = Field( default=None, exclude_if=_is_none, description=( @@ -330,6 +357,30 @@ class WarningModel(BaseModel): "an absent member as zero" ), ) + unserved_entities: CountedRefsModel | None = Field( + default=None, + exclude_if=_is_none, + description=( + "coverage_below_scope only: the entities the request binds that served no " + "observation, sorted; absent when there are none" + ), + ) + omitted_entities: CountedRefsModel | None = Field( + default=None, + exclude_if=_is_none, + description=( + "currency_unmapped_omitted only: the entities whose rows the conversion left " + "out, sorted" + ), + ) + omitted_entity_years: CountedEntityYearsModel | None = Field( + default=None, + exclude_if=_is_none, + description=( + "deflator_missing_omitted only: the entities whose rows the conversion left out " + "for a missing deflator, sorted, each with the years of those rows" + ), + ) class DiagnosticModel(BaseModel): diff --git a/tests/datasets_api/test_warning_model.py b/tests/datasets_api/test_warning_model.py index e4b4a9f6..d9718b24 100644 --- a/tests/datasets_api/test_warning_model.py +++ b/tests/datasets_api/test_warning_model.py @@ -1,7 +1,9 @@ -"""Test a stored `stat_vars_summed` warning's structured fields and their absence elsewhere.""" +"""Test a stored warning's structured fields and their absence elsewhere.""" from __future__ import annotations +import pytest + from datasets_api.manifest import warning_model from datasets_api.manifest_models import WarningModel @@ -41,18 +43,62 @@ def test_a_stored_stat_vars_summed_warning_without_a_label_omits_the_label_key() assert dumped["contributing_member_count"] == 74 -def test_a_stored_coverage_warning_with_member_fields_reads_back_unchanged(): - """A stored version body whose group `coverage_below_scope` carries the member fields - reads back, so a revision that does not write them still serves versions that do.""" - stored = { - "code": "coverage_below_scope", - "severity": "warn", - "message": "2 of 9 series report at a different frequency and are left out of the figures.", - "member_count": 9, - "off_cadence_members": {"count": 2, "refs": ["dcid:MemberA", "dcid:MemberB"]}, - "out_of_window_members": {"count": 1, "refs": ["dcid:MemberC"]}, - } - +@pytest.mark.parametrize( + "stored", + [ + pytest.param( + { + "code": "coverage_below_scope", + "severity": "warn", + "message": "2 of 9 series report at a different frequency and are left out " + "of the figures.", + "member_count": 9, + "off_cadence_members": {"count": 2, "refs": ["dcid:MemberA", "dcid:MemberB"]}, + "out_of_window_members": {"count": 1, "refs": ["dcid:MemberC"]}, + }, + id="group-members", + ), + pytest.param( + { + "code": "coverage_below_scope", + "severity": "warn", + "message": "No observations were served for 6 requested entities.", + "unserved_entities": { + "count": 6, + "refs": [f"dcid:country/C{i}" for i in range(5)], + }, + }, + id="unserved-entities", + ), + pytest.param( + { + "code": "currency_unmapped_omitted", + "severity": "warn", + "message": "It left out 1 entity.", + "omitted_entities": {"count": 1, "refs": ["dcid:country/PSE"]}, + }, + id="currency-omitted-entities", + ), + pytest.param( + { + "code": "deflator_missing_omitted", + "severity": "warn", + "message": "It left out rows of 2 entities.", + "omitted_entity_years": { + "count": 2, + "entities": [ + {"ref": "dcid:country/AZE", "years": ["2022"]}, + {"ref": "dcid:country/TWN", "years": ["1990-2001", "2004"]}, + ], + }, + }, + id="deflator-omitted-entities", + ), + ], +) +def test_a_stored_warning_with_counted_refs_reads_back_unchanged(stored): + """A stored version body whose warning carries counted refs reads back, so a revision + that does not write them still serves versions that do.""" assert WarningModel.model_validate(stored).model_dump(mode="json") == stored From e5745836f8e0c9141dea1f9f4cf5dde226aaa118 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sat, 24 Oct 2026 22:29:23 +0200 Subject: [PATCH 7/7] fix(materialise): state counts in stored warnings and carry the refs they named (#129) coverage_below_scope's unserved-entity list, currency_unmapped_omitted and deflator_missing_omitted named raw refs in their stored messages, which no later code change could relabel. Their messages now give counts only, and the warning carries the refs as `unserved_entities`, `omitted_entities` and `omitted_entity_years`: five unserved entities, and the ten entities the conversion messages used to name, each deflator entity with its omitted years. Cards label the refs in the frame store's existing label batch, and Search's two omission warnings carry the entities, their labels and the deflator years in `params`. Structural tests tie every counted field to a writer and a card row, and pin that no stored message names its refs. --- docs/conformance-datasets-api-v1.md | 2 +- docs/conformance-search-api-v1.md | 2 +- .../src/assembler/mint/__init__.py | 4 +- .../src/assembler/mint/provenance.py | 81 +++++++--- .../dfo-data-tools/src/materialise/codes.py | 37 ++++- .../dfo-data-tools/src/materialise/finish.py | 152 +++++++++++------- src/datasets_api/manifest.py | 5 +- src/datasets_api/warnings.py | 5 +- src/presentation_store/frame_store.py | 21 +-- .../engine/region/candidate_collapse.py | 6 +- src/search/engine/region/convertibility.py | 49 +++++- src/search/engine/region/phase_answer.py | 4 +- src/search/engine/region/promote_finalize.py | 5 +- src/search/models/envelope.py | 15 +- tests/assembler/test_mint.py | 94 ++++++++++- .../test_currency_unmapped_omitted_gate.py | 2 +- tests/engine/test_served_slot_options.py | 33 +++- tests/engine/test_unit_coverage.py | 48 ++++-- tests/materialise/test_conversion_finish.py | 23 +-- tests/materialise/test_pipeline.py | 91 ++++++++++- tests/presentation_store/test_frame_store.py | 19 ++- 21 files changed, 527 insertions(+), 171 deletions(-) diff --git a/docs/conformance-datasets-api-v1.md b/docs/conformance-datasets-api-v1.md index 53b6d127..52688d5a 100644 --- a/docs/conformance-datasets-api-v1.md +++ b/docs/conformance-datasets-api-v1.md @@ -148,7 +148,7 @@ Five routes are outside the V1 contract text. The code registry's mappings for ` | C150 | On definition retrieval and its options sibling, `trace` records `served_from`, `head` when the stored record was served, either inside the re-check interval or after a matching probe, and `materialise` when this request rebuilt the frame, on a probe mismatch, past the maximum age, or when a window the definition left open now ends at another year than the stored record read, inside the re-check interval too. The options route reports `served_from` under the same rule; a `materialise` answer records the version and advances the head, as definition retrieval does (C158 for a posted definition) | implemented | | | C151 | `trace` records `confirmed_at`, the UTC timestamp of when the stored head was last confirmed. Definition retrieval records a probe match, so its value moves. The options route confirms no probe match, so a head it serves reports the stored instant. A `done` operation poll reports `served_from: materialise` and the `confirmed_at` its run recorded when it completed. Snapshot retrieval carries neither field, since it reads one stored version with no currency judgement | implemented | | | C76 | `coverage_clamp` marks a window narrowed to what the source holds, paired with `coverage_below_scope` | implemented | | -| C77 | `warnings`: `{code, severity, message}` from the platform code registry, plus `contributing_member_count`, `stat_var_ref`, and `stat_var_label` on `stat_vars_summed` only (C118), and on a group's `coverage_below_scope` `member_count` with `off_cadence_members` (members reporting the requested years at another frequency, left out of the figures) and `out_of_window_members` (members with no observation in the requested years, absent when the served provenance reads an absent member as zero, the rule the rendered sum reads), each `{count, refs}` with the `dcid:` refs of the first five, while the message counts them and names none. `WarningModel` forbids unknown fields, so a revision without these fields cannot read a version recorded with them; a warning scoped to one rendering (`enrichment_unavailable`) rides its artifact entry instead of the top-level list | partial | `enrichment_unavailable` is emitted on the top-level `warnings` list; artifact entries carry no `warnings` member. Issue: none | +| C77 | `warnings`: `{code, severity, message}` from the platform code registry, plus `contributing_member_count`, `stat_var_ref`, and `stat_var_label` on `stat_vars_summed` only (C118), and on a group's `coverage_below_scope` `member_count` with `off_cadence_members` (members reporting the requested years at another frequency, left out of the figures) and `out_of_window_members` (members with no observation in the requested years, absent when the served provenance reads an absent member as zero, the rule the rendered sum reads), on `coverage_below_scope` `unserved_entities` (the bound entities that served no observation), and on `currency_unmapped_omitted` `omitted_entities` (the entities whose rows the conversion left out), each `{count, refs}` with the `dcid:` refs of the first five, ten for `omitted_entities`, a group's members in request order and entities sorted; and on `deflator_missing_omitted` `omitted_entity_years`, `{count, entities}` with the first ten entities, sorted, each `{ref, years}`, its `dcid:` ref and the runs of years its rows were left out (`1990-2001`, `2004`). Each message counts these and names none. `WarningModel` forbids unknown fields, so a revision without these fields cannot read a version recorded with them; a warning scoped to one rendering (`enrichment_unavailable`) rides its artifact entry instead of the top-level list | partial | `enrichment_unavailable` is emitted on the top-level `warnings` list; artifact entries carry no `warnings` member. Issue: none | | C78 | `artifacts` names exact bytes, the rendering that produced them, size, media type, creation time, tier, and `url`, the artifact's stable download link `/api/datasets/{id}/{content_version}/files/{artifact_digest}`, which never expires | implemented | | | C79 | A mint or definition retrieval returns the one artifact for its rendering (C165), plus its `series-statistics/1` artifact whenever `series_count` is present. A snapshot manifest lists every currently servable listable rendering, working and pinned alike, except an unpinned superseded bundle (C164), plus that same statistics artifact regardless of narrowing. An unnarrowed snapshot manifest also lists the version's canonical frame, rendering `{"format": "frame"}` and media type `application/vnd.apache.parquet`, after the requested renderings and before the statistics artifact. A frame1 document is never listed | implemented | | diff --git a/docs/conformance-search-api-v1.md b/docs/conformance-search-api-v1.md index 43362527..7b76b609 100644 --- a/docs/conformance-search-api-v1.md +++ b/docs/conformance-search-api-v1.md @@ -255,7 +255,7 @@ Gap rows state what is needed to close the clause and name the open issue that t | C107 | Named codes: date_coverage_partial, binding_folded, observations_omitted | partial | `date_coverage_partial` is minted when a unit conversion's tables cover only part of the served years (C178). `binding_folded` and `observations_omitted` are minted and reachable. The registry keeps `default_not_declared` active on search.resolve, and nothing emits it, because unit completion reads declared sources or refuses (C100). Issue: none | | C107a | Origin catalogue warnings on search.resolve: `un_system_unavailable` on a part the UN pass could change, one whose own pass ends no_data for a reason other than its entities or answers doubted (C51e), when the UN resolve or its shape derivation failed upstream, the two were not in within SEARCH_UN_FIRST_PASS_WAIT_S of the start of derivation, or the UN pass does not complete, and with SEARCH_UN_ON_FIRST_MENU on any part whose weak first pass waited for the UN leg and found it failed or not in, and on any part when the UN front door refuses places a read named, such as `country/GRC`, `country/SVN` or `country/ESH` (the catalogue's `un.json` lists them) or its part of a read that also names our variables fails while our part serves, including in a trailing observations or slot-options read, whose warning carries its spec_id in the done event, and whose `params` hold the refused place dcids as `places`, the failed reads' variables as `variables` and each variable's name as `variable_labels`, parallel to `variables`, taken from the members of the Specs the part holds with no graph read, and "" where it holds none (a member the graph names no label for, and every variable on search.options), so a client tells a refusal from a failed read and names the variables without the message, and in search.options diagnostics; `withheld_by_registry` on the request's first part that ends no_data with reason `no_observations` and gate `no_member_has_data_at_place` after electing a subject-only shape, naming the provenances the catalogue withholds that hold data for its elected StatVars at the places the part names (one unfiltered facet read per request at most); no other no-data reason triggers it; `origin_catalogue_stale` in diagnostics when the loaded catalogue is more than 14 days old, added after the response cache so a cached answer reports the catalogue's age when served | implemented | Search has no door that pins a variable dcid (a posted `stat_var` shape refuses), so the pinned-variable trigger applies to Datasets only. Pinned by `tests/engine/test_origin_wire.py` (`test_trailing_observations_report_places_the_un_refused`, `test_refused_places_alone_name_no_variable`) and, for a failed, slow or unreadable UN pool and the UN pass's failures, `tests/engine/test_pool_election.py` and `tests/engine/test_recall_transfer.py` (`test_a_un_resolve_that_never_answers_leaves_a_transferred_answer_intact`). | | C107b | Artifact warnings on search.resolve, one per response at most each: `artifact_pair_stale` in diagnostics when the bound origin catalogue and coverage builds record different `ingestion.workflow_execution_id` values (a build recording none never warns), naming both builds and their ingestions, added after the response cache, whose key holds both builds; `catalogue_index_drift` in diagnostics when, for a StatVar a definite or candidate Spec serves that the packaged code index lists and the catalogue's arc tables hold, the two disagree on the five-tuple, the constraint properties and their values, or the roles (`observationProperties`), naming each such StatVar with the aspects, added before the response cache stores the answer, whose key holds the catalogue build and the build commit that packages the code index. Each also logs one WARNING line, `artifact_pair_stale` or `catalogue_index_drift` followed by a JSON object. Neither fails the request or readiness. `ORIGIN_CATALOGUE_WARN_PAIR_STALE=false` and `ORIGIN_CATALOGUE_WARN_INDEX_DRIFT=false` silence the warning and keep the log line | implemented | Pinned by `tests/engine/test_artifact_checks.py`. | -| C149 | A warning carries `params` only for `observations_series_truncated` (C150), `elected_shape_no_data` (C151) and `un_system_unavailable` (C107a), holding the counts and labels its message states so a client composes its own text without parsing the message; every other code's warning omits the key | implemented | Pinned by the C150 and C151 tests. | +| C149 | A warning carries `params` only for `observations_series_truncated` (C150), `elected_shape_no_data` (C151), `un_system_unavailable` (C107a), `whole_total_available`, `currency_unmapped_omitted` and `deflator_missing_omitted`, holding the counts and labels its message states so a client composes its own text without parsing the message, and for the last two the first ten entities the message counts, with their labels (`""` where the conversion tables hold none, as for most entities no currency names) and, for `deflator_missing_omitted`, the years left out; every other code's warning omits the key | implemented | Pinned by the C150 and C151 tests. | | C150 | `observations_series_truncated` carries `params: {shown, total, order}`: the series the preview ships and the confirmed series behind them, both integers, `shown` < `total`, and `order`, `"ranked"` when the preview read every series and ships the most recent and largest (latest year, then latest value, then identity) or `"fixed"` when the observations the answer keeps in each member's years exceed `role_bulk_rank_max_rows` or the read converts into each entity's own currency and the preview ships a selection made before any value was read, StatVars newest year first and taken round-robin. The message states the same order. A fixed selection takes only series with data in their member's years, StatVars ordered by their newest year with data there. The observations themselves carry the same `order` whenever they show fewer series than the answer holds, and omit it otherwise. The Datasets observations door selects a snapshot share's series by the same rule over its frame's rows and states its own `order` on the observations it serves | implemented | Pinned by `tests/engine/test_bulk_role_series_observations.py`, `tests/engine/test_population_headline.py` and, for the door against the live read on each window, `tests/datasets_api/test_observations_door.py` (`test_the_door_and_the_live_role_bulk_preview_agree_on_each_series_axes`). | | C151 | A part carries at most one `elected_shape_no_data` per election, however many empty readings it dropped, with `params: {no_data, elected}`: the shape labels of the readings with no data and of the readings answered instead, each listed once in first-seen order. A single-sibling re-election names the elected shape and its sibling; a candidates answer of several siblings names the elected shape and each candidate's shape label. A same-shape merge folds its parts' warnings that share a message into one, whose `no_data` and `elected` are the first-seen unions over the parts | implemented | Pinned by `tests/engine/test_zero_observation_election.py`, `tests/engine/test_candidates_measure_sibling_collapse.py`, and `tests/engine/test_conjoin_warnings_and_primary.py`. | | C194 | A price-basis requirement the query names on `measurementQualifier` ("constant", "real", "at constant prices") is met by an answer served under a unit request whose price basis reads as one of the requirement's codes (`requirements._serves_price_basis`): constant prices meet `RealValue` and `InflationAdjusted`, current prices meet `Nominal`, and no unit request meets `PurchasingPowerParity`. A definite answer, or a candidate set every member of which is served so, carries no `query_qualifier_dropped` warning, `measure_not_applied` departure or sentence fact for it. The requirement still elects a publisher's series at that basis where one exists. An answer served under no unit request, or at the other basis, leaves it unmet, so "constant prices" over a series holding only current prices still warns | implemented | Pinned by `tests/engine/test_pinned_code_refusal.py`. | diff --git a/packages/dfo-data-tools/src/assembler/mint/__init__.py b/packages/dfo-data-tools/src/assembler/mint/__init__.py index 2a8c3323..ad99ff59 100644 --- a/packages/dfo-data-tools/src/assembler/mint/__init__.py +++ b/packages/dfo-data-tools/src/assembler/mint/__init__.py @@ -10,7 +10,7 @@ from assembler.mint.builders import complete_window, default_window from assembler.mint.card_description import DescribedRevision, with_card_description from assembler.mint.dispatch import MintedComponent, mint_component, stamp_envelope -from assembler.mint.provenance import DESCRIPTION_PROVENANCE_MEMBERS, warned_member_refs +from assembler.mint.provenance import DESCRIPTION_PROVENANCE_MEMBERS, warned_refs from assembler.mint.selection import selection_facts __all__ = [ @@ -22,6 +22,6 @@ "mint_component", "selection_facts", "stamp_envelope", - "warned_member_refs", + "warned_refs", "with_card_description", ] diff --git a/packages/dfo-data-tools/src/assembler/mint/provenance.py b/packages/dfo-data-tools/src/assembler/mint/provenance.py index 529601c3..f4fc7fa9 100644 --- a/packages/dfo-data-tools/src/assembler/mint/provenance.py +++ b/packages/dfo-data-tools/src/assembler/mint/provenance.py @@ -180,28 +180,58 @@ def _build_citation( return citation_line(retrieved=retrieved, sources=names, processed=processed)[:_CITATION_MAX] -# Each member field of a group's `coverage_below_scope` warning -# (`datasets_api.manifest_models.WarningModel`), and the words that lead its names on a card. -_MEMBER_FIELDS = ( - ("off_cadence_members", "At a different frequency"), - ("out_of_window_members", "No observations in the requested years"), +@dataclass(frozen=True) +class _RefField: + """A counted field of a manifest warning: its key, the words that lead its names on a + card, and the noun, one and many, that counts them when no name fits. A field `with_years` + is `{count, entities}`, each entity a `{ref, years}` (`datasets_api.manifest_models. + CountedEntityYearsModel`); any other is `{count, refs}` (`CountedRefsModel`).""" + + key: str + lead: str + one: str + many: str + with_years: bool = False + + +_REF_FIELDS = ( + _RefField("unserved_entities", "Not served", "entity", "entities"), + _RefField("off_cadence_members", "At a different frequency", "series", "series"), + _RefField( + "out_of_window_members", "No observations in the requested years", "series", "series" + ), + _RefField("omitted_entities", "Left out", "entity", "entities"), + _RefField("omitted_entity_years", "Left out", "entity", "entities", with_years=True), ) -def warned_member_refs(*, manifest: Mapping[str, object]) -> list[str]: - """The `dcid:` refs the member fields of `manifest`'s warnings name, in warning and field - order: the group members a card names by label (`_translate_warnings`).""" +def warned_refs(*, manifest: Mapping[str, object]) -> list[str]: + """The `dcid:` refs the counted fields of `manifest`'s warnings carry, in warning and + field order: the entities and group members a card names by label + (`_translate_warnings`).""" warnings = cast("Sequence[Mapping[str, object]]", manifest.get("warnings") or ()) return [ ref for warning in warnings - for field, _lead in _MEMBER_FIELDS - for ref in cast("Sequence[str]", _members(warning, field).get("refs", ())) + for field in _REF_FIELDS + for ref, _years in _named_items(_counted(warning, field), field) ] -def _members(warning: Mapping[str, object], field: str) -> Mapping[str, object]: - return cast("Mapping[str, object]", warning.get(field) or {}) +def _counted(warning: Mapping[str, object], field: _RefField) -> Mapping[str, object]: + return cast("Mapping[str, object]", warning.get(field.key) or {}) + + +def _named_items(counted: Mapping[str, object], field: _RefField) -> list[tuple[str, str]]: + """Each ref `counted` carries, with the text that follows its name on a card: its years + in brackets for a field `with_years`, `" (1990-2001, 2004)"`, else nothing.""" + if not field.with_years: + return [(ref, "") for ref in cast("Sequence[str]", counted.get("refs", ()))] + entities = cast("Sequence[Mapping[str, object]]", counted.get("entities", ())) + return [ + (cast(str, entity["ref"]), f" ({', '.join(cast('Sequence[str]', entity['years']))})") + for entity in entities + ] def _translate_warnings( @@ -209,9 +239,9 @@ def _translate_warnings( ) -> list[dict[str, object]]: """Maps manifest warnings to the envelope's closed `{kind, detail}` vocabulary. - A warning's `detail` is its message, then, for each member field it carries, the field's - lead words and its refs' `labels`, the bare dcid where `labels` holds none - (`_named_members`). The fields share the room the message leaves within + A warning's `detail` is its message, then, for each counted field it carries, the field's + lead words and its refs' `labels`, the bare dcid where `labels` holds none, each followed + by its years where the field holds them (`_named_refs`). The fields share the room the message leaves within `_WARNING_DETAIL_MAX` equally, and a field's unused share passes to the next. Drops `severity` and raises `AssertionError` for an unmapped code.""" translated: list[dict[str, object]] = [] @@ -224,33 +254,32 @@ def _translate_warnings( ) detail = str(warning["message"]) present = [ - (lead, members) - for field, lead in _MEMBER_FIELDS - if (members := _members(warning, field)) + (field, counted) for field in _REF_FIELDS if (counted := _counted(warning, field)) ] - for index, (lead, members) in enumerate(present): + for index, (field, counted) in enumerate(present): names = [ - labels.get(ref, ref.removeprefix(STAT_VAR_REF_PREFIX))[:LABEL_MAX] - for ref in cast("Sequence[str]", members["refs"]) + labels.get(ref, ref.removeprefix(STAT_VAR_REF_PREFIX))[:LABEL_MAX] + years + for ref, years in _named_items(counted, field) ] room = (_WARNING_DETAIL_MAX - len(detail)) // (len(present) - index) - detail += _named_members(lead, names, count=cast(int, members["count"]), budget=room) + detail += _named_refs(field, names, count=cast(int, counted["count"]), budget=room) translated.append( {"kind": _WARNING_KIND_BY_CODE[code], "detail": detail[:_WARNING_DETAIL_MAX]} ) return translated -def _named_members(lead: str, names: Sequence[str], *, count: int, budget: int) -> str: +def _named_refs(field: _RefField, names: Sequence[str], *, count: int, budget: int) -> str: """`" {lead}: "`, then as many whole `names` as fit within `budget` characters, with - `" and N more"` for the `count` members not named, closed by a full stop. When no name - fits, the count stands for them all, whatever the budget, so the field keeps its count.""" + `" and N more"` for the `count` not named, closed by a full stop. When no name fits, the + count stands for them all, whatever the budget, so the field keeps its count.""" + lead = field.lead for shown in range(len(names), 0, -1): hidden = count - shown text = f" {lead}: {', '.join(names[:shown])}{f' and {hidden} more' if hidden > 0 else ''}." if len(text) <= budget: return text - return f" {lead}: {count} series." + return f" {lead}: {count} {field.one if count == 1 else field.many}." @dataclass(frozen=True) diff --git a/packages/dfo-data-tools/src/materialise/codes.py b/packages/dfo-data-tools/src/materialise/codes.py index 392eecc9..010fc0f7 100644 --- a/packages/dfo-data-tools/src/materialise/codes.py +++ b/packages/dfo-data-tools/src/materialise/codes.py @@ -8,14 +8,31 @@ from dfo_platform.diagnostics import WarningSeverity -class UnservedMembers(TypedDict): - """The group members of one kind that served nothing: how many, and the `dcid:` refs of - the first five in request order.""" +class CountedRefs(TypedDict): + """The things of one kind a warning counts: how many, and the `dcid:` refs of the first + five, a group's members in request order and entities sorted, or of the first ten + entities a conversion left out.""" count: int refs: list[str] +class EntityYears(TypedDict): + """One entity a conversion left rows out of: its `dcid:` ref, and the years of those rows + as runs of consecutive years, `1990-2001` or `2004`.""" + + ref: str + years: list[str] + + +class CountedEntityYears(TypedDict): + """The entities a conversion left rows out of: how many, and the first ten, sorted, each + with its years.""" + + count: int + entities: list[EntityYears] + + class MaterialisationWarning(TypedDict): """One `{code, severity, message}` warning entry, as carried on `MaterialisationResult.warnings` (pipeline.py). `severity` is always one @@ -26,7 +43,12 @@ class MaterialisationWarning(TypedDict): A group's `coverage_below_scope` also carries `member_count`, the series the group requested, with `off_cadence_members`, those reporting the requested years at another frequency than the group serves, and `out_of_window_members`, those holding no observation - in the requested years, each present only when it names a member. + in the requested years, each present only when it names a member. `coverage_below_scope` + carries `unserved_entities`, the bound entities that served no observation, + `currency_unmapped_omitted` `omitted_entities`, the entities whose rows the conversion + left out, and `deflator_missing_omitted` `omitted_entity_years`, those entities with the + years of their rows. Each message counts what these fields hold + and names none of it, so a stored version holds no name that can go stale. This is a `TypedDict`. No shared platform pydantic warning model exists yet to alias. @@ -36,8 +58,11 @@ class MaterialisationWarning(TypedDict): severity: WarningSeverity message: str member_count: NotRequired[int] - off_cadence_members: NotRequired[UnservedMembers] - out_of_window_members: NotRequired[UnservedMembers] + off_cadence_members: NotRequired[CountedRefs] + out_of_window_members: NotRequired[CountedRefs] + unserved_entities: NotRequired[CountedRefs] + omitted_entities: NotRequired[CountedRefs] + omitted_entity_years: NotRequired[CountedEntityYears] MATERIALISE_CODE_CATALOGUE: CodeCatalogue = { diff --git a/packages/dfo-data-tools/src/materialise/finish.py b/packages/dfo-data-tools/src/materialise/finish.py index aa58e201..f444c073 100644 --- a/packages/dfo-data-tools/src/materialise/finish.py +++ b/packages/dfo-data-tools/src/materialise/finish.py @@ -15,7 +15,12 @@ # `pipeline` imports this module during initialisation. `MaterialisationResult` resolves when # `_finish_member` runs, after `pipeline` has loaded. from materialise import pipeline -from materialise.codes import MATERIALISE_CODE_CATALOGUE, MaterialisationWarning, UnservedMembers +from materialise.codes import ( + MATERIALISE_CODE_CATALOGUE, + CountedEntityYears, + CountedRefs, + MaterialisationWarning, +) from materialise.content_version import mint_version_from_digest from materialise.convert import ConversionRecord from materialise.dependencies import Dependencies @@ -385,18 +390,18 @@ def _build_warnings( ) -> tuple[MaterialisationWarning, ...]: """Adds `empty_result` for zero rows, which suppresses `coverage_below_scope`. Its message says the request's places hold no observations when the facet probe returned no candidate - (`_empty_result_warning`). `coverage_below_scope` covers window clamps and unserved bound - entities, naming up to five entities, a request binding no place that was served the - countries with data in place of every place, and, when the group served rows, its members - that served nothing, which it counts in its message and names by ref in its structured - fields: those at another observation period in the requested years + (`_empty_result_warning`). `coverage_below_scope` covers window clamps, unserved bound + entities, a request binding no place that was served the countries with data in place of + every place, and, when the group served rows, its members that served nothing. It counts + the entities and members in its message and names them by ref in its structured fields. + The members are those at another observation period in the requested years (`_GroupOutcome.off_cadence_members`), and those holding observations only outside them (`_GroupOutcome.out_of_window_members`). The second are left out with `absent_reads_zero`, when the served dataset reads an absent member as zero, since such a member adds nothing to the requested years. One `coverage_below_scope` states every cause that applies. `currency_unmapped_omitted` applies when any conversion record omitted an entity's rows; an omitted entity the request bound counts as reported there, so `coverage_below_scope` - does not name it. `deflator_missing_omitted` applies when any record left out rows for a + does not count it. `deflator_missing_omitted` applies when any record left out rows for a missing deflator. Values priced from a projected rate or deflator raise no warning: the trace's `conversion[].projected` counts them, and a description states them as an `estimated` caveat. @@ -502,8 +507,13 @@ def _empty_result_warning( return {**warning, "message": f"{warning['message']} {where}"} -# Maximum unserved dcids included in a warning. -_MAX_UNSERVED_NAMED = 5 +# Maximum refs a warning's counted field carries. A conversion's left-out entities keep the +# ten the warnings named in their message, `_MAX_TRACE_EXAMPLES`. +_MAX_COUNTED_REFS = 5 + +# Entities are bare dcids; a warning's counted field carries them as `dcid:` refs, as it +# carries a group's members. +_DCID_REF_PREFIX = "dcid:" def _coverage_below_scope_warning( @@ -514,21 +524,24 @@ def _coverage_below_scope_warning( out_of_window: Sequence[str], member_count: int, ) -> MaterialisationWarning: - """Adds up to five unserved entity dcids to the catalogue detail. Then, of the group's + """Counts the `unserved` entity dcids after the catalogue detail. Then, of the group's `member_count` series, it counts those in `off_cadence`, which report the requested years at another frequency than the group serves and are left out, and those in `out_of_window`, which hold no observation in the requested years. The message names no - member, so a stored version holds no name that can go stale. The warning's - `off_cadence_members` and `out_of_window_members` carry each count and the first five - refs (`UnservedMembers`), beside `member_count`, for a reader to name them. Then, with - `countries_fallback`, it says a request binding no place was served the countries with - data because the full list of places with data was too large or unavailable - (`place_scope.DefaultPlaceScope`). Without any the detail stands for a pure window - clamp.""" + entity or member, so a stored version holds no name that can go stale. The warning's + `unserved_entities`, `off_cadence_members` and `out_of_window_members` carry each count + and the first five refs (`CountedRefs`), beside `member_count` for the members, for a + reader to name them. Then, with `countries_fallback`, it says a request binding no place + was served the countries with data because the full list of places with data was too + large or unavailable (`place_scope.DefaultPlaceScope`). Without any the detail stands for a + pure window clamp.""" _title, detail = MATERIALISE_CODE_CATALOGUE["coverage_below_scope"] message = detail if unserved: - message = f"{message} No observations were served for {_capped_names(unserved)}." + message = ( + f"{message} No observations were served for {len(unserved)} requested " + f"{_entities(len(unserved))}." + ) if off_cadence: message = ( f"{message} {len(off_cadence)} of {member_count} series report at a different " @@ -549,24 +562,57 @@ def _coverage_below_scope_warning( "severity": "warn", "message": message, } + if unserved: + warning["unserved_entities"] = _counted_entities(unserved) if off_cadence or out_of_window: warning["member_count"] = member_count if off_cadence: - warning["off_cadence_members"] = _unserved(off_cadence) + warning["off_cadence_members"] = _counted(off_cadence) if out_of_window: - warning["out_of_window_members"] = _unserved(out_of_window) + warning["out_of_window_members"] = _counted(out_of_window) return warning -def _unserved(refs: Sequence[str]) -> UnservedMembers: - return {"count": len(refs), "refs": list(refs[:_MAX_UNSERVED_NAMED])} +def _counted(refs: Sequence[str], *, limit: int = _MAX_COUNTED_REFS) -> CountedRefs: + return {"count": len(refs), "refs": list(refs[:limit])} + +def _counted_entities(entities: Iterable[str], *, limit: int = _MAX_COUNTED_REFS) -> CountedRefs: + """The distinct bare `entities`, sorted, as `dcid:` refs, the first `limit` of them + carried (`_counted`).""" + return _counted( + [f"{_DCID_REF_PREFIX}{entity}" for entity in sorted(set(entities))], limit=limit + ) + + +def _counted_entity_years(omitted: Iterable[tuple[str, int]]) -> CountedEntityYears: + """The distinct bare entities of `omitted`, counted, and the first `_MAX_TRACE_EXAMPLES`, + sorted, as `dcid:` refs each with its years (`_year_spans`).""" + by_entity: dict[str, set[int]] = {} + for entity, year in omitted: + by_entity.setdefault(entity, set()).add(year) + return { + "count": len(by_entity), + "entities": [ + {"ref": f"{_DCID_REF_PREFIX}{entity}", "years": _year_spans(by_entity[entity])} + for entity in sorted(by_entity)[:_MAX_TRACE_EXAMPLES] + ], + } + + +def _year_spans(years: Iterable[int]) -> list[str]: + """`years` as runs of consecutive years, `["1990-2001", "2004"]`.""" + runs: list[list[int]] = [] + for year in sorted(set(years)): + if runs and year == runs[-1][1] + 1: + runs[-1][1] = year + else: + runs.append([year, year]) + return [str(first) if first == last else f"{first}-{last}" for first, last in runs] -def _capped_names(names: Sequence[str]) -> str: - """The first five of `names`, comma-joined, with a count of the rest.""" - named = ", ".join(names[:_MAX_UNSERVED_NAMED]) - remaining = len(names) - _MAX_UNSERVED_NAMED - return f"{named} and {remaining} more" if remaining > 0 else named + +def _entities(count: int) -> str: + return "entity" if count == 1 else "entities" class _ProjectedConversion(Protocol): @@ -595,42 +641,32 @@ def deflator_projected_warning( def currency_unmapped_omitted_warning(entities: Iterable[str]) -> MaterialisationWarning: - """Reports how many distinct entities a conversion into each entity's own currency left - out, naming them sorted and capped as `deflator_projected_warning` caps its examples.""" + """Counts the distinct entities a conversion into each entity's own currency left out. + `omitted_entities` carries the first `_MAX_TRACE_EXAMPLES`, sorted (`_counted_entities`).""" _title, detail = MATERIALISE_CODE_CATALOGUE["currency_unmapped_omitted"] - distinct = set(entities) - named, truncated = _capped_distinct_examples(distinct) - truncated_note = " (truncated)" if truncated else "" - message = f"{detail} {len(distinct)} entities: {', '.join(named)}{truncated_note}" - return {"code": "currency_unmapped_omitted", "severity": "warn", "message": message} + counted = _counted_entities(entities, limit=_MAX_TRACE_EXAMPLES) + return { + "code": "currency_unmapped_omitted", + "severity": "warn", + "message": f"{detail} It left out {counted['count']} {_entities(counted['count'])}.", + "omitted_entities": counted, + } def deflator_missing_omitted_warning(omitted: Iterable[tuple[str, int]]) -> MaterialisationWarning: - """Reports how many distinct entities a constant-price conversion left rows out of for a - missing deflator, naming each as `entity` with its omitted years as spans - (`entity (1990-2001, 2004)`), sorted and capped as `deflator_projected_warning` caps its - examples.""" + """Counts the distinct entities a constant-price conversion left rows out of for a + missing deflator. `omitted_entity_years` carries the first `_MAX_TRACE_EXAMPLES`, sorted, + each with the years of those rows (`_counted_entity_years`).""" _title, detail = MATERIALISE_CODE_CATALOGUE["deflator_missing_omitted"] - by_entity: dict[str, set[int]] = {} - for entity, year in omitted: - by_entity.setdefault(entity, set()).add(year) - named, truncated = _capped_distinct_examples( - f"{entity} ({_year_spans(years)})" for entity, years in by_entity.items() - ) - truncated_note = " (truncated)" if truncated else "" - message = f"{detail} {len(by_entity)} entities: {', '.join(named)}{truncated_note}" - return {"code": "deflator_missing_omitted", "severity": "warn", "message": message} - - -def _year_spans(years: Iterable[int]) -> str: - """`years` as runs of consecutive years, `1990-2001, 2004`.""" - runs: list[list[int]] = [] - for year in sorted(set(years)): - if runs and year == runs[-1][1] + 1: - runs[-1][1] = year - else: - runs.append([year, year]) - return ", ".join(str(first) if first == last else f"{first}-{last}" for first, last in runs) + counted = _counted_entity_years(omitted) + return { + "code": "deflator_missing_omitted", + "severity": "warn", + "message": ( + f"{detail} It left out rows of {counted['count']} {_entities(counted['count'])}." + ), + "omitted_entity_years": counted, + } def _plain_role_bindings(role_bindings: Mapping[str, EntityFilter]) -> dict[str, Sequence[str]]: diff --git a/src/datasets_api/manifest.py b/src/datasets_api/manifest.py index c03c07d6..e99c6650 100644 --- a/src/datasets_api/manifest.py +++ b/src/datasets_api/manifest.py @@ -126,9 +126,8 @@ def warning_model(warning: Mapping[str, object]) -> WarningModel: - """Converts a complete `{code, severity, message}` warning mapping, with any member fields - a group's `coverage_below_scope` carries (`materialise.codes.MaterialisationWarning`), - without reformatting it.""" + """Converts a complete `{code, severity, message}` warning mapping, with any structured + fields it carries (`materialise.codes.MaterialisationWarning`), without reformatting it.""" return WarningModel.model_validate(warning) diff --git a/src/datasets_api/warnings.py b/src/datasets_api/warnings.py index faa3f952..a73f5650 100644 --- a/src/datasets_api/warnings.py +++ b/src/datasets_api/warnings.py @@ -45,14 +45,15 @@ "currency_unmapped_omitted": ( "Currency Unmapped Omitted", "A conversion into each entity's own currency left out the rows of the entities " - "that no single currency names through nationalCurrencyOf; the warning names them.", + "that no single currency names through nationalCurrencyOf; the warning counts them " + "and carries the refs of the first ten.", "warn", ), "deflator_missing_omitted": ( "Deflator Missing Omitted", "A constant-price conversion left out each entity's rows in the years the deflator " "table holds no value for it, and every row of an entity it holds no value for in " - "the base year; the warning names them.", + "the base year; the warning counts them and carries the refs of the first ten.", "warn", ), "stat_vars_summed": ( diff --git a/src/presentation_store/frame_store.py b/src/presentation_store/frame_store.py index c77038af..7d47efa4 100644 --- a/src/presentation_store/frame_store.py +++ b/src/presentation_store/frame_store.py @@ -59,9 +59,10 @@ record not final, so a later load reads the inputs again. A failed input read raises as a failed record read does. -A version's `coverage_below_scope` warning names the group members that served nothing by -ref (`assembler.mint.warned_member_refs`). They hold no row, so their refs join the frame's -label read ahead of its own refs, and a card names them by label. +A version's warnings carry by ref the entities and group members they count and their +messages do not name: the bound entities and members that served nothing, and the entities a +conversion left out (`assembler.mint.warned_refs`). These hold no row, so their refs join the +frame's label read ahead of its own refs, and a card names them by label. A frame's continuation (`ResolvedFrame.continuation`, `assembler.frame_view.frame_continuation`) comes from its stored version alone: a dataset version's `trace.continuations`, or the @@ -158,7 +159,7 @@ frame_continuation, ) from assembler.limits import LABEL_MAX, joined_label -from assembler.mint import warned_member_refs +from assembler.mint import warned_refs from assembler.ports import ( FrameRef, FrameUnavailable, @@ -778,7 +779,7 @@ def _resolve( record=record, composed=_composed(record, stored=stored), inputs=inputs, - warned=warned_member_refs(manifest=document), + warned=warned_refs(manifest=document), deadline=deadline, ) continuation = frame_continuation(manifest=document) @@ -843,7 +844,7 @@ def _resolve( record=record, composed=_composed(record, stored=stored), inputs=inputs, - warned=warned_member_refs(manifest=document), + warned=warned_refs(manifest=document), deadline=deadline, ) if record_changed and not labels_resolved: @@ -1381,10 +1382,10 @@ def _resolve_labels( warned: Sequence[str], deadline: float, ) -> tuple[dict[str, str], dict[str, str], bool]: - """Resolve the frame's ref labels (`_resolve_ref_labels`), with the group members its - version's warnings name (`warned`, `assembler.mint.warned_member_refs`) under their - `dcid:` refs, since a member that served nothing has no row, and the refs each of - `inputs` names in its title (`datasets_api.description_record.title_named`) and the + """Resolve the frame's ref labels (`_resolve_ref_labels`), with the entities and group + members its version's warnings count (`warned`, `assembler.mint.warned_refs`) under + their `dcid:` refs, since what served nothing or was left out has no row, and the refs + each of `inputs` names in its title (`datasets_api.description_record.title_named`) and the entities `record`'s steps record on their sides (`bound_entities`) under their bare dcids, then add the name of each variable `record` and `composed` name (`_computed_labels`), under the ref its `stat_var` cells hold, and read the description diff --git a/src/search/engine/region/candidate_collapse.py b/src/search/engine/region/candidate_collapse.py index 72270d94..fa3cd44d 100644 --- a/src/search/engine/region/candidate_collapse.py +++ b/src/search/engine/region/candidate_collapse.py @@ -94,6 +94,7 @@ from search.engine.retrieval.standard import Narrow from search.engine.unit.bind import with_unit_lexicon_version from search.engine.unit.conversion import served_prices +from search.engine.unit.conversion_catalogue import current_conversion_catalogue from search.models import ( CandidateBasis, IdentityUnitRequest, @@ -145,7 +146,10 @@ def _apply_bound_unit_filter( coverage = grounded.cand_unit_coverage.get(s.spec_id, UnitCoverage()) if coverage.refusal is None: _matched_specs.append(s) - if (omitted := coverage.omitted_warning(spec_id=s.spec_id)) is not None: + omitted = coverage.omitted_warning( + catalogue=current_conversion_catalogue(), spec_id=s.spec_id + ) + if omitted is not None: grounded.cand_unit_omitted[s.spec_id] = omitted continue sv, exc = coverage.refusal diff --git a/src/search/engine/region/convertibility.py b/src/search/engine/region/convertibility.py index 649555d1..7c54dcf4 100644 --- a/src/search/engine/region/convertibility.py +++ b/src/search/engine/region/convertibility.py @@ -561,18 +561,23 @@ def omitted_entities(self) -> tuple[str, ...]: """The entities, sorted, whose rows `omitted` names.""" return tuple(sorted({entity for _sv, _role, entity in self.omitted})) - def omitted_warning(self, *, spec_id: str | None = None) -> Warning | None: - """The `currency_unmapped_omitted` warning naming `omitted_entities`, the one a read + def omitted_warning( + self, *, catalogue: ConversionCatalogue | None, spec_id: str | None = None + ) -> Warning | None: + """The `currency_unmapped_omitted` warning counting `omitted_entities`, the one a read of the same rows carries (`materialise.finish.currency_unmapped_omitted_warning`), + with the entities it carries as `params` (`_omitted_params`) named from `catalogue`, or None when no entity was omitted.""" if not self.omitted_entities: return None built = currency_unmapped_omitted_warning(self.omitted_entities) + counted = built["omitted_entities"] return Warning( code=CURRENCY_UNMAPPED_OMITTED, severity=built["severity"], message=built["message"], spec_id=spec_id, + params=_omitted_params(counted["count"], counted["refs"], catalogue=catalogue), ) @@ -1146,10 +1151,11 @@ def deflator_omission_departure( deflator the tables lack in some served years, or in the base year (`download.deflator_omissions`, over the rows the answer's counts read). The fact names the entities that lose the most years first, by the name the catalogue holds for each - (`ConversionCatalogue.entity_names`), and counts those it holds no name for. When the - conversion cannot be read in memory (`unit.conversion.counted_conversion` is - `UNDECIDED`), a constant-price answer warns that rows may be left out and departs - naming no entity, since the mint still leaves out the rows its deflators lack. None + (`ConversionCatalogue.entity_names`), and counts those it holds no name for. The + warning's `params` carry the entities it counts, named from the same catalogue, with + their years (`_omitted_params`). When the conversion cannot be read in memory + (`unit.conversion.counted_conversion` is `UNDECIDED`), a constant-price answer warns, + with no `params`, that rows may be left out and departs naming no entity, since the mint still leaves out the rows its deflators lack. None when no row is left out or at current prices. Every door reads it after the period a unit request shows (`unit_request_period`).""" if unit_request.prices != "constant": @@ -1176,6 +1182,7 @@ def deflator_omission_departure( built = deflator_missing_omitted_warning( (entity, year) for entity, years in omitted.items() for year in years.lost ) + counted = built["omitted_entity_years"] ranked = sorted(omitted.items(), key=lambda item: (-len(item[1].lost), item[0])) names = catalogue.entity_names fact = deflator_omission_fact( @@ -1187,11 +1194,39 @@ def deflator_omission_departure( unnamed=sum(1 for entity in omitted if entity not in names), ) warning = Warning( - code=DEFLATOR_MISSING_OMITTED, severity=built["severity"], message=built["message"] + code=DEFLATOR_MISSING_OMITTED, + severity=built["severity"], + message=built["message"], + params={ + **_omitted_params( + counted["count"], + [entity["ref"] for entity in counted["entities"]], + catalogue=catalogue, + ), + "entity_years": [", ".join(entity["years"]) for entity in counted["entities"]], + }, ) return warning, fact +def _omitted_params( + count: int, refs: Sequence[str], *, catalogue: ConversionCatalogue | None +) -> dict[str, int | str | list[str]]: + """The `params` of a conversion's omission warning: `count`, the entities it counts; + `entities`, the bare dcids of `refs`, the `dcid:` refs of the first ten it carries; and + `entity_labels`, each one's name in `catalogue` (`ConversionCatalogue.entity_names`), + `""` where it holds none, in `entities` order. The catalogue names deflator-table + entities and places with a national currency, so an entity no currency names, such as + a regional aggregate, usually has `""`; its dcid stays in `entities`.""" + names = catalogue.entity_names if catalogue is not None else {} + entities = [ref.removeprefix(_STAT_VAR_REF_PREFIX) for ref in refs] + return { + "count": count, + "entities": entities, + "entity_labels": [names.get(entity, "") for entity in entities], + } + + def _reads_tables(selections: Mapping[str, FacetSelection] | None) -> bool: """Whether rows served under `selections` (`facet_selections`) read conversion tables: a request-form unit other than a current-price request for the facet's own currency, diff --git a/src/search/engine/region/phase_answer.py b/src/search/engine/region/phase_answer.py index aa48a588..6bc2a4e7 100644 --- a/src/search/engine/region/phase_answer.py +++ b/src/search/engine/region/phase_answer.py @@ -768,7 +768,9 @@ def _bound_unit_refusal( # Grounding ran the conversion on every path that reaches here (`_runs_unit_conversion`). coverage = state.unit_coverage if coverage.refusal is None: - if (omitted := coverage.omitted_warning()) is not None: + if ( + omitted := coverage.omitted_warning(catalogue=current_conversion_catalogue()) + ) is not None: state.warnings.append(omitted) return None # A bound outcome guarantees the unit request. diff --git a/src/search/engine/region/promote_finalize.py b/src/search/engine/region/promote_finalize.py index 248690ac..b4e690e3 100644 --- a/src/search/engine/region/promote_finalize.py +++ b/src/search/engine/region/promote_finalize.py @@ -31,6 +31,7 @@ from search.engine.retrieval.member_facts import ArcsMemberFacts from search.engine.retrieval.read_plan import plan_window_gap, serves_nothing from search.engine.retrieval.retrieve import Materialised +from search.engine.unit.conversion_catalogue import current_conversion_catalogue from search.models import ( IdentityFacetValueRequest, IdentityUnitRequest, @@ -121,7 +122,9 @@ def _bound_unit_refusal( ) coverage = state.unit_coverage if coverage.refusal is None: - if (omitted := coverage.omitted_warning()) is not None: + if ( + omitted := coverage.omitted_warning(catalogue=current_conversion_catalogue()) + ) is not None: state.warnings.append(omitted) return None exc = coverage.refusal[1] diff --git a/src/search/models/envelope.py b/src/search/models/envelope.py index 65f95473..836cd28b 100644 --- a/src/search/models/envelope.py +++ b/src/search/models/envelope.py @@ -35,8 +35,9 @@ def _is_none(value: object) -> bool: class Warning(StrictModel): """A run-health signal. Note: this name shadows the Python builtin Warning. - Only `observations_series_truncated`, `elected_shape_no_data`, `un_system_unavailable` - and `whole_total_available` set `params`. A `None` `params` is left out of every dump, so other codes' warnings stay + Only `observations_series_truncated`, `elected_shape_no_data`, `un_system_unavailable`, + `whole_total_available`, `currency_unmapped_omitted` and `deflator_missing_omitted` set + `params`. A `None` `params` is left out of every dump, so other codes' warnings stay exactly `{code, severity, message}` plus `spec_id` where one applies. """ @@ -73,7 +74,15 @@ class Warning(StrictModel): "unavailable. whole_total_available: {total, total_label, whole_of}, the dcid " "and label of the total the source publishes for the side the answer lists " "member by member, and the classes of party it covers, comma-joined (official, " - "private, residual). Omitted for every other code" + "private, residual). currency_unmapped_omitted: {count, entities, entity_labels}, " + "how many entities the conversion left out, the dcids of the first ten, sorted, " + "and each one's name, in entities order, '' where the conversion tables hold " + "none, which is usual here, since an entity no currency names, such as a " + "regional aggregate, is rarely in those tables; its dcid stays in entities. " + "deflator_missing_omitted: the same with entity_years, the years each " + "entity's rows were left out, as runs of consecutive years ('1990-2001, 2004'), " + "in entities order; a warning without params says the loaded tables cannot tell " + "which entities. Omitted for every other code" ), ) diff --git a/tests/assembler/test_mint.py b/tests/assembler/test_mint.py index 79f6a519..7288adc6 100644 --- a/tests/assembler/test_mint.py +++ b/tests/assembler/test_mint.py @@ -4,14 +4,19 @@ import dataclasses import datetime +from typing import get_args import pytest from assembler.bind import BoundSeries, bind_all from assembler.errors import BindingInvalid, SchemaInvalid from assembler.limits import LABEL_MAX -from assembler.mint import default_window, mint_component, stamp_envelope, warned_member_refs -from assembler.mint.provenance import _WARNING_KIND_BY_CODE, DESCRIPTION_PROVENANCE_MEMBERS +from assembler.mint import default_window, mint_component, stamp_envelope, warned_refs +from assembler.mint.provenance import ( + _REF_FIELDS, + _WARNING_KIND_BY_CODE, + DESCRIPTION_PROVENANCE_MEMBERS, +) from assembler.ports import FrameRef from assembler.resolve import ResolvedSeries, resolve_selectors from assembler.select import normalize_select @@ -22,6 +27,7 @@ malaria_ethiopia_resolved_frame, malaria_ethiopia_rows, ) +from datasets_api.manifest_models import CountedEntityYearsModel, CountedRefsModel, WarningModel from datasets_api.warnings import DATASETS_WARNING_CATALOGUE from presentation_api.refusals import binding_invalid_spec @@ -1839,6 +1845,29 @@ def _long_label(index: int) -> str: "No observations in the requested years: 6 series.", id="both-long", ), + # Entities lead the members, as the message counts them first, and a field fitting + # no name counts them in its own noun. + pytest.param( + { + "unserved_entities": {"count": 1, "refs": ["dcid:country/KEN"]}, + "off_cadence_members": {"count": 1, "refs": ["dcid:Health"]}, + }, + "Short of scope. Not served: Kenya. At a different frequency: Basic health.", + id="unserved-entities", + ), + pytest.param( + { + "unserved_entities": {"count": 2, "refs": ["dcid:Long0", "dcid:Long1"]}, + "off_cadence_members": { + "count": 6, + "refs": [f"dcid:Long{i}" for i in range(2, 7)], + }, + "out_of_window_members": {"count": 1, "refs": ["dcid:Long7"]}, + }, + "Short of scope. Not served: 2 entities. At a different frequency: 6 series. " + f"No observations in the requested years: {_long_label(7)}.", + id="entities-long", + ), ], ) def test_a_coverage_warning_names_its_members_by_label(release, members, expected): @@ -1852,17 +1881,74 @@ def test_a_coverage_warning_names_its_members_by_label(release, members, expecte **members, } ], - labels={"dcid:Health": "Basic health", "dcid:Water": "Water", **_LONG}, + labels={ + "dcid:Health": "Basic health", + "dcid:Water": "Water", + "dcid:country/KEN": "Kenya", + **_LONG, + }, ) provenance = _mint_line_chart_provenance(resolved_frame=resolved_frame, release=release) assert provenance["warnings"] == [{"kind": "coverage_gap", "detail": expected}] - assert warned_member_refs(manifest=resolved_frame.manifest) == [ + assert warned_refs(manifest=resolved_frame.manifest) == [ ref for field in members.values() for ref in field["refs"] ] +@pytest.mark.parametrize( + ("code", "field", "named"), + [ + pytest.param( + "currency_unmapped_omitted", + {"omitted_entities": {"count": 3, "refs": ["dcid:country/KEN", "dcid:XYZ"]}}, + "Kenya, XYZ", + id="currency", + ), + # Each entity's name carries the years its rows were left out. + pytest.param( + "deflator_missing_omitted", + { + "omitted_entity_years": { + "count": 3, + "entities": [ + {"ref": "dcid:country/KEN", "years": ["1990-2001", "2004"]}, + {"ref": "dcid:XYZ", "years": ["2020"]}, + ], + } + }, + "Kenya (1990-2001, 2004), XYZ (2020)", + id="deflator", + ), + ], +) +def test_a_conversion_warning_names_the_entities_it_left_out_by_label(release, code, field, named): + resolved_frame = _resolved_frame_with_warnings( + [{"code": code, "severity": "warn", "message": "It left out 3 entities.", **field}], + labels={"dcid:country/KEN": "Kenya"}, + ) + + provenance = _mint_line_chart_provenance(resolved_frame=resolved_frame, release=release) + + assert provenance["warnings"] == [ + {"kind": "coverage_gap", "detail": f"It left out 3 entities. Left out: {named} and 1 more."} + ] + assert warned_refs(manifest=resolved_frame.manifest) == ["dcid:country/KEN", "dcid:XYZ"] + + +def test_a_card_names_every_counted_field_a_stored_warning_holds(): + """Each `WarningModel` field holding counted refs has a `_REF_FIELDS` row, so the frame + store labels its refs and a card names them; a field without one would go unnamed.""" + counted = { + name + for name, field in WarningModel.model_fields.items() + if {CountedRefsModel, CountedEntityYearsModel} & set(get_args(field.annotation)) + } + + assert counted == {field.key for field in _REF_FIELDS} + + def test_provenance_warnings_refuses_a_code_absent_from_the_translation_table(release): """Unknown manifest codes raise `AssertionError` before minting an invalid envelope.""" resolved_frame = _resolved_frame_with_warnings( diff --git a/tests/engine/test_currency_unmapped_omitted_gate.py b/tests/engine/test_currency_unmapped_omitted_gate.py index a4e2f15a..da0e6e3d 100644 --- a/tests/engine/test_currency_unmapped_omitted_gate.py +++ b/tests/engine/test_currency_unmapped_omitted_gate.py @@ -133,7 +133,7 @@ def test_a_passing_gate_that_omits_entities_warns_once(monkeypatch, module, gate assert result is None assert [(w.code, w.severity) for w in warnings] == [("currency_unmapped_omitted", "warn")] - assert warnings[0].message.endswith("2 entities: africaRegion, southOfSahara") + assert warnings[0].message.endswith("It left out 2 entities.") def test_a_bulk_latest_gate_reads_the_plan_its_read_serves(monkeypatch, search_settings): diff --git a/tests/engine/test_served_slot_options.py b/tests/engine/test_served_slot_options.py index 8436f44b..6a2f6732 100644 --- a/tests/engine/test_served_slot_options.py +++ b/tests/engine/test_served_slot_options.py @@ -1416,19 +1416,26 @@ def test_a_series_two_facets_serve_in_different_years_counts_an_upper_bound(monk @pytest.mark.parametrize( - ("names", "named", "unnamed"), + ("names", "named", "unnamed", "label"), [ - pytest.param(_CATALOGUE.entity_names, (("Germany", (2019,), (2020,)),), 0, id="named"), - pytest.param({}, (), 1, id="no-name-held-counts-it"), + pytest.param( + _CATALOGUE.entity_names, + (("Germany", (2019,), (2020,)),), + 0, + "Germany", + id="named", + ), + pytest.param({}, (), 1, "", id="no-name-held-counts-it"), ], ) def test_a_constant_price_answer_names_the_donor_years_its_deflators_leave_out( - monkeypatch, names, named, unnamed + monkeypatch, names, named, unnamed, label ): """Germany's deflator lacks 2019 and France's holds it, so the answer keeps 2019 for France, leaves out Germany's 2019 row, and departs on the unit token naming Germany - with the year it loses, or counting it when the catalogue holds no name for it. Its - download counts the three rows its conversion keeps.""" + with the year it loses, or counting it when the catalogue holds no name for it. The + warning's params carry Germany with that name and year. Its download counts the three + rows its conversion keeps.""" region = _crs_region( monkeypatch, bound={}, @@ -1440,7 +1447,17 @@ def test_a_constant_price_answer_names_the_donor_years_its_deflators_leave_out( ) assert region.status == "definite", region.no_data_reason - assert [w.code for w in region.warnings] == ["deflator_missing_omitted"] + assert [(w.code, w.params) for w in region.warnings] == [ + ( + "deflator_missing_omitted", + { + "count": 1, + "entities": [_DEU], + "entity_labels": [label], + "entity_years": ["2019"], + }, + ) + ] assert [ (fact.binding_ref, fact.omitted_years, fact.omitted_unnamed) for fact in region.departure_facts.values() @@ -1468,7 +1485,7 @@ def test_a_constant_price_answer_the_tables_cannot_count_warns_rows_may_be_left_ ) assert region.status == "definite", region.no_data_reason - assert [w.code for w in region.warnings] == ["deflator_missing_omitted"] + assert [(w.code, w.params) for w in region.warnings] == [("deflator_missing_omitted", None)] assert [ (fact.binding_ref, fact.omitted_years, fact.omission_undecided) for fact in region.departure_facts.values() diff --git a/tests/engine/test_unit_coverage.py b/tests/engine/test_unit_coverage.py index df0821ec..319df49e 100644 --- a/tests/engine/test_unit_coverage.py +++ b/tests/engine/test_unit_coverage.py @@ -10,6 +10,7 @@ from collections.abc import Mapping from dataclasses import dataclass, replace from functools import partial +from types import MappingProxyType import httpx import pytest @@ -43,6 +44,7 @@ from search.engine.retrieval.read_plan import BulkPlan, PairPlan, PlanSeries, PlanYears, ReadPlan from search.engine.retrieval.retrieve import Materialised from search.engine.retrieval.role_resolve import _role_plan +from search.engine.unit.conversion_catalogue import ConversionCatalogue from search.models import CoverageBare, CoverageExact, TimeWindow from series_coverage.index import CoverageIndex from series_coverage.schema import FacetMetadata, RoleNames @@ -206,33 +208,57 @@ def _destination_coverage(plan) -> UnitCoverage: _KES_RATES = {**_RATES, "CurrencyKES": {2020: 100.0, 2021: 110.0}} _TWELVE_REGIONS = tuple(f"region/R{i:02d}" for i in range(12)) +# A catalogue holding a name for one omitted region only. +_NAMING_CATALOGUE = ConversionCatalogue( + recipes=MappingProxyType({}), + tables=MappingProxyType({}), + national_currencies=MappingProxyType({}), + default_base_year=2024, + entity_names=MappingProxyType({"africaRegion": "Africa"}), +) @pytest.mark.parametrize( - ("regions", "named"), + ("regions", "named", "params"), [ - pytest.param(_REGIONS, "2 entities: africaRegion, southOfSahara", id="named"), + pytest.param( + _REGIONS, + "It left out 2 entities.", + { + "count": 2, + "entities": list(_REGIONS), + "entity_labels": ["Africa", ""], + }, + id="two", + ), pytest.param( _TWELVE_REGIONS, - "12 entities: " + ", ".join(_TWELVE_REGIONS[:10]) + " (truncated)", - id="truncated", + "It left out 12 entities.", + { + "count": 12, + "entities": list(_TWELVE_REGIONS[:10]), + "entity_labels": [""] * 10, + }, + id="twelve", ), ], ) -def test_regional_recipients_under_a_destination_currency_are_omitted_and_named( - regions: tuple[str, ...], named: str +def test_regional_recipients_under_a_destination_currency_are_omitted_and_counted( + regions: tuple[str, ...], named: str, params: dict[str, object] ): """Regional recipients beside Kenya: no region has a national currency, so their rows are - omitted, Kenya's convert, and the answer's warning names every region.""" + omitted, Kenya's convert, and the answer's warning counts every region. Its params carry + the first ten with the names the conversion catalogue holds.""" coverage = _destination_coverage(_pair_plan(_KEN, *regions)) assert coverage.refusal is None assert coverage.omitted_entities == tuple(sorted(regions)) - warning = coverage.omitted_warning(spec_id="spec-1") + warning = coverage.omitted_warning(catalogue=_NAMING_CATALOGUE, spec_id="spec-1") assert warning is not None - assert (warning.code, warning.severity, warning.spec_id) == ( + assert (warning.code, warning.severity, warning.spec_id, warning.params) == ( "currency_unmapped_omitted", "warn", "spec-1", + params, ) assert warning.message.endswith(named) @@ -248,7 +274,7 @@ def test_only_regional_recipients_under_a_destination_currency_refuse_naming_eac _REGIONS, False, ) - assert coverage.omitted_warning() is None + assert coverage.omitted_warning(catalogue=None) is None def test_regional_recipients_beside_rows_no_conversion_reads_are_omitted_without_refusing(): @@ -382,7 +408,7 @@ def plan_of(*of: str) -> ReadPlan: preview = _preview_of( tuple((_DONOR, r) for r in recipients), recipient_years=years, window=window ) - search = _destination_coverage(plan_of(*recipients)).omitted_warning() + search = _destination_coverage(plan_of(*recipients)).omitted_warning(catalogue=None) assert search is not None (datasets,) = [w for w in preview.warnings if w["code"] == "currency_unmapped_omitted"] assert (datasets["severity"], datasets["message"]) == (search.severity, search.message) diff --git a/tests/materialise/test_conversion_finish.py b/tests/materialise/test_conversion_finish.py index 99f79dc3..cf80c6ed 100644 --- a/tests/materialise/test_conversion_finish.py +++ b/tests/materialise/test_conversion_finish.py @@ -621,9 +621,9 @@ def test_a_year_missing_its_deflator_is_left_out_and_named_in_the_manifest(): assert result.coverage.rows == 1 (warning,) = [w for w in result.warnings if w["code"] == "deflator_missing_omitted"] - assert (warning["severity"], warning["message"].endswith("1 entities: country/FRA (2023)")) == ( + assert (warning["severity"], warning["omitted_entity_years"]) == ( "warn", - True, + {"count": 1, "entities": [{"ref": "dcid:country/FRA", "years": ["2023"]}]}, ) # The trace counts what the conversion left out, which the description's step states. (conversion,) = result.trace.conversion or () @@ -850,29 +850,32 @@ def _served_recipients(result: MaterialisationResult | PreviewResult) -> set[str @pytest.mark.parametrize("entry", _ENTRIES) @pytest.mark.parametrize( - ("regions", "named"), + ("regions", "counted"), [ pytest.param( - ["africaRegion", "southOfSahara"], "2 entities: africaRegion, southOfSahara", id="named" + ["southOfSahara", "africaRegion"], + {"count": 2, "refs": ["dcid:africaRegion", "dcid:southOfSahara"]}, + id="named", ), pytest.param( [f"region/R{i:02d}" for i in range(12)], - "12 entities: " + ", ".join(f"region/R{i:02d}" for i in range(10)) + " (truncated)", - id="truncated", + {"count": 12, "refs": [f"dcid:region/R{i:02d}" for i in range(10)]}, + id="capped", ), ], ) def test_destination_currency_omits_the_recipients_no_currency_names( - entry: str, regions: list[str], named: str + entry: str, regions: list[str], counted: dict[str, object] ): - """Kenya converts; the regions' rows are left out and one warning names them. Bound - explicitly, they raise no second `coverage_below_scope` warning.""" + """Kenya converts; the regions' rows are left out and one warning counts them and carries + the first five by ref, sorted. Bound explicitly, they raise no second + `coverage_below_scope` warning.""" result = _destination_currency_result(entry, ["country/KEN", *regions]) assert _served_recipients(result) == {"country/KEN"} omitted = [w for w in result.warnings if w["code"] == "currency_unmapped_omitted"] assert [w["severity"] for w in omitted] == ["warn"] - assert omitted[0]["message"].endswith(named) + assert omitted[0]["omitted_entities"] == counted assert not any(w["code"] == "coverage_below_scope" for w in result.warnings) if isinstance(result, MaterialisationResult): (conversion,) = result.trace.conversion or () diff --git a/tests/materialise/test_pipeline.py b/tests/materialise/test_pipeline.py index dda94489..e56829e6 100644 --- a/tests/materialise/test_pipeline.py +++ b/tests/materialise/test_pipeline.py @@ -4,8 +4,10 @@ import io import time +from collections.abc import Callable, Sequence from dataclasses import replace from types import SimpleNamespace +from typing import get_type_hints import httpx import pyarrow.parquet as pq @@ -21,6 +23,7 @@ materialise_spec, ) from materialise import finish as finish_module +from materialise.codes import CountedEntityYears, CountedRefs, MaterialisationWarning from materialise.content_version import mint_version from materialise.errors import ( DefaultFacetUndefinedError, @@ -873,7 +876,7 @@ def test_latest_window_over_a_huge_raw_response_does_not_early_refuse_and_succee RequestedWindow(kind="all"), "default", "The served coverage falls short of the requested scope. " - "No observations were served for country/KEN.", + "No observations were served for 1 requested entity.", id="entity_shortfall", ), pytest.param( @@ -921,7 +924,7 @@ def test_coverage_below_scope_warning( assert warning["message"] == expected_message -def test_coverage_below_scope_names_only_the_unserved_entity(): +def test_coverage_below_scope_carries_only_the_unserved_entity(): csv_body = _CSV_HEADER + _csv_row(recipient="country/TGO", year="2015", value="1000") sdmx, _requests = _sdmx_client(csv_body) graph = FakeGraphReader(facets={(_BARE_STAT_VAR, "country/TGO"): [_FACET]}) @@ -942,11 +945,10 @@ def test_coverage_below_scope_names_only_the_unserved_entity(): ) warning = {w["code"]: w for w in result.warnings}["coverage_below_scope"] - assert "country/SEN" in warning["message"] - assert "country/TGO" not in warning["message"] + assert warning["unserved_entities"] == {"count": 1, "refs": ["dcid:country/SEN"]} -def test_coverage_below_scope_caps_named_entities_at_five(): +def test_coverage_below_scope_caps_carried_entities_at_five(): bound = ["country/TGO"] + [f"country/Unserved{i}" for i in range(6)] csv_body = _CSV_HEADER + _csv_row(recipient="country/TGO", year="2015", value="1000") sdmx, _requests = _sdmx_client(csv_body) @@ -967,9 +969,84 @@ def test_coverage_below_scope_caps_named_entities_at_five(): warning = {w["code"]: w for w in result.warnings}["coverage_below_scope"] assert warning["message"] == ( "The served coverage falls short of the requested scope. " - "No observations were served for country/Unserved0, country/Unserved1, " - "country/Unserved2, country/Unserved3, country/Unserved4 and 1 more." + "No observations were served for 6 requested entities." ) + assert warning["unserved_entities"] == { + "count": 6, + "refs": [f"dcid:country/Unserved{i}" for i in range(5)], + } + + +def _coverage_warning( + *, + unserved: Sequence[str] = (), + off_cadence: Sequence[str] = (), + out_of_window: Sequence[str] = (), +) -> MaterialisationWarning: + return finish_module._coverage_below_scope_warning( + unserved, + countries_fallback=False, + off_cadence=off_cadence, + out_of_window=out_of_window, + member_count=9, + ) + + +# The writer of each counted field `MaterialisationWarning` declares, given the refs, bare +# entity dcids or member `dcid:` refs, that it is to count. +_COUNTED_FIELD_WRITERS: dict[str, Callable[[Sequence[str]], MaterialisationWarning]] = { + "unserved_entities": lambda refs: _coverage_warning(unserved=refs), + "off_cadence_members": lambda refs: _coverage_warning(off_cadence=refs), + "out_of_window_members": lambda refs: _coverage_warning(out_of_window=refs), + "omitted_entities": finish_module.currency_unmapped_omitted_warning, + "omitted_entity_years": lambda refs: finish_module.deflator_missing_omitted_warning( + (entity, 2020) for entity in refs + ), +} +_COUNTED_FIELDS = frozenset( + key + for key, hint in get_type_hints(MaterialisationWarning).items() + if hint in (CountedRefs, CountedEntityYears) +) +_PIN_REFS = { + "off_cadence_members": ["dcid:YouthRate", "dcid:AdultRate"], + "out_of_window_members": ["dcid:YouthRate", "dcid:AdultRate"], +} + + +def test_every_counted_warning_field_has_a_pinned_writer(): + """A new counted field on `MaterialisationWarning` fails here until its writer joins + `test_a_stored_warning_message_names_none_of_its_refs`.""" + assert set(_COUNTED_FIELD_WRITERS) == _COUNTED_FIELDS + + +@pytest.mark.parametrize("field", sorted(_COUNTED_FIELD_WRITERS)) +def test_a_stored_warning_message_names_none_of_its_refs(field: str): + """A stored message that names a ref keeps that name after the label changes, so each + writer of a counted field counts its refs in the message and carries them in the field.""" + refs = _PIN_REFS.get(field, ["africaRegion", "country/PSE"]) + + warning = _COUNTED_FIELD_WRITERS[field](refs) + + assert field in warning + assert not [ref for ref in refs if ref.removeprefix("dcid:") in warning["message"]] + + +def test_a_deflator_warning_carries_the_first_ten_entities_with_their_years_as_runs(): + """The ten entities the old message named survive, each with its years as runs.""" + omitted = [("country/TWN", year) for year in (2004, 1990, 1991, 1992)] + omitted += [(f"country/C{i:02d}", 2022) for i in range(11)] + + warning = finish_module.deflator_missing_omitted_warning(omitted) + + assert warning["omitted_entity_years"] == { + "count": 12, + "entities": [{"ref": f"dcid:country/C{i:02d}", "years": ["2022"]} for i in range(10)], + } + assert finish_module.deflator_missing_omitted_warning(omitted[:4])["omitted_entity_years"] == { + "count": 1, + "entities": [{"ref": "dcid:country/TWN", "years": ["1990-1992", "2004"]}], + } def test_as_issue_carries_the_wrapped_errors_code_and_message(): diff --git a/tests/presentation_store/test_frame_store.py b/tests/presentation_store/test_frame_store.py index 5d6cfc09..3fddfd13 100644 --- a/tests/presentation_store/test_frame_store.py +++ b/tests/presentation_store/test_frame_store.py @@ -808,15 +808,16 @@ def test_load_labels_a_node_of_several_names_as_the_node_door_does(bind_catalogu assert resolved.labels["country/ETH"] == "Federal Democratic Republic of Ethiopia" -def test_load_labels_the_group_members_a_coverage_warning_names(): - """A member that served nothing holds no row, so the refs a `coverage_below_scope` - warning names join the frame's one label batch.""" +def test_load_labels_the_entities_and_members_a_coverage_warning_counts(): + """An entity or member that served nothing holds no row, so the refs a + `coverage_below_scope` warning carries join the frame's one label batch.""" snapshots, artifacts, frame_ref, _digest = _happy_path_fixture() version = snapshots.versions[(frame_ref.spec_id, frame_ref.content_version)] warning = { "code": "coverage_below_scope", "severity": "warn", "message": "Short of scope.", + "unserved_entities": {"count": 1, "refs": ["dcid:country/SEN"]}, "member_count": 3, "off_cadence_members": {"count": 1, "refs": ["dcid:YouthRate"]}, "out_of_window_members": {"count": 1, "refs": ["dcid:AdultRate"]}, @@ -829,18 +830,20 @@ def test_load_labels_the_group_members_a_coverage_warning_names(): "MalariaGrantDisbursements": "Malaria Grant Disbursements", "YouthRate": "Youth rate", "AdultRate": "Adult rate", + "country/SEN": "Senegal", } ) store = _store(snapshots=snapshots, artifacts=artifacts, graph=graph) resolved = store.load(frame_ref=frame_ref) - assert (resolved.labels["dcid:YouthRate"], resolved.labels["dcid:AdultRate"]) == ( - "Youth rate", - "Adult rate", - ) + assert ( + resolved.labels["dcid:country/SEN"], + resolved.labels["dcid:YouthRate"], + resolved.labels["dcid:AdultRate"], + ) == ("Senegal", "Youth rate", "Adult rate") assert graph.calls == 1 - assert {"YouthRate", "AdultRate"} <= set(graph.node_arcs_many_calls[0]) + assert {"country/SEN", "YouthRate", "AdultRate"} <= set(graph.node_arcs_many_calls[0]) _WDI_GDP = "worldBank/NY_GDP_MKTP_CD"